From pypy.commits at gmail.com Wed Mar 1 02:27:53 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 28 Feb 2017 23:27:53 -0800 (PST) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <58b677f9.124b2e0a.7ed29.49b1@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r866:2970c7ecb9a9 Date: 2017-03-01 08:27 +0100 http://bitbucket.org/pypy/pypy.org/changeset/2970c7ecb9a9/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $66667 of $105000 (63.5%) + $66677 of $105000 (63.5%)
@@ -23,7 +23,7 @@
  • From pypy.commits at gmail.com Wed Mar 1 06:43:06 2017 From: pypy.commits at gmail.com (stevie_92) Date: Wed, 01 Mar 2017 03:43:06 -0800 (PST) Subject: [pypy-commit] pypy fix-global: Fixed bug translating SyntaxWarning to SyntaxError Message-ID: <58b6b3ca.0e572e0a.96d52.5880@mx.google.com> Author: Stefan Beyer Branch: fix-global Changeset: r90434:2e54fda87e45 Date: 2017-02-28 18:14 +0100 http://bitbucket.org/pypy/pypy/changeset/2e54fda87e45/ Log: Fixed bug translating SyntaxWarning to SyntaxError Tests in lib- python/3/test/test_global.py are now sucessful diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py --- a/pypy/interpreter/astcompiler/misc.py +++ b/pypy/interpreter/astcompiler/misc.py @@ -9,7 +9,7 @@ try: warnings.warn_explicit(msg, SyntaxWarning, fn, lineno) except SyntaxWarning: - raise SyntaxError(msg, fn, lineno, offset) + raise SyntaxError(msg, (fn, lineno, offset, msg)) """, filename=__file__) _emit_syntax_warning = app.interphook("syntax_warning") del app From pypy.commits at gmail.com Wed Mar 1 06:43:01 2017 From: pypy.commits at gmail.com (rlamy) Date: Wed, 01 Mar 2017 03:43:01 -0800 (PST) Subject: [pypy-commit] pypy default: Merged in robert-zaremba/pypy/fix-global (pull request #525) Message-ID: <58b6b3c5.0528190a.a556c.54be@mx.google.com> Author: Ronan Lamy Branch: Changeset: r90436:205b129940e0 Date: 2017-03-01 11:42 +0000 http://bitbucket.org/pypy/pypy/changeset/205b129940e0/ Log: Merged in robert-zaremba/pypy/fix-global (pull request #525) Fixed bug translating SyntaxWarning to SyntaxError diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py --- a/pypy/interpreter/astcompiler/misc.py +++ b/pypy/interpreter/astcompiler/misc.py @@ -9,7 +9,7 @@ try: warnings.warn_explicit(msg, SyntaxWarning, fn, lineno) except SyntaxWarning: - raise SyntaxError(msg, fn, lineno, offset) + raise SyntaxError(msg, (fn, lineno, offset, msg)) """, filename=__file__) _emit_syntax_warning = app.interphook("syntax_warning") del app diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py --- a/pypy/interpreter/astcompiler/test/test_misc.py +++ b/pypy/interpreter/astcompiler/test/test_misc.py @@ -12,3 +12,23 @@ assert mangle("__foo", "__Bar") == "_Bar__foo" assert mangle("__foo", "___") == "__foo" assert mangle("___foo", "__Bar") == "_Bar___foo" + +def app_test_warning_to_error_translation(): + import warnings + + with warnings.catch_warnings(): + warnings.filterwarnings("error", module="") + statement = """\ +def wrong1(): + a = 1 + b = 2 + global a + global b +""" + try: + compile(statement, '', 'exec') + except SyntaxError as err: + assert err.lineno is not None + assert err.filename is not None + assert err.offset is not None + assert err.message is not None From pypy.commits at gmail.com Wed Mar 1 06:43:09 2017 From: pypy.commits at gmail.com (stevie_92) Date: Wed, 01 Mar 2017 03:43:09 -0800 (PST) Subject: [pypy-commit] pypy fix-global: Added tests for bugfix Message-ID: <58b6b3cd.02aa190a.b5f80.574f@mx.google.com> Author: Stefan Beyer Branch: fix-global Changeset: r90435:b8d66cc71819 Date: 2017-02-28 21:59 +0100 http://bitbucket.org/pypy/pypy/changeset/b8d66cc71819/ Log: Added tests for bugfix diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py --- a/pypy/interpreter/astcompiler/test/test_misc.py +++ b/pypy/interpreter/astcompiler/test/test_misc.py @@ -12,3 +12,23 @@ assert mangle("__foo", "__Bar") == "_Bar__foo" assert mangle("__foo", "___") == "__foo" assert mangle("___foo", "__Bar") == "_Bar___foo" + +def app_test_warning_to_error_translation(): + import warnings + + with warnings.catch_warnings(): + warnings.filterwarnings("error", module="") + statement = """\ +def wrong1(): + a = 1 + b = 2 + global a + global b +""" + try: + compile(statement, '', 'exec') + except SyntaxError as err: + assert err.lineno is not None + assert err.filename is not None + assert err.offset is not None + assert err.message is not None From pypy.commits at gmail.com Wed Mar 1 08:15:52 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 01 Mar 2017 05:15:52 -0800 (PST) Subject: [pypy-commit] pypy default: update idea list for gsoc 2017 Message-ID: <58b6c988.02a5190a.502c7.5db9@mx.google.com> Author: Richard Plangger Branch: Changeset: r90437:ad2578b00750 Date: 2017-03-01 14:14 +0100 http://bitbucket.org/pypy/pypy/changeset/ad2578b00750/ Log: update idea list for gsoc 2017 diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,9 +1,24 @@ -Potential project list +Potential Project List ====================== -========================== +Google Summer of Code 2017 +-------------------------- + +PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. + + +* **Compile to multiple shared objects**: PyPy currently emits a small executable file and a large shared object file. To reduce the base interpreter size we think it would be helpful to have several shared object files that can be dynamically loaded if the module is needed. + +* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. + +* **VMProf visualisations**: vmprof just shows a flamgraph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). + +* **Implicit typing in RPython**: PyPy wants to have better ways to specify the signature andclass attribute types in RPython. See more information about this topic below on this page. + +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either Universities provide such hardware in any other case we potentially can lend the VR hardware setup. + Simple tasks for newcomers -========================== +-------------------------- * Tkinter module missing support for threads: https://bitbucket.org/pypy/pypy/issue/1929/tkinter-broken-for-threaded-python-on-both @@ -15,9 +30,8 @@ https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets -================== Mid-to-large tasks -================== +------------------ Below is a list of projects that are interesting for potential contributors who are seriously interested in the PyPy project. They mostly share common @@ -81,7 +95,7 @@ module. Improving the jitviewer ------------------------- +----------------------- Analyzing performance of applications is always tricky. We have various tools, for example a `jitviewer`_ that help us analyze performance. From pypy.commits at gmail.com Wed Mar 1 08:31:24 2017 From: pypy.commits at gmail.com (antocuni) Date: Wed, 01 Mar 2017 05:31:24 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: (antocuni, rlamy, arigo around): fix test_releasebuffer by making sure that we do the correct amount of decrefs, so that the original object is correctly deallocated. Message-ID: <58b6cd2c.c59c190a.752b.5b1b@mx.google.com> Author: Antonio Cuni Branch: fix-cpyext-releasebuffer Changeset: r90438:6b28fd38f5bc Date: 2017-03-01 14:30 +0100 http://bitbucket.org/pypy/pypy/changeset/6b28fd38f5bc/ Log: (antocuni, rlamy, arigo around): fix test_releasebuffer by making sure that we do the correct amount of decrefs, so that the original object is correctly deallocated. diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -72,6 +72,8 @@ format = 'B' if view.c_format: format = rffi.charp2str(view.c_format) + incref(space, view.c_obj) # we need this incref because + # CPyBuffer.releasebuffer does a decref buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj), format=format, shape=shape, strides=strides, ndim=ndim, itemsize=view.c_itemsize, diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -14,7 +14,7 @@ ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, readbufferproc, getbufferproc, releasebufferproc, ssizessizeobjargproc) -from pypy.module.cpyext.pyobject import make_ref, decref +from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.memoryobject import fill_Py_buffer from pypy.module.cpyext.state import State @@ -333,7 +333,7 @@ self.ptr = ptr self.size = size self.w_obj = w_obj # kept alive - self.pyobj = make_ref(space, w_obj) + self.pyobj = as_pyobj(space, w_obj) self.format = format if not shape: self.shape = [size] @@ -350,6 +350,9 @@ def releasebuffer(self): if self.pyobj: + # the CPython docs mandates that you do an incref whenever you + # call bf_getbuffer. This is the corresponding decref: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer decref(self.space, self.pyobj) self.pyobj = lltype.nullptr(PyObject.TO) else: From pypy.commits at gmail.com Wed Mar 1 08:34:36 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 01 Mar 2017 05:34:36 -0800 (PST) Subject: [pypy-commit] pypy default: some changes to the ideas page Message-ID: <58b6cdec.02aa190a.b5f80.5dc4@mx.google.com> Author: Richard Plangger Branch: Changeset: r90439:0bf2eb520cbb Date: 2017-03-01 14:33 +0100 http://bitbucket.org/pypy/pypy/changeset/0bf2eb520cbb/ Log: some changes to the ideas page diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -7,13 +7,13 @@ PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. -* **Compile to multiple shared objects**: PyPy currently emits a small executable file and a large shared object file. To reduce the base interpreter size we think it would be helpful to have several shared object files that can be dynamically loaded if the module is needed. +* **Optimize PyPy Memory Usage**: PyPy currently emits a small executable file and a large shared object file. To reduce the base interpreter size we think it would be helpful to have several shared object files that can be dynamically loaded if the module is needed. There are several other potential places where we could improve. * **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. * **VMProf visualisations**: vmprof just shows a flamgraph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). -* **Implicit typing in RPython**: PyPy wants to have better ways to specify the signature andclass attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature andclass attribute types in RPython. See more information about this topic below on this page. * **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either Universities provide such hardware in any other case we potentially can lend the VR hardware setup. From pypy.commits at gmail.com Wed Mar 1 08:42:51 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 01 Mar 2017 05:42:51 -0800 (PST) Subject: [pypy-commit] pypy.org extradoc: add the new logo Message-ID: <58b6cfdb.4f9e190a.29559.5e1b@mx.google.com> Author: Richard Plangger Branch: extradoc Changeset: r867:3206d2fd3d54 Date: 2017-03-01 14:42 +0100 http://bitbucket.org/pypy/pypy.org/changeset/3206d2fd3d54/ Log: add the new logo diff --git a/image/pypy-logo2-white-background.png b/image/pypy-logo2-white-background.png new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5e4b3ed732549b0d60e655b552f31f1baadf8e14 GIT binary patch [cut] From pypy.commits at gmail.com Wed Mar 1 08:48:50 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 01 Mar 2017 05:48:50 -0800 (PST) Subject: [pypy-commit] pypy default: typo Message-ID: <58b6d142.42452e0a.a9d96.5e6c@mx.google.com> Author: Richard Plangger Branch: Changeset: r90440:8a379af198e4 Date: 2017-03-01 14:48 +0100 http://bitbucket.org/pypy/pypy/changeset/8a379af198e4/ Log: typo diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -13,7 +13,7 @@ * **VMProf visualisations**: vmprof just shows a flamgraph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature andclass attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. * **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either Universities provide such hardware in any other case we potentially can lend the VR hardware setup. From pypy.commits at gmail.com Wed Mar 1 08:50:27 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 01 Mar 2017 05:50:27 -0800 (PST) Subject: [pypy-commit] pypy default: and another change in the project idea list Message-ID: <58b6d1a3.8f1f190a.e975c.5b96@mx.google.com> Author: Richard Plangger Branch: Changeset: r90441:4763c7fec8cf Date: 2017-03-01 14:49 +0100 http://bitbucket.org/pypy/pypy/changeset/4763c7fec8cf/ Log: and another change in the project idea list diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -15,7 +15,7 @@ * **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either Universities provide such hardware in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. Simple tasks for newcomers -------------------------- From pypy.commits at gmail.com Wed Mar 1 10:10:10 2017 From: pypy.commits at gmail.com (rlamy) Date: Wed, 01 Mar 2017 07:10:10 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58b6e452.01132e0a.d7b0.60c3@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90442:7e102a870512 Date: 2017-03-01 16:09 +0100 http://bitbucket.org/pypy/pypy/changeset/7e102a870512/ Log: hg merge default diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -496,6 +496,11 @@ the rest is kept. If you return an unexpected string from ``__hex__()`` you get an exception (or a crash before CPython 2.7.13). +* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes. + CPython allows that for module subtypes, but not for e.g. ``int`` + or ``float`` subtypes. Currently PyPy does not support the + ``__class__`` attribute assignment for any non heaptype subtype. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,9 +1,24 @@ -Potential project list +Potential Project List ====================== -========================== +Google Summer of Code 2017 +-------------------------- + +PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. + + +* **Optimize PyPy Memory Usage**: PyPy currently emits a small executable file and a large shared object file. To reduce the base interpreter size we think it would be helpful to have several shared object files that can be dynamically loaded if the module is needed. There are several other potential places where we could improve. + +* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. + +* **VMProf visualisations**: vmprof just shows a flamgraph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). + +* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. + +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. + Simple tasks for newcomers -========================== +-------------------------- * Tkinter module missing support for threads: https://bitbucket.org/pypy/pypy/issue/1929/tkinter-broken-for-threaded-python-on-both @@ -15,9 +30,8 @@ https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets -================== Mid-to-large tasks -================== +------------------ Below is a list of projects that are interesting for potential contributors who are seriously interested in the PyPy project. They mostly share common @@ -81,7 +95,7 @@ module. Improving the jitviewer ------------------------- +----------------------- Analyzing performance of applications is always tricky. We have various tools, for example a `jitviewer`_ that help us analyze performance. diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -4,16 +4,24 @@ import os import py +TAR_OPTIONS = '-x -v --strip-components=2' +TAR = 'tar {options} -f {tarfile} {files}' + +def untar(tarfile, files): + cmd = TAR.format(options=TAR_OPTIONS, tarfile=tarfile, files=files) + os.system(cmd) + if sys.platform.startswith('linux'): arch = 'linux' cmd = 'wget "%s"' - tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'" + TAR_OPTIONS += ' --wildcards' + binfiles = "'*/bin/pypy' '*/bin/libpypy-c.so'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): arch = 'osx' cmd = 'curl -O "%s"' - tar = "tar -x -v --strip-components=2 -f %s '*/bin/pypy'" + binfiles = "'*/bin/pypy'" else: print 'Cannot determine the platform, please update this script' sys.exit(1) @@ -34,6 +42,7 @@ filename = 'pypy-c-%s-latest-%s.tar.bz2' % (kind, arch) url = 'http://buildbot.pypy.org/nightly/%s/%s' % (branch, filename) tmp = py.path.local.mkdtemp() +pypy_latest = tmp.join(filename) mydir = tmp.chdir() print 'Downloading pypy to', tmp if os.system(cmd % url) != 0: @@ -41,4 +50,10 @@ print 'Extracting pypy binary' mydir.chdir() -os.system(tar % tmp.join(filename)) +untar(pypy_latest, binfiles) +include_dir = py.path.local('../../include') +if include_dir.check(dir=True): + include_dir.chdir() + untar(pypy_latest, '*/include/*') +else: + print 'WARNING: could not find the include/ dir' diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py --- a/pypy/interpreter/astcompiler/misc.py +++ b/pypy/interpreter/astcompiler/misc.py @@ -9,7 +9,7 @@ try: warnings.warn_explicit(msg, SyntaxWarning, fn, lineno) except SyntaxWarning: - raise SyntaxError(msg, fn, lineno, offset) + raise SyntaxError(msg, (fn, lineno, offset, msg)) """, filename=__file__) _emit_syntax_warning = app.interphook("syntax_warning") del app diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py --- a/pypy/interpreter/astcompiler/test/test_misc.py +++ b/pypy/interpreter/astcompiler/test/test_misc.py @@ -12,3 +12,23 @@ assert mangle("__foo", "__Bar") == "_Bar__foo" assert mangle("__foo", "___") == "__foo" assert mangle("___foo", "__Bar") == "_Bar___foo" + +def app_test_warning_to_error_translation(): + import warnings + + with warnings.catch_warnings(): + warnings.filterwarnings("error", module="") + statement = """\ +def wrong1(): + a = 1 + b = 2 + global a + global b +""" + try: + compile(statement, '', 'exec') + except SyntaxError as err: + assert err.lineno is not None + assert err.filename is not None + assert err.offset is not None + assert err.message is not None diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4636,3 +4636,28 @@ return i self.meta_interp(f, [10]) + + @py.test.skip("loops!") + def test_finalizer_bug(self): + from rpython.rlib import rgc + driver = JitDriver(greens=[], reds=[]) + class Fin(object): + @rgc.must_be_light_finalizer + def __del__(self): + holder[0].field = 7 + class Un(object): + def __init__(self): + self.field = 0 + holder = [Un()] + + def f(): + while True: + driver.jit_merge_point() + holder[0].field = 0 + Fin() + if holder[0].field: + break + return holder[0].field + + f() # finishes + self.meta_interp(f, []) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -1138,7 +1138,7 @@ # Check if the object at 'addr' is young. if not self.is_valid_gc_object(addr): return False # filter out tagged pointers explicitly. - if self.nursery <= addr < self.nursery_top: + if self.is_in_nursery(addr): return True # addr is in the nursery # Else, it may be in the set 'young_rawmalloced_objects' return (bool(self.young_rawmalloced_objects) and diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -359,6 +359,10 @@ def ll_str(self, nothing): raise AssertionError("unreachable code") impossible_repr = VoidRepr() +class __extend__(pairtype(Repr, VoidRepr)): + def convert_from_to((r_from, r_to), v, llops): + return inputconst(lltype.Void, None) + class SimplePointerRepr(Repr): "Convenience Repr for simple ll pointer types with no operation on them." diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py --- a/rpython/rtyper/test/test_rpbc.py +++ b/rpython/rtyper/test/test_rpbc.py @@ -1746,6 +1746,29 @@ res = self.interpret(g, [1]) assert res == True + def test_convert_from_anything_to_impossible(self): + def f1(): + return 42 + def f2(): + raise ValueError + def f3(): + raise ValueError + def f(i): + if i > 5: + f = f2 + else: + f = f3 + try: + f() + except ValueError: + pass + if i > 1: + f = f2 + else: + f = f1 + return f() + self.interpret(f, [-5]) + # ____________________________________________________________ def test_hlinvoke_simple(): diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1123,7 +1123,7 @@ def compile(self, entry_point, no__thread=True): t = TranslationContext(self.config) - t.config.translation.gc = "semispace" + t.config.translation.gc = "incminimark" t.config.translation.gcrootfinder = self.gcrootfinder t.config.translation.thread = True t.config.translation.no__thread = no__thread From pypy.commits at gmail.com Wed Mar 1 10:30:24 2017 From: pypy.commits at gmail.com (antocuni) Date: Wed, 01 Mar 2017 07:30:24 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: CPyBuffer is used in various places but not all of them needs the additional decref: make sure that we do it only when it's needed, i.e. when we wrap the new style bf_getbuffer Message-ID: <58b6e910.14db190a.6f008.6266@mx.google.com> Author: Antonio Cuni Branch: fix-cpyext-releasebuffer Changeset: r90443:8229228f828e Date: 2017-03-01 16:29 +0100 http://bitbucket.org/pypy/pypy/changeset/8229228f828e/ Log: CPyBuffer is used in various places but not all of them needs the additional decref: make sure that we do it only when it's needed, i.e. when we wrap the new style bf_getbuffer diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -72,8 +72,6 @@ format = 'B' if view.c_format: format = rffi.charp2str(view.c_format) - incref(space, view.c_obj) # we need this incref because - # CPyBuffer.releasebuffer does a decref buf = CPyBuffer(space, view.c_buf, view.c_len, from_ref(space, view.c_obj), format=format, shape=shape, strides=strides, ndim=ndim, itemsize=view.c_itemsize, diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -327,8 +327,9 @@ _immutable_ = True def __init__(self, space, ptr, size, w_obj, format='B', shape=None, - strides=None, ndim=1, itemsize=1, readonly=True, - releasebufferproc=rffi.cast(rffi.VOIDP, 0)): + strides=None, ndim=1, itemsize=1, readonly=True, + needs_decref=False, + releasebufferproc=rffi.cast(rffi.VOIDP, 0)): self.space = space self.ptr = ptr self.size = size @@ -346,14 +347,13 @@ self.ndim = ndim self.itemsize = itemsize self.readonly = readonly + self.needs_decref = needs_decref self.releasebufferproc = releasebufferproc def releasebuffer(self): if self.pyobj: - # the CPython docs mandates that you do an incref whenever you - # call bf_getbuffer. This is the corresponding decref: - # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer - decref(self.space, self.pyobj) + if self.needs_decref: + decref(self.space, self.pyobj) self.pyobj = lltype.nullptr(PyObject.TO) else: #do not call twice @@ -477,10 +477,15 @@ format = rffi.charp2str(pybuf.c_format) else: format = 'B' + # the CPython docs mandates that you do an incref whenever you call + # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't + # leak we release the buffer: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer buf = CPyBuffer(space, ptr, size, w_self, format=format, ndim=ndim, shape=shape, strides=strides, itemsize=pybuf.c_itemsize, readonly=widen(pybuf.c_readonly), + needs_decref=True, releasebufferproc = rbp) fq.register_finalizer(buf) return space.newbuffer(buf) From pypy.commits at gmail.com Wed Mar 1 12:28:07 2017 From: pypy.commits at gmail.com (Raemi) Date: Wed, 01 Mar 2017 09:28:07 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) implement a synchronisation scheme for safepoints (WIP) Message-ID: <58b704a7.4f9e190a.29559.6948@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90444:65be98dc2aee Date: 2017-03-01 18:27 +0100 http://bitbucket.org/pypy/pypy/changeset/65be98dc2aee/ Log: (arigo, remi) implement a synchronisation scheme for safepoints (WIP) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -72,7 +72,7 @@ from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib import rthread +from rpython.rlib import rgil, rthread from rpython.memory.gc.minimarkpage import out_of_memory # @@ -191,7 +191,7 @@ NURSERY_FREE = rthread.ThreadLocalField(llmemory.Address, 'nursery_free') NURSERY_TOP = rthread.ThreadLocalField(llmemory.Address, 'nursery_top') -NEXT_NUBLOCK = rthread.ThreadLocalField(llmemory.Address, 'next_nublock') + # ____________________________________________________________ @@ -438,11 +438,11 @@ self.old_objects_pointing_to_pinned = self.AddressStack() self.updated_old_objects_pointing_to_pinned = False # - # Allocate lock(s) - ll_lock = lltype.malloc(rthread.TLOCKP.TO, flavor='raw', - track_allocation=False) - rthread.c_thread_lock_init(ll_lock) - self.ll_lock = ll_lock + # # Allocate lock(s) + # ll_lock = lltype.malloc(rthread.TLOCKP.TO, flavor='raw', + # track_allocation=False) + # rthread.c_thread_lock_init(ll_lock) + # self.ll_lock = ll_lock # # Allocate a nursery. In case of auto_nursery_size, start by # allocating a very small nursery, enough to do things like look @@ -650,9 +650,6 @@ get_nursery_top = staticmethod(NURSERY_TOP.getraw) set_nursery_top = staticmethod(NURSERY_TOP.setraw) - get_next_nublock = staticmethod(NEXT_NUBLOCK.getraw) - set_next_nublock = staticmethod(NEXT_NUBLOCK.setraw) - @property def nursery_top(self): XXX # fix caller @@ -859,7 +856,8 @@ major collection, and finally reserve totalsize bytes. """ - rthread.acquire_NOAUTO(self.ll_lock, 1) + # rthread.acquire_NOAUTO(self.ll_lock, 1) + rgil.enter_master_section() minor_collection_count = 0 while True: @@ -898,6 +896,8 @@ self.set_nursery_free(self.nursery_barriers.popleft()) self.set_nursery_top(self.nursery_barriers.popleft()) else: + rgil.master_request_safepoint() + minor_collection_count += 1 if minor_collection_count == 1: self.minor_collection_with_major_progress() @@ -936,7 +936,8 @@ self.set_nursery_free(self.get_nursery_top() - self.debug_tiny_nursery) # - rthread.release_NOAUTO(self.ll_lock) + rgil.leave_master_section() + # rthread.release_NOAUTO(self.ll_lock) return result collect_and_reserve._dont_inline_ = True diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -227,9 +227,11 @@ tl_shadowstack = rthread.ThreadLocalField(llmemory.Address, 'shadowstack') + tl_synclock = rthread.ThreadLocalField(lltype.Signed, 'synclock') def thread_setup(): allocate_shadow_stack() + tl_synclock.get_or_make_raw() def thread_run(): # If it's the first time we see this thread, allocate diff --git a/rpython/rlib/rgil.py b/rpython/rlib/rgil.py --- a/rpython/rlib/rgil.py +++ b/rpython/rlib/rgil.py @@ -22,7 +22,7 @@ _nowrapper=True, sandboxsafe=True, compilation_info=eci) -_gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed, +_gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Void, _nowrapper=True, sandboxsafe=True, compilation_info=eci) @@ -38,6 +38,20 @@ _nowrapper=True, sandboxsafe=True, compilation_info=eci) +enter_master_section = llexternal( + 'RPyGilEnterMasterSection', [], lltype.Void, + _nowrapper=True, sandboxsafe=True, + compilation_info=eci) + +leave_master_section = llexternal( + 'RPyGilLeaveMasterSection', [], lltype.Void, + _nowrapper=True, sandboxsafe=True, + compilation_info=eci) + +master_request_safepoint = llexternal( + 'RPyGilMasterRequestSafepoint', [], lltype.Void, + _nowrapper=True, sandboxsafe=True, + compilation_info=eci) # ____________________________________________________________ @@ -133,10 +147,11 @@ # explicitly release the gil, in a way that tries to give more # priority to other threads (as opposed to continuing to run in # the same thread). - if _gil_yield_thread(): - from rpython.rlib import rthread - rthread.gc_thread_run() - _after_thread_switch() + # if _gil_yield_thread(): + # from rpython.rlib import rthread + # rthread.gc_thread_run() + # _after_thread_switch() + _gil_yield_thread() yield_thread._gctransformer_hint_close_stack_ = True yield_thread._dont_reach_me_in_del_ = True yield_thread._dont_inline_ = True diff --git a/rpython/translator/c/src/thread.c b/rpython/translator/c/src/thread.c --- a/rpython/translator/c/src/thread.c +++ b/rpython/translator/c/src/thread.c @@ -9,11 +9,9 @@ #include "common_header.h" #endif -#ifdef PYPY_USE_ASMGCC # include "common_header.h" # include "structdef.h" # include "forwarddecl.h" -#endif #ifdef _WIN32 #include "src/thread_nt.c" diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h --- a/rpython/translator/c/src/thread.h +++ b/rpython/translator/c/src/thread.h @@ -30,8 +30,15 @@ #endif /* !_WIN32 */ RPY_EXTERN void RPyGilAllocate(void); -RPY_EXTERN long RPyGilYieldThread(void); -RPY_EXTERN void RPyGilAcquireSlowPath(long); +RPY_EXTERN void RPyGilYieldThreadSlowPath(void); +RPY_EXTERN void RPyGilAcquireSlowPath(void); +RPY_EXTERN void RPyGilReleaseSlowPath(void); + +RPY_EXTERN void RPyGilEnterMasterSection(void); +RPY_EXTERN void RPyGilLeaveMasterSection(void); +RPY_EXTERN void RPyGilMasterRequestSafepoint(void); + + #define RPyGilAcquire _RPyGilAcquire #define RPyGilRelease _RPyGilRelease #define RPyFetchFastGil _RPyFetchFastGil @@ -43,21 +50,33 @@ #endif //RPY_EXTERN long rpy_fastgil; +#include "threadlocal.h" -static inline void _RPyGilAcquire(void) { -// long old_fastgil = pypy_lock_test_and_set(&rpy_fastgil, 1); -// if (old_fastgil != 0) -// RPyGilAcquireSlowPath(old_fastgil); -} -static inline void _RPyGilRelease(void) { -// assert(RPY_FASTGIL_LOCKED(rpy_fastgil)); -// pypy_lock_release(&rpy_fastgil); -} +#define _RPyGilAcquire() do { \ + if (!__sync_bool_compare_and_swap( \ + &RPY_THREADLOCALREF_GET(synclock), 0L, 1L)) \ + RPyGilAcquireSlowPath(); \ + } while (0) + +#define _RPyGilRelease() do { \ + assert(RPY_THREADLOCALREF_GET(synclock) != 0L); \ + if (!__sync_bool_compare_and_swap( \ + &RPY_THREADLOCALREF_GET(synclock), 1L, 0L)) \ + RPyGilReleaseSlowPath(); \ + } while (0) + static inline long *_RPyFetchFastGil(void) { abort(); // return &rpy_fastgil; } +#define RPyGilYieldThread() do { \ + assert(RPY_THREADLOCALREF_GET(synclock) & 1L); \ + if (RPY_THREADLOCALREF_GET(synclock) == 3L) { \ + RPyGilYieldThreadSlowPath(); \ + } \ + } while (0) + typedef unsigned char rpy_spinlock_t; static inline void rpy_spinlock_acquire(rpy_spinlock_t *p) { diff --git a/rpython/translator/c/src/thread_gil.c b/rpython/translator/c/src/thread_gil.c --- a/rpython/translator/c/src/thread_gil.c +++ b/rpython/translator/c/src/thread_gil.c @@ -1,239 +1,142 @@ +#include +#include +#include +#include "threadlocal.h" -/* Idea: +static pthread_mutex_t master_mutex; +static pthread_mutex_t sync_mutex; +static pthread_cond_t sync_cond; - - "The GIL" is a composite concept. There are two locks, and "the - GIL is locked" when both are locked. +static long counter_of_threes = 0; - - The first lock is a simple global variable 'rpy_fastgil'. With - shadowstack, we use the most portable definition: 0 means unlocked - and != 0 means locked. With asmgcc, 0 means unlocked but only 1 - means locked. A different value means unlocked too, but the value - is used by the JIT to contain the stack top for stack root scanning. - - - The second lock is a regular mutex. In the fast path, it is never - unlocked. Remember that "the GIL is unlocked" means that either - the first or the second lock is unlocked. It should never be the - case that both are unlocked at the same time. - - - Let's call "thread 1" the thread with the GIL. Whenever it does an - external function call, it sets 'rpy_fastgil' to 0 (unlocked). - This is the cheapest way to release the GIL. When it returns from - the function call, this thread attempts to atomically change - 'rpy_fastgil' to 1. In the common case where it works, thread 1 - has got the GIL back and so continues to run. - - - Say "thread 2" is eagerly waiting for thread 1 to become blocked in - some long-running call. Regularly, it checks if 'rpy_fastgil' is 0 - and tries to atomically change it to 1. If it succeeds, it means - that the GIL was not previously locked. Thread 2 has now got the GIL. - - - If there are more than 2 threads, the rest is really sleeping by - waiting on the 'mutex_gil_stealer' held by thread 2. - - - An additional mechanism is used for when thread 1 wants to - explicitly yield the GIL to thread 2: it does so by releasing - 'mutex_gil' (which is otherwise not released) but keeping the - value of 'rpy_fastgil' to 1. -*/ - - -/* The GIL is initially released; see pypy_main_function(), which calls - RPyGilAcquire/RPyGilRelease. The point is that when building - RPython libraries, they can be a collection of regular functions that - also call RPyGilAcquire/RPyGilRelease; see test_standalone.TestShared. -*/ -long rpy_fastgil = 0; -static long rpy_waiting_threads = -42; /* GIL not initialized */ -static volatile int rpy_early_poll_n = 0; -static mutex1_t mutex_gil_stealer; -static mutex2_t mutex_gil; +static long rpy_initialize = -42; static void rpy_init_mutexes(void) { - mutex1_init(&mutex_gil_stealer); - mutex2_init_locked(&mutex_gil); - rpy_waiting_threads = 0; + int err = pthread_mutex_init(&master_mutex, NULL); + if (err) + abort(); + + err = pthread_mutex_init(&sync_mutex, NULL); + if (err) + abort(); + + err = pthread_cond_init(&sync_cond, NULL); + if (err) + abort(); + + counter_of_threes = 0; // XXX: fork? + rpy_initialize = 0; } void RPyGilAllocate(void) { -// if (rpy_waiting_threads < 0) { -// assert(rpy_waiting_threads == -42); -// rpy_init_mutexes(); + if (rpy_initialize < 0) { + assert(rpy_initialize == -42); + rpy_init_mutexes(); #ifdef HAVE_PTHREAD_ATFORK -// pthread_atfork(NULL, NULL, rpy_init_mutexes); + pthread_atfork(NULL, NULL, rpy_init_mutexes); #endif -// } + } } -static void check_and_save_old_fastgil(long old_fastgil) + +void RPyGilAcquireSlowPath(void) { - assert(RPY_FASTGIL_LOCKED(rpy_fastgil)); + assert(RPY_THREADLOCALREF_GET(synclock) == 2); -#ifdef PYPY_USE_ASMGCC - if (old_fastgil != 0) { - /* this case only occurs from the JIT compiler */ - struct pypy_ASM_FRAMEDATA_HEAD0 *new = - (struct pypy_ASM_FRAMEDATA_HEAD0 *)old_fastgil; - struct pypy_ASM_FRAMEDATA_HEAD0 *root = &pypy_g_ASM_FRAMEDATA_HEAD; - struct pypy_ASM_FRAMEDATA_HEAD0 *next = root->as_next; - new->as_next = next; - new->as_prev = root; - root->as_next = new; - next->as_prev = new; - } -#else - assert(old_fastgil == 0); -#endif + /* wait until the master leaves the safe point */ + pthread_mutex_lock(&master_mutex); + RPY_THREADLOCALREF_GET(synclock) = 1; + pthread_mutex_unlock(&master_mutex); } -#define RPY_GIL_POKE_MIN 40 -#define RPY_GIL_POKE_MAX 400 +void RPyGilReleaseSlowPath(void) +{ + assert(RPY_THREADLOCALREF_GET(synclock) == 3); -void RPyGilAcquireSlowPath(long old_fastgil) + pthread_mutex_lock(&sync_mutex); + + /* we are one of the THREES that the master is waiting for. Decrease the + * counter and signal the master if we are the last. */ + counter_of_threes--; + if (counter_of_threes == 0) + pthread_cond_signal(&sync_cond); + + /* set to TWO, so that Acquire above will wait until the master is finished + * with its safe point */ + RPY_THREADLOCALREF_GET(synclock) = 2; + pthread_mutex_unlock(&sync_mutex); + // continue without GIL +} + +void RPyGilYieldThreadSlowPath(void) { - /* Acquires the GIL. This assumes that we already did: + RPyGilRelease(); + RPyGilAcquire(); +} - old_fastgil = pypy_lock_test_and_set(&rpy_fastgil, 1); - */ - if (!RPY_FASTGIL_LOCKED(old_fastgil)) { - /* The fastgil was not previously locked: success. - 'mutex_gil' should still be locked at this point. - */ - } - else { - /* Otherwise, another thread is busy with the GIL. */ - int n; - long old_waiting_threads; +void RPyGilEnterMasterSection(void) +{ + RPyGilRelease(); + pthread_mutex_lock(&master_mutex); +} - if (rpy_waiting_threads < 0) { - /* I tried to have RPyGilAllocate() called from - * here, but it fails occasionally on an example - * (2.7/test/test_threading.py). I think what occurs is - * that if one thread runs RPyGilAllocate(), it still - * doesn't have the GIL; then the other thread might fork() - * at precisely this moment, killing the first thread. - */ - fprintf(stderr, "Fatal RPython error: a thread is trying to wait " - "for the GIL, but the GIL was not initialized\n" - "(For PyPy, see " - "https://bitbucket.org/pypy/pypy/issues/2274)\n"); +void RPyGilLeaveMasterSection(void) +{ + pthread_mutex_unlock(&master_mutex); + RPyGilAcquire(); +} + +void RPyGilMasterRequestSafepoint(void) +{ + pthread_mutex_lock(&sync_mutex); + assert(counter_of_threes == 0); + + /* signal all threads to enter safepoints */ + OP_THREADLOCALREF_ACQUIRE(/* */); + + struct pypy_threadlocal_s *t = NULL; + while (1) { + OP_THREADLOCALREF_ENUM(t, t); + if (t == NULL) + break; + + retry: + switch (t->synclock) { + case 3: + assert(!"unexpected synclock=3 found"); abort(); - } - - /* Register me as one of the threads that is actively waiting - for the GIL. The number of such threads is found in - rpy_waiting_threads. */ - old_waiting_threads = atomic_increment(&rpy_waiting_threads); - - /* Early polling: before entering the waiting queue, we check - a certain number of times if the GIL becomes free. The - motivation for this is issue #2341. Note that we do this - polling even if there are already other threads in the - queue, and one of thesee threads is the stealer. This is - because the stealer is likely sleeping right now. There - are use cases where the GIL will really be released very - soon after RPyGilAcquireSlowPath() is called, so it's worth - always doing this check. - - To avoid falling into bad cases, we "randomize" the number - of iterations: we loop N times, where N is choosen between - RPY_GIL_POKE_MIN and RPY_GIL_POKE_MAX. - */ - n = rpy_early_poll_n * 2 + 1; - while (n >= RPY_GIL_POKE_MAX) - n -= (RPY_GIL_POKE_MAX - RPY_GIL_POKE_MIN); - rpy_early_poll_n = n; - while (n >= 0) { - n--; - if (old_waiting_threads != rpy_waiting_threads) { - /* If the number changed, it is because another thread - entered or left this function. In that case, stop - this loop: if another thread left it means the GIL - has been acquired by that thread; if another thread - entered there is no point in running the present - loop twice. */ + case 2: + /* thread running in C code, already knows we want a safepoint */ + break; + case 0: + /* thread running in C code, make sure it checks for and enters + * the safepoint before acquiring the "gil" again */ + if (__sync_bool_compare_and_swap(&t->synclock, 0, 2)) + break; + goto retry; + case 1: + /* thread running normally, place request to enter safepoint */ + if (__sync_bool_compare_and_swap(&t->synclock, 1, 3)) { + counter_of_threes++; + t->nursery_top = NULL; break; } - RPy_YieldProcessor(); - RPy_CompilerMemoryBarrier(); + goto retry; + } + } + OP_THREADLOCALREF_RELEASE(/* */); - if (!RPY_FASTGIL_LOCKED(rpy_fastgil)) { - old_fastgil = pypy_lock_test_and_set(&rpy_fastgil, 1); - if (!RPY_FASTGIL_LOCKED(old_fastgil)) { - /* We got the gil before entering the waiting - queue. In case there are other threads waiting - for the GIL, wake up the stealer thread now and - go to the waiting queue anyway, for fairness. - This will fall through if there are no other - threads waiting. - */ - check_and_save_old_fastgil(old_fastgil); - mutex2_unlock(&mutex_gil); - break; - } - } - } + /* wait until all THREES entered their safepoints */ + while (counter_of_threes > 0) { + pthread_cond_wait(&sync_cond, &sync_mutex); + } - /* Enter the waiting queue from the end. Assuming a roughly - first-in-first-out order, this will nicely give the threads - a round-robin chance. - */ - mutex1_lock(&mutex_gil_stealer); - mutex2_loop_start(&mutex_gil); + pthread_mutex_unlock(&sync_mutex); - /* We are now the stealer thread. Steals! */ - while (1) { - /* Busy-looping here. Try to look again if 'rpy_fastgil' is - released. - */ - if (!RPY_FASTGIL_LOCKED(rpy_fastgil)) { - old_fastgil = pypy_lock_test_and_set(&rpy_fastgil, 1); - if (!RPY_FASTGIL_LOCKED(old_fastgil)) - /* yes, got a non-held value! Now we hold it. */ - break; - } - /* Sleep for one interval of time. We may be woken up earlier - if 'mutex_gil' is released. - */ - if (mutex2_lock_timeout(&mutex_gil, 0.0001)) { /* 0.1 ms... */ - /* We arrive here if 'mutex_gil' was recently released - and we just relocked it. - */ - old_fastgil = 0; - break; - } - /* Loop back. */ - } - atomic_decrement(&rpy_waiting_threads); - mutex2_loop_stop(&mutex_gil); - mutex1_unlock(&mutex_gil_stealer); - } - check_and_save_old_fastgil(old_fastgil); -} - -long RPyGilYieldThread(void) -{ - /* can be called even before RPyGilAllocate(), but in this case, - 'rpy_waiting_threads' will be -42. */ - assert(RPY_FASTGIL_LOCKED(rpy_fastgil)); - if (rpy_waiting_threads <= 0) - return 0; - - /* Explicitly release the 'mutex_gil'. - */ - mutex2_unlock(&mutex_gil); - - /* Now nobody has got the GIL, because 'mutex_gil' is released (but - rpy_fastgil is still locked). Call RPyGilAcquire(). It will - enqueue ourselves at the end of the 'mutex_gil_stealer' queue. - If there is no other waiting thread, it will fall through both - its mutex_lock() and mutex_lock_timeout() now. But that's - unlikely, because we tested above that 'rpy_waiting_threads > 0'. - */ - RPyGilAcquire(); - return 1; + /* caller can continue; all threads in safepoints */ } /********** for tests only **********/ From pypy.commits at gmail.com Wed Mar 1 12:42:53 2017 From: pypy.commits at gmail.com (Raemi) Date: Wed, 01 Mar 2017 09:42:53 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) acquire gil at start of pypy Message-ID: <58b7081d.090d2e0a.ec00f.6b3a@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90445:0916afccd31b Date: 2017-03-01 18:42 +0100 http://bitbucket.org/pypy/pypy/changeset/0916afccd31b/ Log: (arigo, remi) acquire gil at start of pypy diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -231,7 +231,7 @@ def thread_setup(): allocate_shadow_stack() - tl_synclock.get_or_make_raw() + tl_synclock.setraw(1) # acquire "gil" def thread_run(): # If it's the first time we see this thread, allocate From pypy.commits at gmail.com Wed Mar 1 12:57:04 2017 From: pypy.commits at gmail.com (mjacob) Date: Wed, 01 Mar 2017 09:57:04 -0800 (PST) Subject: [pypy-commit] pypy py3.5: (stevie, mjacob) Try to fix translation. Message-ID: <58b70b70.94472e0a.ae13d.6922@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90446:394bf344ef56 Date: 2017-03-01 18:55 +0100 http://bitbucket.org/pypy/pypy/changeset/394bf344ef56/ Log: (stevie, mjacob) Try to fix translation. diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -512,7 +512,7 @@ msg = "name '%s' is nonlocal and global" % (name,) if old_role & SYM_PARAM: msg = "name '%s' is parameter and nonlocal" % (name,) - if type(self.scope) == ModuleScope: + if isinstance(self.scope, ModuleScope): msg = "nonlocal declaration not allowed at module level" if msg is not "": raise SyntaxError(msg, nonl.lineno, nonl.col_offset) From pypy.commits at gmail.com Wed Mar 1 12:57:07 2017 From: pypy.commits at gmail.com (mjacob) Date: Wed, 01 Mar 2017 09:57:07 -0800 (PST) Subject: [pypy-commit] pypy py3.5: (stevie, mjacob) Fix attribute name. Message-ID: <58b70b73.48572e0a.c8f6.69e4@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90447:1ae026527ab0 Date: 2017-03-01 18:57 +0100 http://bitbucket.org/pypy/pypy/changeset/1ae026527ab0/ Log: (stevie, mjacob) Fix attribute name. diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py --- a/pypy/interpreter/astcompiler/test/test_misc.py +++ b/pypy/interpreter/astcompiler/test/test_misc.py @@ -15,7 +15,7 @@ def app_test_warning_to_error_translation(): import warnings - + with warnings.catch_warnings(): warnings.filterwarnings("error", module="") statement = """\ @@ -31,4 +31,4 @@ assert err.lineno is not None assert err.filename is not None assert err.offset is not None - assert err.message is not None + assert err.msg is not None From pypy.commits at gmail.com Wed Mar 1 13:23:30 2017 From: pypy.commits at gmail.com (Raemi) Date: Wed, 01 Mar 2017 10:23:30 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) disable non-__thread threadlocals Message-ID: <58b711a2.4ee4190a.868a7.69f1@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90448:8343a9230861 Date: 2017-03-01 19:22 +0100 http://bitbucket.org/pypy/pypy/changeset/8343a9230861/ Log: (arigo, remi) disable non-__thread threadlocals Building the threadlocals (memset to 0) overwrites an acquired GIL... diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -231,7 +231,7 @@ def thread_setup(): allocate_shadow_stack() - tl_synclock.setraw(1) # acquire "gil" + tl_synclock.get_or_make_raw() # reference the field at least once def thread_run(): # If it's the first time we see this thread, allocate diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c --- a/rpython/translator/c/src/threadlocal.c +++ b/rpython/translator/c/src/threadlocal.c @@ -101,7 +101,7 @@ { struct pypy_threadlocal_s *tls = (struct pypy_threadlocal_s *)p; struct pypy_threadlocal_s *oldnext; - memset(p, 0, sizeof(struct pypy_threadlocal_s)); + // XXX: memset(p, 0, sizeof(struct pypy_threadlocal_s)); #ifdef RPY_TLOFS_p_errno tls->p_errno = &errno; diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h --- a/rpython/translator/c/src/threadlocal.h +++ b/rpython/translator/c/src/threadlocal.h @@ -37,7 +37,7 @@ /* ------------------------------------------------------------ */ -#ifdef USE___THREAD +// XXX: #ifdef USE___THREAD /* ------------------------------------------------------------ */ @@ -71,47 +71,48 @@ /* ------------------------------------------------------------ */ -#else -/* ------------------------------------------------------------ */ +/* #else */ +/* /\* ------------------------------------------------------------ *\/ */ +/* # error "no." */ +/* /\* Threadlocals_build needs work *\/ */ +/* /\* Don't use '__thread'. *\/ */ -/* Don't use '__thread'. */ +/* #ifdef _WIN32 */ +/* # include */ +/* # include */ +/* # define _RPy_ThreadLocals_Get() TlsGetValue(pypy_threadlocal_key) */ +/* # define _RPy_ThreadLocals_Set(x) TlsSetValue(pypy_threadlocal_key, x) */ +/* typedef DWORD pthread_key_t; */ +/* #else */ +/* # include */ +/* # define _RPy_ThreadLocals_Get() pthread_getspecific(pypy_threadlocal_key) */ +/* # define _RPy_ThreadLocals_Set(x) pthread_setspecific(pypy_threadlocal_key, x) */ +/* #endif */ -#ifdef _WIN32 -# include -# include -# define _RPy_ThreadLocals_Get() TlsGetValue(pypy_threadlocal_key) -# define _RPy_ThreadLocals_Set(x) TlsSetValue(pypy_threadlocal_key, x) -typedef DWORD pthread_key_t; -#else -# include -# define _RPy_ThreadLocals_Get() pthread_getspecific(pypy_threadlocal_key) -# define _RPy_ThreadLocals_Set(x) pthread_setspecific(pypy_threadlocal_key, x) -#endif +/* #define OP_THREADLOCALREF_ADDR(r) \ */ +/* do { \ */ +/* r = (void *)_RPy_ThreadLocals_Get(); \ */ +/* if (!r) \ */ +/* r = _RPython_ThreadLocals_Build(); \ */ +/* } while (0) */ -#define OP_THREADLOCALREF_ADDR(r) \ - do { \ - r = (void *)_RPy_ThreadLocals_Get(); \ - if (!r) \ - r = _RPython_ThreadLocals_Build(); \ - } while (0) +/* #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r) \ */ +/* do { \ */ +/* r = (void *)_RPy_ThreadLocals_Get(); \ */ +/* } while (0) */ -#define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r) \ - do { \ - r = (void *)_RPy_ThreadLocals_Get(); \ - } while (0) +/* #define RPY_THREADLOCALREF_ENSURE() \ */ +/* if (!_RPy_ThreadLocals_Get()) \ */ +/* (void)_RPython_ThreadLocals_Build(); */ -#define RPY_THREADLOCALREF_ENSURE() \ - if (!_RPy_ThreadLocals_Get()) \ - (void)_RPython_ThreadLocals_Build(); +/* #define RPY_THREADLOCALREF_GET(FIELD) \ */ +/* ((struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get())->FIELD */ -#define RPY_THREADLOCALREF_GET(FIELD) \ - ((struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get())->FIELD - -/* ------------------------------------------------------------ */ -#endif +/* /\* ------------------------------------------------------------ *\/ */ +/* #endif */ /* ------------------------------------------------------------ */ diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1472,7 +1472,7 @@ if SUPPORT__THREAD: runme(no__thread=False) - runme(no__thread=True) + #runme(no__thread=True) class TestShared(StandaloneTests): From pypy.commits at gmail.com Wed Mar 1 19:53:06 2017 From: pypy.commits at gmail.com (mjacob) Date: Wed, 01 Mar 2017 16:53:06 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Remove PyPy-only FunctionType special case from lib-python/3/pickle.py. It shouldn't be necessary after e8b1e5c2b023. Also it was outdated. Message-ID: <58b76cf2.5043190a.15bd4.789e@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90449:d775ef7a4405 Date: 2017-03-02 01:52 +0100 http://bitbucket.org/pypy/pypy/changeset/d775ef7a4405/ Log: Remove PyPy-only FunctionType special case from lib- python/3/pickle.py. It shouldn't be necessary after e8b1e5c2b023. Also it was outdated. diff --git a/lib-python/3/pickle.py b/lib-python/3/pickle.py --- a/lib-python/3/pickle.py +++ b/lib-python/3/pickle.py @@ -868,29 +868,6 @@ return None return getattr, (themodule, '__dict__') - def save_function(self, obj): - try: - return self.save_global(obj) - except PicklingError: - pass - # Check copy_reg.dispatch_table - reduce = dispatch_table.get(type(obj)) - if reduce: - rv = reduce(obj) - else: - # Check for a __reduce_ex__ method, fall back to __reduce__ - reduce = getattr(obj, "__reduce_ex__", None) - if reduce: - rv = reduce(self.proto) - else: - reduce = getattr(obj, "__reduce__", None) - if reduce: - rv = reduce() - else: - raise e - return self.save_reduce(obj=obj, *rv) - dispatch[FunctionType] = save_function - def save_set(self, obj): save = self.save write = self.write @@ -1013,6 +990,7 @@ return self.save_reduce(type, (...,), obj=obj) return self.save_global(obj) + dispatch[FunctionType] = save_global dispatch[type] = save_type From pypy.commits at gmail.com Thu Mar 2 03:36:54 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Mar 2017 00:36:54 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Copy test from PR #486 Message-ID: <58b7d9a6.0e152e0a.b9087.83b2@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90450:08e7acf2ce13 Date: 2017-03-02 09:36 +0100 http://bitbucket.org/pypy/pypy/changeset/08e7acf2ce13/ Log: Copy test from PR #486 diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -532,6 +532,47 @@ e = raises(ValueError, int, value) assert str(e.value) == "invalid literal for int() with base 10: %r" % value + def test_non_numeric_input_types(self): + # Test possible non-numeric types for the argument x, including + # subclasses of the explicitly documented accepted types. + class CustomStr(str): pass + class CustomBytes(bytes): pass + class CustomByteArray(bytearray): pass + + factories = [ + bytes, + bytearray, + lambda b: CustomStr(b.decode()), + CustomBytes, + CustomByteArray, + memoryview, + ] + try: + from array import array + except ImportError: + pass + else: + factories.append(lambda b: array('B', b)) + + for f in factories: + x = f(b'100') + assert int(x) == 100 + if isinstance(x, (str, bytes, bytearray)): + assert int(x, 2) == 4 + else: + try: + int(x, 2) + except TypeError as e: + assert "can't convert non-string" in str(e) + else: + assert False, 'did not raise' + try: + int(f(b'A' * 0x10)) + except ValueError as e: + assert "invalid literal" in str(e) + else: + assert False, 'did not raise' + def test_fake_int_as_base(self): class MyInt(object): def __init__(self, x): From pypy.commits at gmail.com Thu Mar 2 04:26:07 2017 From: pypy.commits at gmail.com (amauryfa) Date: Thu, 02 Mar 2017 01:26:07 -0800 (PST) Subject: [pypy-commit] pypy default: Add _sysconfigdata module. Message-ID: <58b7e52f.44a2190a.bde66.0c7d@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r90451:a0d3d088bbeb Date: 2014-04-07 17:31 +0200 http://bitbucket.org/pypy/pypy/changeset/a0d3d088bbeb/ Log: Add _sysconfigdata module. Empty so far, but it's probably a good idea to move some distutils variables there. diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_sysconfigdata.py @@ -0,0 +1,2 @@ +build_time_vars = { +} From pypy.commits at gmail.com Thu Mar 2 04:26:09 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 01:26:09 -0800 (PST) Subject: [pypy-commit] pypy default: Use _sysconfigdata module, which was now added to lib_pypy, in sysconfig module. Message-ID: <58b7e531.94472e0a.ae13d.860d@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90452:c36b1838396c Date: 2017-03-02 10:25 +0100 http://bitbucket.org/pypy/pypy/changeset/c36b1838396c/ Log: Use _sysconfigdata module, which was now added to lib_pypy, in sysconfig module. diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -369,11 +369,8 @@ def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - # in cPython, _sysconfigdata is generated at build time, see _generate_posix_vars() - # in PyPy no such module exists - #from _sysconfigdata import build_time_vars - #vars.update(build_time_vars) - return + from _sysconfigdata import build_time_vars + vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" From pypy.commits at gmail.com Thu Mar 2 04:28:36 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 01:28:36 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58b7e5c4.1c10190a.fc0a4.85c3@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90453:5e7e5a30cf5e Date: 2017-03-02 10:26 +0100 http://bitbucket.org/pypy/pypy/changeset/5e7e5a30cf5e/ Log: hg merge default diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -369,11 +369,8 @@ def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - # in cPython, _sysconfigdata is generated at build time, see _generate_posix_vars() - # in PyPy no such module exists - #from _sysconfigdata import build_time_vars - #vars.update(build_time_vars) - return + from _sysconfigdata import build_time_vars + vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" From pypy.commits at gmail.com Thu Mar 2 04:28:38 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 01:28:38 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Remove wrong comment. Message-ID: <58b7e5c6.1459190a.c46bd.847d@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90454:b5b35960baa6 Date: 2017-03-02 10:27 +0100 http://bitbucket.org/pypy/pypy/changeset/b5b35960baa6/ Log: Remove wrong comment. diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py --- a/lib-python/3/sysconfig.py +++ b/lib-python/3/sysconfig.py @@ -416,7 +416,6 @@ def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() from _sysconfigdata import build_time_vars vars.update(build_time_vars) From pypy.commits at gmail.com Thu Mar 2 05:05:01 2017 From: pypy.commits at gmail.com (Raemi) Date: Thu, 02 Mar 2017 02:05:01 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: avoid clearing threadlocals on thread-exit Message-ID: <58b7ee4d.41152e0a.90125.8bed@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90455:726153037b52 Date: 2017-03-02 11:04 +0100 http://bitbucket.org/pypy/pypy/changeset/726153037b52/ Log: avoid clearing threadlocals on thread-exit there is a release-gil call after the call to thread_die (that clears the threadlocal with 0xdd). But then, synclock is 0xdddddddd. So we don't clear them for now. diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c --- a/rpython/translator/c/src/threadlocal.c +++ b/rpython/translator/c/src/threadlocal.c @@ -136,7 +136,7 @@ if (tls->ready == 42) { tls->next->prev = tls->prev; tls->prev->next = tls->next; - memset(tls, 0xDD, sizeof(struct pypy_threadlocal_s)); /* debug */ + // XXX: called before a release_gil -> synclock set to 0xddddddd... memset(tls, 0xDD, sizeof(struct pypy_threadlocal_s)); /* debug */ tls->ready = 0; } _RPython_ThreadLocals_Release(); From pypy.commits at gmail.com Thu Mar 2 05:08:40 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:40 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: "fix" multibytecodec Message-ID: <58b7ef28.548d190a.b77b5.16eb@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90456:777001a4a191 Date: 2017-03-01 16:52 +0100 http://bitbucket.org/pypy/pypy/changeset/777001a4a191/ Log: "fix" multibytecodec diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py --- a/pypy/module/_multibytecodec/c_codecs.py +++ b/pypy/module/_multibytecodec/c_codecs.py @@ -3,7 +3,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator import cdir -UNICODE_REPLACEMENT_CHARACTER = u'\uFFFD' +UNICODE_REPLACEMENT_CHARACTER = u'\uFFFD'.encode("utf8") class EncodeDecodeError(Exception): @@ -148,15 +148,17 @@ if errors == "strict": raise EncodeDecodeError(start, end, reason) elif errors == "ignore": - replace = u"" + replace = "" + lgt = 0 elif errors == "replace": replace = UNICODE_REPLACEMENT_CHARACTER + lgt = 1 else: assert errorcb - replace, end = errorcb(errors, namecb, reason, + replace, end, lgt = errorcb(errors, namecb, reason, stringdata, start, end) - with rffi.scoped_nonmoving_unicodebuffer(replace) as inbuf: - r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, len(replace), end) + with rffi.scoped_nonmoving_unicodebuffer(replace.decode("utf8")) as inbuf: + r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, lgt, end) if r == MBERR_NOMEMORY: raise MemoryError @@ -255,15 +257,15 @@ replace = "?" else: assert errorcb - retu, rets, end = errorcb(errors, namecb, reason, - unicodedata, start, end) + retu, rets, end, lgt = errorcb(errors, namecb, reason, + unicodedata.encode("utf8"), start, end) if rets is not None: # py3k only replace = rets else: assert retu is not None codec = pypy_cjk_enc_getcodec(encodebuf) - replace = encode(codec, retu, "strict", errorcb, namecb) + replace = encode(codec, retu.decode("utf8"), "strict", errorcb, namecb) with rffi.scoped_nonmovingbuffer(replace) as inbuf: r = pypy_cjk_enc_replace_on_error(encodebuf, inbuf, len(replace), end) if r == MBERR_NOMEMORY: diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py --- a/pypy/module/_multibytecodec/interp_incremental.py +++ b/pypy/module/_multibytecodec/interp_incremental.py @@ -96,8 +96,9 @@ c_codecs.pypy_cjk_enc_free(self.encodebuf) self.encodebuf = lltype.nullptr(c_codecs.ENCODEBUF_P.TO) - @unwrap_spec(object=unicode, final=bool) - def encode_w(self, object, final=False): + @unwrap_spec(utf8object='utf8', final=bool) + def encode_w(self, utf8object, objlen, final=False): + object = utf8object.decode('utf8') space = self.space state = space.fromcache(CodecState) if len(self.pending) > 0: @@ -107,7 +108,7 @@ state.encode_error_handler, self.name, get_ignore_error(final)) except c_codecs.EncodeDecodeError as e: - raise wrap_unicodeencodeerror(space, e, object, self.name) + raise wrap_unicodeencodeerror(space, e, utf8object, self.name) except RuntimeError: raise wrap_runtimeerror(space) pos = c_codecs.pypy_cjk_enc_inbuf_consumed(self.encodebuf) diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py --- a/pypy/module/_multibytecodec/interp_multibytecodec.py +++ b/pypy/module/_multibytecodec/interp_multibytecodec.py @@ -18,30 +18,30 @@ state = space.fromcache(CodecState) # try: - output = c_codecs.decode(self.codec, input, errors, + u_output = c_codecs.decode(self.codec, input, errors, state.decode_error_handler, self.name) except c_codecs.EncodeDecodeError as e: raise wrap_unicodedecodeerror(space, e, input, self.name) except RuntimeError: raise wrap_runtimeerror(space) - return space.newtuple([space.newunicode(output), + return space.newtuple([space.newunicode(u_output), space.newint(len(input))]) - @unwrap_spec(input=unicode, errors="str_or_None") - def encode(self, space, input, errors=None): + @unwrap_spec(input='utf8', errors="str_or_None") + def encode(self, space, input, inputlen, errors=None): if errors is None: errors = 'strict' state = space.fromcache(CodecState) # try: - output = c_codecs.encode(self.codec, input, errors, + output = c_codecs.encode(self.codec, input.decode('utf8'), errors, state.encode_error_handler, self.name) except c_codecs.EncodeDecodeError as e: raise wrap_unicodeencodeerror(space, e, input, self.name) except RuntimeError: raise wrap_runtimeerror(space) return space.newtuple([space.newbytes(output), - space.newint(len(input))]) + space.newint(inputlen)]) MultibyteCodec.typedef = TypeDef( @@ -76,7 +76,7 @@ space.w_UnicodeEncodeError, space.newtuple([ space.newtext(name), - space.newunicode(input), + space.newutf8(input, -1), space.newint(e.start), space.newint(e.end), space.newtext(e.reason)])) diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py b/pypy/module/_multibytecodec/test/test_c_codecs.py --- a/pypy/module/_multibytecodec/test/test_c_codecs.py +++ b/pypy/module/_multibytecodec/test/test_c_codecs.py @@ -126,6 +126,6 @@ def test_encode_custom_error_handler_bytes(): c = getcodec("hz") def errorhandler(errors, enc, msg, t, startingpos, endingpos): - return None, '\xc3', endingpos + return None, '\xc3', endingpos, -1 s = encode(c, u'abc\u1234def', 'foo', errorhandler) assert '\xc3' in s From pypy.commits at gmail.com Thu Mar 2 05:08:47 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:47 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: bah Message-ID: <58b7ef2f.e5d7190a.eaad7.8e5a@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90459:91ec2147c427 Date: 2017-03-01 17:59 +0100 http://bitbucket.org/pypy/pypy/changeset/91ec2147c427/ Log: bah diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py --- a/rpython/rtyper/rpbc.py +++ b/rpython/rtyper/rpbc.py @@ -389,12 +389,6 @@ return v return NotImplemented -class __extend__(pairtype(MethodsPBCRepr, MethodsPBCRepr)): - def convert_from_to((r_mpbc1, r_mpbc2), v, llops): - if r_mpbc1.lowleveltype == r_mpbc2.lowleveltype: - return v - return NotImplemented - class SmallFunctionSetPBCRepr(FunctionReprBase): def __init__(self, rtyper, s_pbc): @@ -1198,3 +1192,9 @@ # now hop2 looks like simple_call(function, self, args...) return hop2.dispatch() + +class __extend__(pairtype(MethodsPBCRepr, MethodsPBCRepr)): + def convert_from_to((r_mpbc1, r_mpbc2), v, llops): + if r_mpbc1.lowleveltype == r_mpbc2.lowleveltype: + return v + return NotImplemented From pypy.commits at gmail.com Thu Mar 2 05:08:42 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:42 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fix pyexpat. skip one failing test on os x Message-ID: <58b7ef2a.6911190a.9b8e7.8911@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90457:572cc70c4d30 Date: 2017-03-01 17:24 +0100 http://bitbucket.org/pypy/pypy/changeset/572cc70c4d30/ Log: fix pyexpat. skip one failing test on os x diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -2,7 +2,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError, oefmt -from rpython.rlib import rgc, jit +from rpython.rlib import rgc, jit, rutf8 from rpython.rlib.objectmodel import specialize from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform @@ -487,8 +487,15 @@ def w_convert(self, space, s): if self.returns_unicode: - from pypy.interpreter.unicodehelper import decode_utf8 - return space.newunicode(decode_utf8(space, s)) + # I suppose this is a valid utf8, but there is noone to check + # and noone to catch an error either + try: + rutf8.str_check_utf8(s, len(s), final=True) + return space.newutf8(s, -1) + except rutf8.Utf8CheckError as e: + from pypy.interpreter import unicodehelper + unicodehelper.decode_error_handler(space)('strict', 'utf-8', + e.msg, s, e.startpos, e.endpos) else: return space.newtext(s) diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py --- a/pypy/module/pyexpat/test/test_parser.py +++ b/pypy/module/pyexpat/test/test_parser.py @@ -192,6 +192,11 @@ spaceconfig = dict(usemodules=['pyexpat', 'itertools', '_socket', 'time', 'struct', 'binascii']) + def setup_class(cls): + import py, sys + if sys.platform != 'linux2': + py.test.skip("even if we add all the crazy includes to run ctypes, it ends with MallocMismatch") + def test_django_bug(self): xml_str = '' diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -59,12 +59,12 @@ def _include_dirs_for_openssl(self): return self._pkg_config("openssl", "--cflags-only-I", - ['/usr/include'], + ['/usr/include', '/usr/local/opt/openssl/include'], check_result_dir=True) def _library_dirs_for_openssl(self): return self._pkg_config("openssl", "--libs-only-L", - ['/usr/lib'], + ['/usr/lib', '/usr/local/opt/openssl/lib'], check_result_dir=True) def _frameworks(self, frameworks): From pypy.commits at gmail.com Thu Mar 2 05:08:45 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:45 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: trivial conversion between MethodsPBCRepr Message-ID: <58b7ef2d.010e2e0a.6a40b.8abb@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90458:ff30e62aa310 Date: 2017-03-01 17:53 +0100 http://bitbucket.org/pypy/pypy/changeset/ff30e62aa310/ Log: trivial conversion between MethodsPBCRepr diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py --- a/rpython/rtyper/rpbc.py +++ b/rpython/rtyper/rpbc.py @@ -389,6 +389,12 @@ return v return NotImplemented +class __extend__(pairtype(MethodsPBCRepr, MethodsPBCRepr)): + def convert_from_to((r_mpbc1, r_mpbc2), v, llops): + if r_mpbc1.lowleveltype == r_mpbc2.lowleveltype: + return v + return NotImplemented + class SmallFunctionSetPBCRepr(FunctionReprBase): def __init__(self, rtyper, s_pbc): From pypy.commits at gmail.com Thu Mar 2 05:08:49 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:49 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fix fix fix Message-ID: <58b7ef31.5d532e0a.d82cc.8d97@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90460:8f690010d092 Date: 2017-03-01 19:16 +0100 http://bitbucket.org/pypy/pypy/changeset/8f690010d092/ Log: fix fix fix diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -42,15 +42,6 @@ """representation for debugging purposes""" return "%s(%r)" % (self.__class__.__name__, self._utf8) - def unwrap(self, space): - # for testing - return self._value - - def create_if_subclassed(self): - if type(self) is W_UnicodeObject: - return self - return W_UnicodeObject(self._value) - def is_w(self, space, w_other): if not isinstance(w_other, W_UnicodeObject): return False @@ -103,6 +94,7 @@ charbuf_w = str_w def listview_unicode(self): + XXX # fix at some point return _create_list_from_unicode(self._value) def ord(self, space): @@ -130,6 +122,8 @@ return rutf8.compute_length_utf8(self._utf8) def _val(self, space): + import pdb + pdb.set_trace() return self._utf8.decode('utf8') @staticmethod @@ -534,11 +528,10 @@ @unwrap_spec(maxsplit=int) def descr_split(self, space, w_sep=None, maxsplit=-1): - # XXX maybe optimize? res = [] value = self._utf8 if space.is_none(w_sep): - res = split(value, maxsplit=maxsplit) + res = split(value, maxsplit=maxsplit, isutf8=1) return space.newlist_from_unicode(res) by = self.convert_arg_to_w_unicode(space, w_sep)._utf8 @@ -582,6 +575,12 @@ return W_UnicodeObject(centered, self._len() + d) + def descr_contains(self, space, w_sub): + value = self._utf8 + w_other = self.convert_arg_to_w_unicode(space, w_sub) + return space.newbool(value.find(w_other._utf8) >= 0) + + def wrapunicode(space, uni): return W_UnicodeObject(uni) diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -16,17 +16,31 @@ # -------------- public API for string functions ----------------------- - at specialize.argtype(0) -def _isspace(char): - if isinstance(char, str): - return char.isspace() + at specialize.ll_and_arg(2) +def _isspace(s, pos, isutf8=0): + if isutf8: + from rpython.rlib import rutf8 + char = rutf8.codepoint_at_pos(s, pos) + return unicodedb.isspace(char) else: - assert isinstance(char, unicode) - return unicodedb.isspace(ord(char)) + char = s[pos] + if isinstance(char, str): + return char.isspace() + else: + assert isinstance(char, unicode) + return unicodedb.isspace(ord(char)) + at specialize.arg(2) +def _incr(s, pos, isutf8): + from rpython.rlib.rutf8 import next_codepoint_pos - at specialize.argtype(0, 1) -def split(value, by=None, maxsplit=-1): + if isutf8: + return next_codepoint_pos(s, pos) + else: + return pos + 1 + + at specialize.ll_and_arg(3) +def split(value, by=None, maxsplit=-1, isutf8=0): if by is None: length = len(value) i = 0 @@ -34,9 +48,9 @@ while True: # find the beginning of the next word while i < length: - if not _isspace(value[i]): + if not _isspace(value, i, isutf8): break # found - i += 1 + i = _incr(value, i, isutf8) else: break # end of string, finished @@ -44,16 +58,19 @@ if maxsplit == 0: j = length # take all the rest of the string else: - j = i + 1 - while j < length and not _isspace(value[j]): - j += 1 + j = _incr(value, i, isutf8) + while j < length and not _isspace(value, j, isutf8): + j = _incr(value, j, isutf8) maxsplit -= 1 # NB. if it's already < 0, it stays < 0 # the word is value[i:j] res.append(value[i:j]) # continue to look from the character following the space after the word - i = j + 1 + if j < length: + i = _incr(value, j, isutf8) + else: + break return res if isinstance(value, unicode): @@ -66,6 +83,8 @@ bylen = len(by) if bylen == 0: raise ValueError("empty separator") + # XXX measure if preallocating the result list to the correct + # size is faster, should be start = 0 if bylen == 1: @@ -102,8 +121,8 @@ return res - at specialize.argtype(0, 1) -def rsplit(value, by=None, maxsplit=-1): + at specialize.ll_and_arg(3) +def rsplit(value, by=None, maxsplit=-1, isutf8=0): if by is None: res = [] diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py --- a/rpython/rtyper/rpbc.py +++ b/rpython/rtyper/rpbc.py @@ -1134,6 +1134,8 @@ self.lowleveltype = self.r_im_self.lowleveltype def convert_const(self, method): + if method is None: + return nullptr(self.lowleveltype.TO) if getattr(method, 'im_func', None) is None: raise TyperError("not a bound method: %r" % method) return self.r_im_self.convert_const(method.im_self) From pypy.commits at gmail.com Thu Mar 2 05:08:52 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:52 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: get things going for now Message-ID: <58b7ef34.e5d7190a.eaad7.8e5b@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90461:94c5b4b5fb85 Date: 2017-03-01 19:24 +0100 http://bitbucket.org/pypy/pypy/changeset/94c5b4b5fb85/ Log: get things going for now diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -122,8 +122,6 @@ return rutf8.compute_length_utf8(self._utf8) def _val(self, space): - import pdb - pdb.set_trace() return self._utf8.decode('utf8') @staticmethod diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -30,7 +30,7 @@ assert isinstance(char, unicode) return unicodedb.isspace(ord(char)) - at specialize.arg(2) + at specialize.ll_and_arg(2) def _incr(s, pos, isutf8): from rpython.rlib.rutf8 import next_codepoint_pos @@ -130,7 +130,7 @@ while True: # starting from the end, find the end of the next word while i >= 0: - if not _isspace(value[i]): + if not _isspace(value, i): break # found i -= 1 else: @@ -142,7 +142,7 @@ j = -1 # take all the rest of the string else: j = i - 1 - while j >= 0 and not _isspace(value[j]): + while j >= 0 and not _isspace(value, j): j -= 1 maxsplit -= 1 # NB. if it's already < 0, it stays < 0 From pypy.commits at gmail.com Thu Mar 2 05:08:59 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:59 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: basic getitem implemented Message-ID: <58b7ef3b.1b582e0a.468e3.83ad@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90464:4a89aa28bd21 Date: 2017-03-01 21:26 +0100 http://bitbucket.org/pypy/pypy/changeset/4a89aa28bd21/ Log: basic getitem implemented diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -798,6 +798,7 @@ def descr_getnewargs(self, space): return space.newtuple([self._new(self._val(space))]) + # ____________________________________________________________ # helpers for slow paths, moved out because they contain loops diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -1,3 +1,4 @@ + import py import sys @@ -64,6 +65,11 @@ check(u'a' + 'b', u'ab') check('a' + u'b', u'ab') + def test_getitem(self): + assert u'abc'[2] == 'c' + raises(IndexError, u'abc'.__getitem__, 15) + assert u'g\u0105\u015b\u0107'[2] == u'\u015b' + def test_join(self): def check(a, b): assert a == b diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -612,6 +612,17 @@ descr_rmul = descr_mul + def _getitem_result(self, space, index): + if self._ucs4 is None: + self._ucs4 = self._utf8.decode('utf-8') + try: + return W_UnicodeObject(self._ucs4[index].encode('utf-8'), 1) + except IndexError: + raise oefmt(space.w_IndexError, "string index out of range") + + def descr_getnewargs(self, space): + return space.newtuple([W_UnicodeObject(self._utf8, self._length)]) + def wrapunicode(space, uni): From pypy.commits at gmail.com Thu Mar 2 05:08:54 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:54 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: improve the tests Message-ID: <58b7ef36.15502e0a.e4311.8605@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90462:50905f927559 Date: 2017-03-01 19:31 +0100 http://bitbucket.org/pypy/pypy/changeset/50905f927559/ Log: improve the tests diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -100,6 +100,8 @@ assert u'\n\n'.splitlines() == [u'', u''] assert u'a\nb\nc'.splitlines(1) == [u'a\n', u'b\n', u'c'] assert u'\na\nb\n'.splitlines(1) == [u'\n', u'a\n', u'b\n'] + assert ((u'a' + '\xc2\x85'.decode('utf8') + u'b\n').splitlines() == + ['a', 'b']) def test_zfill(self): assert u'123'.zfill(2) == u'123' @@ -176,6 +178,7 @@ raises(ValueError, 'abc'.rsplit, u'') assert u' a b c '.rsplit(None, 0) == [u' a b c'] assert u''.rsplit('aaa') == [u''] + assert u'a\nb\u1680c'.rsplit() == [u'a', u'b', u'c'] def test_split_rsplit_str_unicode(self): x = 'abc'.split(u'b') From pypy.commits at gmail.com Thu Mar 2 05:08:56 2017 From: pypy.commits at gmail.com (fijal) Date: Thu, 02 Mar 2017 02:08:56 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: replace and mul Message-ID: <58b7ef38.5e142e0a.8567d.6afc@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90463:818637e5be35 Date: 2017-03-01 19:55 +0100 http://bitbucket.org/pypy/pypy/changeset/818637e5be35/ Log: replace and mul diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -4,7 +4,8 @@ compute_hash, compute_unique_id, import_from_mixin, enforceargs, newlist_hint, specialize) from rpython.rlib.buffer import StringBuffer -from rpython.rlib.rstring import StringBuilder, split, rsplit, UnicodeBuilder +from rpython.rlib.rstring import StringBuilder, split, rsplit, UnicodeBuilder,\ + replace from rpython.rlib.runicode import make_unicode_escape_function from rpython.rlib import rutf8, jit @@ -579,6 +580,40 @@ return space.newbool(value.find(w_other._utf8) >= 0) + @unwrap_spec(count=int) + def descr_replace(self, space, w_old, w_new, count=-1): + input = self._utf8 + + w_sub = self.convert_arg_to_w_unicode(space, w_old) + w_by = self.convert_arg_to_w_unicode(space, w_new) + # the following two lines are for being bug-to-bug compatible + # with CPython: see issue #2448 + if count >= 0 and len(input) == 0: + return self._empty() + try: + res = replace(input, w_sub._utf8, w_by._utf8, count) + except OverflowError: + raise oefmt(space.w_OverflowError, "replace string is too long") + + return W_UnicodeObject(res, -1) + + def descr_mul(self, space, w_times): + try: + times = space.getindex_w(w_times, space.w_OverflowError) + except OperationError as e: + if e.match(space, space.w_TypeError): + return space.w_NotImplemented + raise + if times <= 0: + return self._empty() + if len(self._utf8) == 1: + return W_UnicodeObject(self._utf8[0] * times, times) + return W_UnicodeObject(self._utf8 * times, times * self._len()) + + descr_rmul = descr_mul + + + def wrapunicode(space, uni): return W_UnicodeObject(uni) From pypy.commits at gmail.com Thu Mar 2 05:18:46 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 02:18:46 -0800 (PST) Subject: [pypy-commit] pypy default: Use space.eq_w() instead of space.eq() in Method.descr_method_eq(). Message-ID: <58b7f186.94472e0a.ae13d.8906@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90466:23186813950a Date: 2017-03-02 11:10 +0100 http://bitbucket.org/pypy/pypy/changeset/23186813950a/ Log: Use space.eq_w() instead of space.eq() in Method.descr_method_eq(). diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py --- a/pypy/interpreter/function.py +++ b/pypy/interpreter/function.py @@ -556,7 +556,7 @@ return space.w_False if not space.eq_w(self.w_instance, w_other.w_instance): return space.w_False - return space.eq(self.w_function, w_other.w_function) + return space.newbool(space.eq_w(self.w_function, w_other.w_function)) def is_w(self, space, other): if not isinstance(other, Method): diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py --- a/pypy/interpreter/test/test_function.py +++ b/pypy/interpreter/test/test_function.py @@ -560,6 +560,18 @@ assert A().m == X() assert X() == A().m + def test_method_equals_with_identity(self): + from types import MethodType + class CallableBadEq(object): + def __call__(self): + pass + def __eq__(self, other): + raise ZeroDivisionError + func = CallableBadEq() + meth = MethodType(func, object) + assert meth == meth + assert meth == MethodType(func, object) + @pytest.mark.skipif("config.option.runappdirect") def test_method_identity(self): class A(object): From pypy.commits at gmail.com Thu Mar 2 05:18:49 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 02:18:49 -0800 (PST) Subject: [pypy-commit] pypy default: space.is_true(space.eq(...)) -> space.eq_w(...) in interp_array.py. Message-ID: <58b7f189.0528190a.a556c.88d0@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90467:ce53039393d6 Date: 2017-03-02 11:17 +0100 http://bitbucket.org/pypy/pypy/changeset/ce53039393d6/ Log: space.is_true(space.eq(...)) -> space.eq_w(...) in interp_array.py. It's slightly shorter and uses the identity check fast path. diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -76,7 +76,7 @@ w_elem1 = arr1.w_getitem(space, i) w_elem2 = arr2.w_getitem(space, i) if comp_op == EQ: - res = space.is_true(space.eq(w_elem1, w_elem2)) + res = space.eq_w(w_elem1, w_elem2) if not res: return space.w_False elif comp_op == NE: @@ -90,7 +90,7 @@ res = space.is_true(space.gt(w_elem1, w_elem2)) if res: return space.w_True - elif not space.is_true(space.eq(w_elem1, w_elem2)): + elif not space.eq_w(w_elem1, w_elem2): return space.w_False else: if comp_op == LE: @@ -99,7 +99,7 @@ res = space.is_true(space.ge(w_elem1, w_elem2)) if not res: return space.w_False - elif not space.is_true(space.eq(w_elem1, w_elem2)): + elif not space.eq_w(w_elem1, w_elem2): return space.w_True # we have some leftovers if comp_op == EQ: @@ -266,7 +266,7 @@ for i in range(self.len): # XXX jitdriver w_item = self.w_getitem(space, i) - if space.is_true(space.eq(w_item, w_val)): + if space.eq_w(w_item, w_val): cnt += 1 return space.newint(cnt) @@ -277,7 +277,7 @@ """ for i in range(self.len): w_item = self.w_getitem(space, i) - if space.is_true(space.eq(w_item, w_x)): + if space.eq_w(w_item, w_x): return space.newint(i) raise oefmt(space.w_ValueError, "array.index(x): x not in list") From pypy.commits at gmail.com Thu Mar 2 05:18:44 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 02:18:44 -0800 (PST) Subject: [pypy-commit] pypy default: Use space.eq_w() instead of space.eq() in W_DictViewItemsObject.descr_contains(). Message-ID: <58b7f184.6911190a.9b8e7.89d1@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90465:e8bf94789967 Date: 2017-03-02 10:55 +0100 http://bitbucket.org/pypy/pypy/changeset/e8bf94789967/ Log: Use space.eq_w() instead of space.eq() in W_DictViewItemsObject.descr_contains(). diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -1597,7 +1597,7 @@ w_found = None if w_found is None: return space.w_False - return space.eq(w_value, w_found) + return space.newbool(space.eq_w(w_value, w_found)) class W_DictViewKeysObject(W_DictViewObject, SetLikeDictView): def descr_iter(self, space): diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -842,6 +842,14 @@ assert d.viewitems() != e.viewitems() assert not d.viewitems() == 42 + def test_dict_items_contains_with_identity(self): + class BadEq(object): + def __eq__(self, other): + raise ZeroDivisionError + k = BadEq() + v = BadEq() + assert (k, v) in {k: v}.viewitems() + def test_dict_mixed_keys_items(self): d = {(1, 1): 11, (2, 2): 22} e = {1: 1, 2: 2} From pypy.commits at gmail.com Thu Mar 2 05:25:56 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 02:25:56 -0800 (PST) Subject: [pypy-commit] pypy default: space.is_true(space.eq(...)) -> space.eq_w(...) in two more places. Message-ID: <58b7f334.c27d190a.47015.87ba@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90468:21d6cfb9ce7e Date: 2017-03-02 11:25 +0100 http://bitbucket.org/pypy/pypy/changeset/21d6cfb9ce7e/ Log: space.is_true(space.eq(...)) -> space.eq_w(...) in two more places. diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -848,7 +848,7 @@ if whence == 1: # seek relative to current position - if not space.is_true(space.eq(w_pos, space.newint(0))): + if not space.eq_w(w_pos, space.newint(0)): raise oefmt(space.w_IOError, "can't do nonzero cur-relative seeks") # Seeking to the current position should attempt to sync the @@ -857,7 +857,7 @@ elif whence == 2: # seek relative to end of file - if not space.is_true(space.eq(w_pos, space.newint(0))): + if not space.eq_w(w_pos, space.newint(0)): raise oefmt(space.w_IOError, "can't do nonzero end-relative seeks") space.call_method(self, "flush") diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -230,7 +230,7 @@ if e.match(space, space.w_StopIteration): break raise - if space.is_true(space.eq(w_next, w_obj)): + if space.eq_w(w_next, w_obj): return idx idx += 1 From pypy.commits at gmail.com Thu Mar 2 05:38:52 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 02:38:52 -0800 (PST) Subject: [pypy-commit] pypy default: Make BadEq object hashable. I forgot to do this a few commits ago. Message-ID: <58b7f63c.110f2e0a.5bc7f.8fd6@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90469:dca7eeaa51de Date: 2017-03-02 11:38 +0100 http://bitbucket.org/pypy/pypy/changeset/dca7eeaa51de/ Log: Make BadEq object hashable. I forgot to do this a few commits ago. diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -846,6 +846,8 @@ class BadEq(object): def __eq__(self, other): raise ZeroDivisionError + def __hash__(self): + return 7 k = BadEq() v = BadEq() assert (k, v) in {k: v}.viewitems() From pypy.commits at gmail.com Thu Mar 2 05:41:44 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 02:41:44 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58b7f6e8.0e572e0a.96d52.8d36@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90470:e761f5e9359b Date: 2017-03-02 11:38 +0100 http://bitbucket.org/pypy/pypy/changeset/e761f5e9359b/ Log: hg merge default diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py --- a/pypy/interpreter/function.py +++ b/pypy/interpreter/function.py @@ -575,7 +575,7 @@ return space.w_NotImplemented if not space.eq_w(self.w_instance, w_other.w_instance): return space.w_False - return space.eq(self.w_function, w_other.w_function) + return space.newbool(space.eq_w(self.w_function, w_other.w_function)) def is_w(self, space, other): if not isinstance(other, Method): diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py --- a/pypy/interpreter/test/test_function.py +++ b/pypy/interpreter/test/test_function.py @@ -574,6 +574,18 @@ assert A().m == X() assert X() == A().m + def test_method_equals_with_identity(self): + from types import MethodType + class CallableBadEq(object): + def __call__(self): + pass + def __eq__(self, other): + raise ZeroDivisionError + func = CallableBadEq() + meth = MethodType(func, object) + assert meth == meth + assert meth == MethodType(func, object) + @pytest.mark.skipif("config.option.runappdirect") def test_method_identity(self): class A(object): diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -881,7 +881,7 @@ if whence == 1: # seek relative to current position - if not space.is_true(space.eq(w_pos, space.newint(0))): + if not space.eq_w(w_pos, space.newint(0)): self._unsupportedoperation( space, "can't do nonzero cur-relative seeks") # Seeking to the current position should attempt to sync the @@ -890,7 +890,7 @@ elif whence == 2: # seek relative to end of file - if not space.is_true(space.eq(w_pos, space.newint(0))): + if not space.eq_w(w_pos, space.newint(0)): self._unsupportedoperation( space, "can't do nonzero end-relative seeks") space.call_method(self, "flush") diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -95,7 +95,7 @@ w_elem1 = arr1.w_getitem(space, i) w_elem2 = arr2.w_getitem(space, i) if comp_op == EQ: - res = space.is_true(space.eq(w_elem1, w_elem2)) + res = space.eq_w(w_elem1, w_elem2) if not res: return space.w_False elif comp_op == NE: @@ -109,7 +109,7 @@ res = space.is_true(space.gt(w_elem1, w_elem2)) if res: return space.w_True - elif not space.is_true(space.eq(w_elem1, w_elem2)): + elif not space.eq_w(w_elem1, w_elem2): return space.w_False else: if comp_op == LE: @@ -118,7 +118,7 @@ res = space.is_true(space.ge(w_elem1, w_elem2)) if not res: return space.w_False - elif not space.is_true(space.eq(w_elem1, w_elem2)): + elif not space.eq_w(w_elem1, w_elem2): return space.w_True # we have some leftovers if comp_op == EQ: @@ -282,7 +282,7 @@ for i in range(self.len): # XXX jitdriver w_item = self.w_getitem(space, i) - if space.is_true(space.eq(w_item, w_val)): + if space.eq_w(w_item, w_val): cnt += 1 return space.newint(cnt) @@ -293,7 +293,7 @@ """ for i in range(self.len): w_item = self.w_getitem(space, i) - if space.is_true(space.eq(w_item, w_x)): + if space.eq_w(w_item, w_x): return space.newint(i) raise oefmt(space.w_ValueError, "array.index(x): x not in list") diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -230,7 +230,7 @@ if e.match(space, space.w_StopIteration): break raise - if space.is_true(space.eq(w_next, w_obj)): + if space.eq_w(w_next, w_obj): return idx idx += 1 diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -1549,7 +1549,7 @@ w_found = self.w_dict.getitem(w_key) if w_found is None: return space.w_False - return space.eq(w_value, w_found) + return space.newbool(space.eq_w(w_value, w_found)) class W_DictViewKeysObject(W_DictViewObject, SetLikeDictView): def descr_iter(self, space): diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -813,6 +813,14 @@ e["a"] = "def" assert d.items() != e.items() + def test_dict_items_contains_with_identity(self): + class BadEq(object): + def __eq__(self, other): + raise ZeroDivisionError + k = BadEq() + v = BadEq() + assert (k, v) in {k: v}.viewitems() + def test_dict_mixed_keys_items(self): d = {(1, 1): 11, (2, 2): 22} e = {1: 1, 2: 2} From pypy.commits at gmail.com Thu Mar 2 05:41:46 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 02:41:46 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58b7f6ea.5038190a.3ab0d.9062@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90471:dc0536431eaa Date: 2017-03-02 11:41 +0100 http://bitbucket.org/pypy/pypy/changeset/dc0536431eaa/ Log: hg merge default diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -817,9 +817,11 @@ class BadEq(object): def __eq__(self, other): raise ZeroDivisionError + def __hash__(self): + return 7 k = BadEq() v = BadEq() - assert (k, v) in {k: v}.viewitems() + assert (k, v) in {k: v}.items() def test_dict_mixed_keys_items(self): d = {(1, 1): 11, (2, 2): 22} From pypy.commits at gmail.com Thu Mar 2 06:27:28 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Mar 2017 03:27:28 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Add new buffer to array.c and test that bf_releasebuffer is called (failing) Message-ID: <58b801a0.5125190a.8e02f.86eb@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90472:99ed56433685 Date: 2017-03-02 11:07 +0100 http://bitbucket.org/pypy/pypy/changeset/99ed56433685/ Log: Add new buffer to array.c and test that bf_releasebuffer is called (failing) diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -468,11 +468,12 @@ ptr = pybuf.c_buf size = pybuf.c_len ndim = widen(pybuf.c_ndim) - shape = [pybuf.c_shape[i] for i in range(ndim)] + shape = None + if pybuf.c_shape: + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = None if pybuf.c_strides: strides = [pybuf.c_strides[i] for i in range(ndim)] - else: - strides = [1] if pybuf.c_format: format = rffi.charp2str(pybuf.c_format) else: diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1893,10 +1893,10 @@ } else if(obj1->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); else if(obj2->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -1947,10 +1947,10 @@ } else if(obj1->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); else if(obj2->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -2006,6 +2006,29 @@ return 1; } +static int +array_getbuffer(PyObject* obj, Py_buffer* view, int flags) +{ + arrayobject* self = (arrayobject*)obj; + return PyBuffer_FillInfo(view, obj, self->ob_item, + Py_SIZE(self)*self->ob_descr->itemsize, 0, flags); +} + +static long releasebuffer_cnt = 0; + +static PyObject * +get_releasebuffer_cnt(void) +{ + return PyLong_FromLong(releasebuffer_cnt); +} + +static void +array_releasebuffer(arrayobject* self) +{ + releasebuffer_cnt++; + return; +} + static PySequenceMethods array_as_sequence = { (lenfunc)array_length, /*sq_length*/ (binaryfunc)array_concat, /*sq_concat*/ @@ -2024,6 +2047,8 @@ (writebufferproc)array_buffer_getwritebuf, (segcountproc)array_buffer_getsegcount, NULL, + (getbufferproc)array_getbuffer, + (releasebufferproc)array_releasebuffer }; static PyObject * @@ -2237,7 +2262,7 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &array_as_buffer, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ arraytype_doc, /* tp_doc */ 0, /* tp_traverse */ @@ -2280,8 +2305,9 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &array_as_buffer, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES | + Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */ arraytype_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ @@ -2409,6 +2435,16 @@ return PyString_FromStringAndSize((char*)ptr, size); } +static PyObject * +create_and_release_buffer(PyObject *self, PyObject *obj) +{ + Py_buffer view; + int res = PyObject_GetBuffer(obj, &view, 0); + if (res < 0) + return NULL; + PyBuffer_Release(&view); + Py_RETURN_NONE; +} /*********************** Install Module **************************/ @@ -2417,6 +2453,8 @@ {"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL}, {"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL}, {"readbuffer_as_string", (PyCFunction)readbuffer_as_string, METH_VARARGS, NULL}, + {"get_releasebuffer_cnt", (PyCFunction)get_releasebuffer_cnt, METH_NOARGS, NULL}, + {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -78,6 +78,23 @@ '\x03\0\0\0' '\x04\0\0\0') + def test_releasebuffer(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert module.get_releasebuffer_cnt() == 0 + module.create_and_release_buffer(arr) + assert module.get_releasebuffer_cnt() == 1 + + def test_Py_buffer(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert module.get_releasebuffer_cnt() == 0 + m = memoryview(arr) + assert module.get_releasebuffer_cnt() == 0 + del m + self.debug_collect() + assert module.get_releasebuffer_cnt() == 1 + def test_pickle(self): import pickle module = self.import_module(name='array') @@ -107,7 +124,7 @@ arr = Sub('i', [2]) res = [1, 2, 3] * arr assert res == [1, 2, 3, 1, 2, 3] - + val = module.readbuffer_as_string(arr) assert val == struct.pack('i', 2) From pypy.commits at gmail.com Thu Mar 2 06:27:30 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Mar 2017 03:27:30 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Fix call to bf_releasebuffer: don't segfault, don't do it for old-style buffers Message-ID: <58b801a2.53be190a.f3883.8afc@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90473:4d7d7bc8925c Date: 2017-03-02 12:20 +0100 http://bitbucket.org/pypy/pypy/changeset/4d7d7bc8925c/ Log: Fix call to bf_releasebuffer: don't segfault, don't do it for old- style buffers diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -7,7 +7,7 @@ from rpython.rlib import rgc # Force registration of gc.collect from pypy.module.cpyext.api import ( slot_function, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES, - pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr) + pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr, cts) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, ternaryfunc, binaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, @@ -353,26 +353,27 @@ def releasebuffer(self): if self.pyobj: if self.needs_decref: + if self.releasebufferproc: + func_target = rffi.cast(releasebufferproc, self.releasebufferproc) + with lltype.scoped_alloc(Py_buffer) as pybuf: + pybuf.c_buf = self.ptr + pybuf.c_len = self.size + pybuf.c_ndim = cts.cast('int', self.ndim) + pybuf.c_shape = cts.cast('Py_ssize_t*', pybuf.c__shape) + pybuf.c_strides = cts.cast('Py_ssize_t*', pybuf.c__strides) + for i in range(self.ndim): + pybuf.c_shape[i] = self.shape[i] + pybuf.c_strides[i] = self.strides[i] + if self.format: + pybuf.c_format = rffi.str2charp(self.format) + else: + pybuf.c_format = rffi.str2charp("B") + generic_cpy_call(self.space, func_target, self.pyobj, pybuf) decref(self.space, self.pyobj) self.pyobj = lltype.nullptr(PyObject.TO) else: #do not call twice return - if self.releasebufferproc: - func_target = rffi.cast(releasebufferproc, self.releasebufferproc) - with lltype.scoped_alloc(Py_buffer) as pybuf: - pybuf.c_buf = self.ptr - pybuf.c_len = self.size - pybuf.c_ndim = rffi.cast(rffi.INT_real, self.ndim) - for i in range(self.ndim): - pybuf.c_shape[i] = self.shape[i] - pybuf.c_strides[i] = self.strides[i] - if self.format: - pybuf.c_format = rffi.str2charp(self.format) - else: - pybuf.c_format = rffi.str2charp("B") - generic_cpy_call(self.space, func_target, self.pyobj, pybuf) - self.releasebufferproc = rffi.cast(rffi.VOIDP, 0) def getlength(self): return self.size From pypy.commits at gmail.com Thu Mar 2 06:31:07 2017 From: pypy.commits at gmail.com (Raemi) Date: Thu, 02 Mar 2017 03:31:07 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) make shadowstack really threadlocal Message-ID: <58b8027b.5e142e0a.8567d.6f47@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90474:b7c2b62351e0 Date: 2017-03-02 12:30 +0100 http://bitbucket.org/pypy/pypy/changeset/b7c2b62351e0/ Log: (arigo, remi) make shadowstack really threadlocal shadowstack push/pop was still non-tl, and we didn't visit all stacks of all threads during minor collection diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -897,6 +897,7 @@ self.set_nursery_top(self.nursery_barriers.popleft()) else: rgil.master_request_safepoint() + # we are the only thread here; all others are in gc-safepoints minor_collection_count += 1 if minor_collection_count == 1: diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -11,6 +11,8 @@ BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr) from rpython.rtyper.rbuiltin import gen_cast +from rpython.rlib import rthread + class ShadowStackFrameworkGCTransformer(BaseFrameworkGCTransformer): def annotate_walker_functions(self, getfn): @@ -24,7 +26,8 @@ inline = True) def build_root_walker(self): - return ShadowStackRootWalker(self) + # XXX: select between GIL and NoGil variant + return NoGilShadowStackRootWalker(self) def push_roots(self, hop, keep_current_args=False): livevars = self.get_livevars_for_roots(hop, keep_current_args) @@ -55,6 +58,108 @@ hop.genop("gc_reload_possibly_moved", [v_newaddr, var]) +class NoGilShadowStackRootWalker(BaseRootWalker): + tl_shadowstack = rthread.ThreadLocalField(llmemory.Address, 'shadowstack') + tl_shadowstack_top = rthread.ThreadLocalField(llmemory.Address, 'shadowstack_top') + tl_synclock = rthread.ThreadLocalField(lltype.Signed, 'synclock') + + def __init__(self, gctransformer): + BaseRootWalker.__init__(self, gctransformer) + + tl_shadowstack_top = self.tl_shadowstack_top + tl_shadowstack = self.tl_shadowstack + + def incr_stack(n): + top = tl_shadowstack_top.getraw() + tl_shadowstack_top.setraw(top + n * sizeofaddr) + return top + self.incr_stack = incr_stack + + def decr_stack(n): + top = tl_shadowstack_top.getraw() - n * sizeofaddr + tl_shadowstack_top.setraw(top) + return top + self.decr_stack = decr_stack + + def walk_stack_root(callback, start, end): + gc = self.gc + addr = end + while addr != start: + addr -= sizeofaddr + if gc.points_to_valid_gc_object(addr): + callback(gc, addr) + self.rootstackhook = walk_stack_root + + def walk_thread_stack(collect_stack_root, tl): + # XXX: only visit if nursery_free was not NULL + base = (tl + tl_shadowstack._offset).address[0] + top = (tl + tl_shadowstack_top._offset).address[0] + self.rootstackhook(collect_stack_root, base, top) + self._walk_thread_stack = walk_thread_stack + + def push_stack(self, addr): + top = self.incr_stack(1) + top.address[0] = addr + + def pop_stack(self): + top = self.decr_stack(1) + return top.address[0] + + + def walk_stack_roots(self, collect_stack_root, is_minor=False): + rthread.enum_all_threadlocals(self._walk_thread_stack, + collect_stack_root) + + def need_thread_support(self, gctransformer, getfn): # NO GIL VERSION + # the interfacing between the threads and the GC is done via + # two completely ad-hoc operations at the moment: + # gc_thread_run and gc_thread_die. See docstrings below. + + tl_shadowstack = self.tl_shadowstack + tl_shadowstack_top = self.tl_shadowstack_top + tl_synclock = self.tl_synclock + + def thread_setup(): + allocate_shadow_stack() + tl_synclock.get_or_make_raw() # reference the field at least once + + def thread_run(): + # If it's the first time we see this thread, allocate + # a shadowstack. + if tl_shadowstack.get_or_make_raw() == llmemory.NULL: + allocate_shadow_stack() + + def allocate_shadow_stack(): + root_stack_depth = 163840 + root_stack_size = sizeofaddr * root_stack_depth + ss = llmemory.raw_malloc(root_stack_size) + if not ss: + raise MemoryError + tl_shadowstack.setraw(ss) + tl_shadowstack_top.setraw(ss) + allocate_shadow_stack._dont_inline_ = True + + def thread_die(): + """Called just before the final GIL release done by a dying + thread. After a thread_die(), no more gc operation should + occur in this thread. + """ + p = tl_shadowstack.get_or_make_raw() + tl_shadowstack.setraw(llmemory.NULL) + tl_shadowstack_top.setraw(llmemory.NULL) + llmemory.raw_free(p) + + self.thread_setup = thread_setup + self.thread_run_ptr = getfn(thread_run, [], annmodel.s_None, + inline=True, minimal_transform=False) + self.thread_die_ptr = getfn(thread_die, [], annmodel.s_None, + minimal_transform=False) + + def need_stacklet_support(self, gctransformer, getfn): + from rpython.rlib import _stacklet_shadowstack + _stacklet_shadowstack.complete_destrptr(gctransformer) + + class ShadowStackRootWalker(BaseRootWalker): def __init__(self, gctransformer): BaseRootWalker.__init__(self, gctransformer) @@ -104,7 +209,7 @@ self.rootstackhook(collect_stack_root, gcdata.root_stack_base, gcdata.root_stack_top) - def need_thread_support_WITH_GIL(self, gctransformer, getfn): + def need_thread_support(self, gctransformer, getfn): from rpython.rlib import rthread # xxx fish gcdata = self.gcdata # the interfacing between the threads and the GC is done via @@ -218,51 +323,6 @@ annmodel.s_None, minimal_transform=False) - def need_thread_support(self, gctransformer, getfn): # NO GIL VERSION - from rpython.rlib import rthread - gcdata = self.gcdata - # the interfacing between the threads and the GC is done via - # two completely ad-hoc operations at the moment: - # gc_thread_run and gc_thread_die. See docstrings below. - - tl_shadowstack = rthread.ThreadLocalField(llmemory.Address, - 'shadowstack') - tl_synclock = rthread.ThreadLocalField(lltype.Signed, 'synclock') - - def thread_setup(): - allocate_shadow_stack() - tl_synclock.get_or_make_raw() # reference the field at least once - - def thread_run(): - # If it's the first time we see this thread, allocate - # a shadowstack. - if tl_shadowstack.get_or_make_raw() == llmemory.NULL: - allocate_shadow_stack() - - def allocate_shadow_stack(): - root_stack_depth = 163840 - root_stack_size = sizeofaddr * root_stack_depth - ss = llmemory.raw_malloc(root_stack_size) - if not ss: - raise MemoryError - tl_shadowstack.setraw(ss) - allocate_shadow_stack._dont_inline_ = True - - def thread_die(): - """Called just before the final GIL release done by a dying - thread. After a thread_die(), no more gc operation should - occur in this thread. - """ - p = tl_shadowstack.get_or_make_raw() - tl_shadowstack.setraw(llmemory.NULL) - llmemory.raw_free(p) - - self.thread_setup = thread_setup - self.thread_run_ptr = getfn(thread_run, [], annmodel.s_None, - inline=True, minimal_transform=False) - self.thread_die_ptr = getfn(thread_die, [], annmodel.s_None, - minimal_transform=False) - def need_stacklet_support(self, gctransformer, getfn): from rpython.rlib import _stacklet_shadowstack _stacklet_shadowstack.complete_destrptr(gctransformer) From pypy.commits at gmail.com Thu Mar 2 06:48:31 2017 From: pypy.commits at gmail.com (stevie_92) Date: Thu, 02 Mar 2017 03:48:31 -0800 (PST) Subject: [pypy-commit] pypy default: (mjacob, stevie) added SO variable to sysconfig Message-ID: <58b8068f.a9212e0a.e4b11.937a@mx.google.com> Author: Stefan Beyer Branch: Changeset: r90475:bbcb5f789c19 Date: 2017-03-02 12:39 +0100 http://bitbucket.org/pypy/pypy/changeset/bbcb5f789c19/ Log: (mjacob, stevie) added SO variable to sysconfig diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py --- a/lib_pypy/_sysconfigdata.py +++ b/lib_pypy/_sysconfigdata.py @@ -1,2 +1,5 @@ +import imp + build_time_vars = { + "SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] } From pypy.commits at gmail.com Thu Mar 2 07:25:51 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 04:25:51 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58b80f4f.97e1190a.47c64.9143@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90477:f8ce293591b5 Date: 2017-03-02 13:22 +0100 http://bitbucket.org/pypy/pypy/changeset/f8ce293591b5/ Log: hg merge default diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py --- a/lib_pypy/_sysconfigdata.py +++ b/lib_pypy/_sysconfigdata.py @@ -1,2 +1,5 @@ +import imp + build_time_vars = { + "SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] } From pypy.commits at gmail.com Thu Mar 2 07:25:53 2017 From: pypy.commits at gmail.com (stevie_92) Date: Thu, 02 Mar 2017 04:25:53 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Added EXT_SUFFIX for sysconfig, changed EXT_SUFFIX to Python 3 format Message-ID: <58b80f51.504f190a.d3b7f.93bf@mx.google.com> Author: Stefan Beyer Branch: py3.5 Changeset: r90478:215564273ab3 Date: 2017-03-02 12:54 +0100 http://bitbucket.org/pypy/pypy/changeset/215564273ab3/ Log: Added EXT_SUFFIX for sysconfig, changed EXT_SUFFIX to Python 3 format diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py --- a/lib-python/3/distutils/sysconfig_pypy.py +++ b/lib-python/3/distutils/sysconfig_pypy.py @@ -60,8 +60,8 @@ def _init_posix(): """Initialize the module as appropriate for POSIX systems.""" - so_list = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION] - so_ext = (so_list or ['.so'])[0] + so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] + g = {} g['CC'] = "gcc -pthread" g['CXX'] = "g++ -pthread" @@ -69,8 +69,9 @@ g['CFLAGS'] = "-DNDEBUG -O2" g['CCSHARED'] = "-fPIC" g['LDSHARED'] = "gcc -pthread -shared" - g['SO'] = so_ext - g['SHLIB_SUFFIX'] = g['SO'] + g['EXT_SUFFIX'] = so_ext + g['SHLIB_SUFFIX'] = so_ext + g['SO'] = so_ext # deprecated in Python 3, for backward compatibility g['AR'] = "ar" g['ARFLAGS'] = "rc" g['EXE'] = "" diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py --- a/lib_pypy/_sysconfigdata.py +++ b/lib_pypy/_sysconfigdata.py @@ -1,5 +1,9 @@ import imp +so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] + build_time_vars = { - "SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] + "EXT_SUFFIX": so_ext, + "SHLIB_SUFFIX": so_ext, + "SO": so_ext # deprecated in Python 3, for backward compatibility } diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -2,7 +2,7 @@ Implementation of the interpreter-level default import logic. """ -import sys, os, stat +import sys, os, stat, platform from pypy.interpreter.module import Module, init_extra_module_attrs from pypy.interpreter.gateway import interp2app, unwrap_spec @@ -22,7 +22,7 @@ SO = '.pyd' if _WIN32 else '.so' PREFIX = 'pypy3-' -DEFAULT_SOABI = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2]) +DEFAULT_SOABI_BASE = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2]) PYC_TAG = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2]) # 'pypy3-XY' @@ -35,7 +35,7 @@ if space.config.objspace.soabi is not None: soabi = space.config.objspace.soabi else: - soabi = DEFAULT_SOABI + soabi = DEFAULT_SOABI_BASE if not soabi: return SO @@ -43,6 +43,17 @@ if not space.config.translating: soabi += 'i' + platform_name = sys.platform + if platform_name == 'linux2': + platform_name = 'linux' + + soabi += '-' + platform.machine() + '-' + platform_name + + if platform_name == 'linux': + soabi += '-gnu' + if sys.maxsize == (2**31 - 1) and platform.machine() == 'x86_64': + soabi += 'x32' + return '.' + soabi + SO def log_pyverbose(space, level, message): From pypy.commits at gmail.com Thu Mar 2 07:19:44 2017 From: pypy.commits at gmail.com (Raemi) Date: Thu, 02 Mar 2017 04:19:44 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) enter safepoint in more places and fix issue with non-empty nursery assert Message-ID: <58b80de0.13542e0a.69d3d.907e@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90476:e1ade06bc0ab Date: 2017-03-02 13:19 +0100 http://bitbucket.org/pypy/pypy/changeset/e1ade06bc0ab/ Log: (arigo, remi) enter safepoint in more places and fix issue with non- empty nursery assert diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -800,6 +800,8 @@ def collect(self, gen=2): """Do a minor (gen=0), start a major (gen=1), or do a full major (gen>=2) collection.""" + rgil.enter_master_section() + rgil.master_request_safepoint() if gen < 0: self._minor_collection() # dangerous! no major GC cycle progress elif gen <= 1: @@ -809,6 +811,7 @@ else: self.minor_and_major_collection() self.rrc_invoke_callback() + rgil.leave_master_section() def minor_collection_with_major_progress(self, extrasize=0): @@ -983,8 +986,11 @@ # then when the major collection finishes it will raise # MemoryError. if self.threshold_reached(raw_malloc_usage(totalsize)): + rgil.enter_master_section() + rgil.master_request_safepoint() self.minor_collection_with_major_progress( raw_malloc_usage(totalsize) + self.nursery_size // 2) + rgil.leave_master_section() # # Check if the object would fit in the ArenaCollection. # Also, an object allocated from ArenaCollection must be old. @@ -1829,6 +1835,7 @@ # pointer. size_gc_header = self.gcheaderbuilder.size_gc_header nursery_barriers = self.AddressDeque() + nursery_barriers.append(self.nursery) if self.surviving_pinned_objects.non_empty(): self.surviving_pinned_objects.sort() next_pinned_object = self.surviving_pinned_objects.pop() @@ -1904,8 +1911,6 @@ self.nursery_barriers = nursery_barriers self.surviving_pinned_objects.delete() # - self.set_nursery_free(self.nursery) - self.set_nursery_top(self.nursery_barriers.popleft()) # # clear GCFLAG_PINNED_OBJECT_PARENT_KNOWN from all parents in the list. self.old_objects_pointing_to_pinned.foreach( From pypy.commits at gmail.com Thu Mar 2 07:57:19 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Mar 2017 04:57:19 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Call bf_releasebuffer slot from PyBuffer_Release Message-ID: <58b816af.0d50190a.d3d5c.9545@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90479:f03c4719447c Date: 2017-03-02 13:56 +0100 http://bitbucket.org/pypy/pypy/changeset/f03c4719447c/ Log: Call bf_releasebuffer slot from PyBuffer_Release diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -2,7 +2,7 @@ from pypy.interpreter.error import oefmt from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts, Py_buffer, - Py_ssize_t, Py_ssize_tP, + Py_ssize_t, Py_ssize_tP, generic_cpy_call, PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES) from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef @@ -60,6 +60,15 @@ Release the buffer view. This should be called when the buffer is no longer being used as it may free memory from it """ - Py_DecRef(space, view.c_obj) + obj = view.c_obj + if not obj: + return + assert obj.c_ob_type + as_buffer = obj.c_ob_type.c_tp_as_buffer + if as_buffer: + func = as_buffer.c_bf_releasebuffer + if func: + generic_cpy_call(space, func, obj, view) + Py_DecRef(space, obj) view.c_obj = lltype.nullptr(PyObject.TO) # XXX do other fields leak memory? diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2023,7 +2023,7 @@ } static void -array_releasebuffer(arrayobject* self) +array_releasebuffer(arrayobject* self, Py_buffer* view) { releasebuffer_cnt++; return; From pypy.commits at gmail.com Thu Mar 2 08:12:27 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 02 Mar 2017 05:12:27 -0800 (PST) Subject: [pypy-commit] pypy cpyext-callopt: add comments we found during the sprint session in the morning Message-ID: <58b81a3b.8c2b190a.1f257.9512@mx.google.com> Author: Richard Plangger Branch: cpyext-callopt Changeset: r90480:0dc57c3147d2 Date: 2017-03-02 14:11 +0100 http://bitbucket.org/pypy/pypy/changeset/0dc57c3147d2/ Log: add comments we found during the sprint session in the morning diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -44,6 +44,7 @@ class W_PyCFunctionObject(W_Root): + # TODO create a slightly different class depending on the c_ml_flags def __init__(self, space, ml, w_self, w_module=None): self.ml = ml self.name = rffi.charp2str(rffi.cast(rffi.CCHARP,self.ml.c_ml_name)) @@ -56,6 +57,7 @@ w_self = self.w_self flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags) flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST) + # XXX spent a lot of time if space.is_true(w_kw) and not flags & METH_KEYWORDS: raise oefmt(space.w_TypeError, "%s() takes no keyword arguments", self.name) @@ -192,8 +194,10 @@ def cfunction_descr_call(space, w_self, __args__): + # specialize depending on the W_PyCFunctionObject self = space.interp_w(W_PyCFunctionObject, w_self) args_w, kw_w = __args__.unpack() + # XXX __args__.unpack is slow w_args = space.newtuple(args_w) w_kw = space.newdict() for key, w_obj in kw_w.items(): diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -14,7 +14,7 @@ return True if '.' in modname: modname, rest = modname.split('.', 1) - if modname in ['unicodedata', 'gc', '_minimal_curses', 'cpyext']: + if modname in ['unicodedata', 'gc', '_minimal_curses']: return False else: rest = '' From pypy.commits at gmail.com Thu Mar 2 09:02:08 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Mar 2017 06:02:08 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Replace implementation of PyObject_GetBuffer with a translation of CPython's Message-ID: <58b825e0.d5c1190a.c932.95fd@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90481:7d9f2e667b3c Date: 2017-03-02 15:01 +0100 http://bitbucket.org/pypy/pypy/changeset/7d9f2e667b3c/ Log: Replace implementation of PyObject_GetBuffer with a translation of CPython's diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -1,15 +1,18 @@ +from rpython.rlib.objectmodel import keepalive_until_here +from pypy.interpreter.error import oefmt from pypy.module.cpyext.api import ( cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct, - bootstrap_function, Py_bufferP, slot_function) + bootstrap_function, Py_bufferP, slot_function, generic_cpy_call) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr, + PyObject, make_ref, as_pyobj, decref, from_ref, make_typedescr, get_typedescr, track_reference) from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import widen from pypy.objspace.std.memoryobject import W_MemoryView from pypy.module.cpyext.object import _dealloc from pypy.module.cpyext.import_ import PyImport_Import +from pypy.module.cpyext.buffer import PyObject_CheckBuffer PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView") @@ -114,18 +117,13 @@ raise an error if the object can't support a simpler view of its memory. 0 is returned on success and -1 on error.""" - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - except ValueError: - if not space.isinstance_w(w_obj, space.w_bytes): - # XXX Python 3? - raise BufferError("could not create buffer from object") - view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.bytes_w(w_obj), track_allocation=False)) - rffi.setintfield(view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, view) - view.c_obj = make_ref(space, w_obj) + if not PyObject_CheckBuffer(space, w_obj): + raise oefmt(space.w_TypeError, + "'%T' does not have the buffer interface", w_obj) + py_obj = as_pyobj(space, w_obj) + func = py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer + ret = generic_cpy_call(space, func, py_obj, view, flags) + keepalive_until_here(w_obj) return ret def fill_Py_buffer(space, buf, view): @@ -259,4 +257,3 @@ py_mem.c_view.c_shape = view.c_shape # XXX ignore suboffsets? return py_obj - From pypy.commits at gmail.com Thu Mar 2 09:10:40 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Mar 2017 06:10:40 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Re-enable slot function filling for __buffer__ -> bf_getbuffer Message-ID: <58b827e0.0be3190a.83fb9.974b@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90482:ec8a1da6df33 Date: 2017-03-02 15:09 +0100 http://bitbucket.org/pypy/pypy/changeset/ec8a1da6df33/ Log: Re-enable slot function filling for __buffer__ -> bf_getbuffer diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -13,10 +13,6 @@ flags = pyobj.c_ob_type.c_tp_flags if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer): return 1 - name = rffi.charp2str(cts.cast('char*', pyobj.c_ob_type.c_tp_name)) - if name in ('str', 'bytes'): - # XXX remove once wrapper of __buffer__ -> bf_getbuffer works - return 1 return 0 @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t, diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -748,8 +748,6 @@ ret = fill_Py_buffer(space, buf, view) return ret return 0 - # XXX remove this when it no longer crashes a translated PyPy - return slot_func = buff_w else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce From pypy.commits at gmail.com Thu Mar 2 09:22:54 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Mar 2017 06:22:54 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Trying to fix kqueue Message-ID: <58b82abe.0b202e0a.e47fc.9ad1@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90483:dd216e917dc6 Date: 2017-03-02 15:22 +0100 http://bitbucket.org/pypy/pypy/changeset/dd216e917dc6/ Log: Trying to fix kqueue diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py --- a/pypy/module/select/interp_kqueue.py +++ b/pypy/module/select/interp_kqueue.py @@ -1,8 +1,10 @@ +import errno from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import oefmt from pypy.interpreter.error import exception_from_saved_errno, wrap_oserror from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.typedef import TypeDef, generic_new_descr, GetSetProperty +from pypy.interpreter import timeutils from rpython.rlib._rsocket_rffi import socketclose_no_errno from rpython.rlib.rarithmetic import r_uint from rpython.rlib import rposix @@ -50,6 +52,12 @@ ("tv_nsec", rffi.LONG), ]) +def fill_timespec(time_float, timespec_ptr): + sec = int(time_float) + nsec = int(1e9 * (time_float - sec)) + rffi.setintfield(timespec_ptr, 'c_tv_sec', sec) + rffi.setintfield(timespec_ptr, 'c_tv_nsec', nsec) + symbol_map = { "KQ_FILTER_READ": "EVFILT_READ", @@ -180,13 +188,11 @@ raise oefmt(space.w_ValueError, "Timeout must be None or >= 0, got %s", str(_timeout)) - XXX # fix test_select_signal.py first, for PEP475! - sec = int(_timeout) - nsec = int(1e9 * (_timeout - sec)) - rffi.setintfield(timeout, 'c_tv_sec', sec) - rffi.setintfield(timeout, 'c_tv_nsec', nsec) + fill_timespec(_timeout, ptimeout) + timeout_at = timeutils.monotonic(space) + _timeout ptimeout = timeout else: + timeout_at = 0.0 ptimeout = lltype.nullptr(timespec) if not space.is_w(w_changelist, space.w_None): @@ -204,29 +210,40 @@ else: pchangelist = lltype.nullptr(rffi.CArray(kevent)) - nfds = syscall_kevent(self.kqfd, - pchangelist, - changelist_len, - eventlist, - max_events, - ptimeout) - if nfds < 0: - raise exception_from_saved_errno(space, space.w_OSError) - else: - elist_w = [None] * nfds - for i in xrange(nfds): + while True: + nfds = syscall_kevent(self.kqfd, + pchangelist, + changelist_len, + eventlist, + max_events, + ptimeout) + if nfds >= 0: + break + if rposix.get_saved_errno() != errno.EINTR: + raise exception_from_saved_errno(space, + space.w_OSError) + space.getexecutioncontext().checksignals() + if ptimeout: + _timeout = (timeout_at - + timeutils.monotonic(space)) + if _timeout < 0.0: + _timeout = 0.0 + fill_timespec(_timeout, ptimeout) - evt = eventlist[i] + elist_w = [None] * nfds + for i in xrange(nfds): - w_event = W_Kevent(space) - w_event.ident = evt.c_ident - w_event.filter = evt.c_filter - w_event.flags = evt.c_flags - w_event.fflags = evt.c_fflags - w_event.data = evt.c_data - w_event.udata = evt.c_udata + evt = eventlist[i] - elist_w[i] = w_event + w_event = W_Kevent(space) + w_event.ident = evt.c_ident + w_event.filter = evt.c_filter + w_event.flags = evt.c_flags + w_event.fflags = evt.c_fflags + w_event.data = evt.c_data + w_event.udata = evt.c_udata + + elist_w[i] = w_event return space.newlist(elist_w) From pypy.commits at gmail.com Thu Mar 2 09:31:01 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 06:31:01 -0800 (PST) Subject: [pypy-commit] pypy py3.5: (arigo) Fix. Message-ID: <58b82ca5.1b582e0a.468e3.9196@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90484:59e2a6469001 Date: 2017-03-02 15:29 +0100 http://bitbucket.org/pypy/pypy/changeset/59e2a6469001/ Log: (arigo) Fix. diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py --- a/pypy/module/select/interp_kqueue.py +++ b/pypy/module/select/interp_kqueue.py @@ -188,7 +188,7 @@ raise oefmt(space.w_ValueError, "Timeout must be None or >= 0, got %s", str(_timeout)) - fill_timespec(_timeout, ptimeout) + fill_timespec(_timeout, timeout) timeout_at = timeutils.monotonic(space) + _timeout ptimeout = timeout else: From pypy.commits at gmail.com Thu Mar 2 09:34:18 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Mar 2017 06:34:18 -0800 (PST) Subject: [pypy-commit] pypy py3.5: FreeBSD "fix", identical to CPython's test_kqueue Message-ID: <58b82d6a.1c10190a.fc0a4.97a8@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90485:eac2dc51d93a Date: 2017-03-02 15:33 +0100 http://bitbucket.org/pypy/pypy/changeset/eac2dc51d93a/ Log: FreeBSD "fix", identical to CPython's test_kqueue diff --git a/pypy/module/select/test/test_kqueue.py b/pypy/module/select/test/test_kqueue.py --- a/pypy/module/select/test/test_kqueue.py +++ b/pypy/module/select/test/test_kqueue.py @@ -102,7 +102,10 @@ except socket.error as e: assert e.args[0] == errno.EINPROGRESS else: - assert False, "EINPROGRESS not raised" + #assert False, "EINPROGRESS not raised" + pass # FreeBSD doesn't raise an exception here + # (the above commented-out code is just like CPython's + # test_kqueue) server, addr = server_socket.accept() if sys.platform.startswith("darwin"): From pypy.commits at gmail.com Thu Mar 2 09:43:05 2017 From: pypy.commits at gmail.com (Raemi) Date: Thu, 02 Mar 2017 06:43:05 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) disable pinning for now Message-ID: <58b82f79.0163190a.399bf.9928@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90486:ed5c6b5b1083 Date: 2017-03-02 15:42 +0100 http://bitbucket.org/pypy/pypy/changeset/ed5c6b5b1083/ Log: (arigo, remi) disable pinning for now diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -530,6 +530,7 @@ # Estimate this number conservatively bigobj = self.nonlarge_max + 1 self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) + self.max_number_of_pinned_objects = 0 # XXX # # Round up ll_assert((self.cache_line_min & (self.cache_line_min - 1)) == 0, @@ -2322,13 +2323,8 @@ debug_start("gc-collect-step") debug_print("starting gc state: ", GC_STATES[self.gc_state]) # Debugging checks - if self.pinned_objects_in_nursery == 0: - ll_assert(self.get_nursery_free() == llmemory.NULL, + ll_assert(self.get_nursery_free() == llmemory.NULL, "nursery not empty in major_collection_step()") - else: - # XXX try to add some similar check to the above one for the case - # that the nursery still contains some pinned objects (groggi) - pass self.debug_check_consistency() # diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1435,12 +1435,20 @@ def __init__(self, prev): self.prev = prev + class State: + pass + state = State() + def bootstrap(): rthread.gc_thread_start() x = None for i in range(1000000): x = X(x) - os.write(1, "hi there\n") + + state.lock.acquire(True) + os.write(1, "counter=%d\n" % state.counter) + state.counter -= 1 + state.lock.release() rthread.gc_thread_die() def new_thread(): @@ -1450,14 +1458,19 @@ def entry_point(argv): os.write(1, "hello world\n") # start 5 new threads - ident1 = new_thread() - ident2 = new_thread() - ident3 = new_thread() - ident4 = new_thread() - ident5 = new_thread() - # wait for the 5 threads to finish - time.sleep(1) - gc.collect() + TS = 30 + state.lock = rthread.allocate_lock() + state.counter = TS + + for _ in range(TS): + new_thread() + + while True: + time.sleep(0.5) + gc.collect() + if state.counter == 0: + break + os.write(1, "all threads done\n") return 0 def runme(no__thread): From pypy.commits at gmail.com Thu Mar 2 09:53:44 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Mar 2017 06:53:44 -0800 (PST) Subject: [pypy-commit] pypy py3.5: no fstatat() function on OS/X Message-ID: <58b831f8.09512e0a.56eb0.9b80@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90487:f3ed9df206eb Date: 2017-03-02 15:53 +0100 http://bitbucket.org/pypy/pypy/changeset/f3ed9df206eb/ Log: no fstatat() function on OS/X diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py --- a/pypy/module/posix/interp_scandir.py +++ b/pypy/module/posix/interp_scandir.py @@ -161,7 +161,7 @@ if (self.flags & FLAG_LSTAT) == 0: # Unlike CPython, try to use fstatat() if possible dirfd = self.scandir_iterator.dirfd - if dirfd != -1: + if dirfd != -1 and rposix.HAVE_FSTATAT: st = rposix_stat.fstatat(self.name, dirfd, follow_symlinks=False) else: @@ -206,7 +206,7 @@ if must_call_stat: # Must call stat(). Try to use fstatat() if possible dirfd = self.scandir_iterator.dirfd - if dirfd != -1: + if dirfd != -1 and rposix.HAVE_FSTATAT: st = rposix_stat.fstatat(self.name, dirfd, follow_symlinks=True) else: From pypy.commits at gmail.com Thu Mar 2 10:03:06 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 02 Mar 2017 07:03:06 -0800 (PST) Subject: [pypy-commit] pypy default: Disable (temporarily?) impl_attach_gdb on Windows Message-ID: <58b8342a.8d202e0a.119d3.a002@mx.google.com> Author: Armin Rigo Branch: Changeset: r90488:529139a8ad35 Date: 2017-03-02 16:02 +0100 http://bitbucket.org/pypy/pypy/changeset/529139a8ad35/ Log: Disable (temporarily?) impl_attach_gdb on Windows diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -524,7 +524,8 @@ ll_attach = rffi.llexternal("AttachToVS", [], lltype.Void, compilation_info=make_vs_attach_eci()) def impl_attach_gdb(): - ll_attach() + #ll_attach() + print "AttachToVS is disabled at the moment (compilation failure)" register_external(attach_gdb, [], result=None, export_name="impl_attach_gdb", llimpl=impl_attach_gdb) From pypy.commits at gmail.com Thu Mar 2 10:15:30 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 02 Mar 2017 07:15:30 -0800 (PST) Subject: [pypy-commit] pypy cpyext-callopt: specialization for METH_NOARGS & METH_O Message-ID: <58b83712.90202e0a.ae30f.a09f@mx.google.com> Author: Richard Plangger Branch: cpyext-callopt Changeset: r90489:68d3252f2685 Date: 2017-03-02 15:17 +0100 http://bitbucket.org/pypy/pypy/changeset/68d3252f2685/ Log: specialization for METH_NOARGS & METH_O diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -42,7 +42,6 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) - class W_PyCFunctionObject(W_Root): # TODO create a slightly different class depending on the c_ml_flags def __init__(self, space, ml, w_self, w_module=None): @@ -57,8 +56,7 @@ w_self = self.w_self flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags) flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST) - # XXX spent a lot of time - if space.is_true(w_kw) and not flags & METH_KEYWORDS: + if not flags & METH_KEYWORDS and space.is_true(w_kw): raise oefmt(space.w_TypeError, "%s() takes no keyword arguments", self.name) @@ -98,6 +96,20 @@ else: return space.w_None +class W_PyCFunctionObjectNoArgs(W_PyCFunctionObject): + def call(self, space, w_self, w_args, w_kw): + # Call the C function + if w_self is None: + w_self = self.w_self + func = self.ml.c_ml_meth + return generic_cpy_call(space, func, w_self, None) + +class W_PyCFunctionObjectSingleObject(W_PyCFunctionObject): + def call(self, space, w_self, w_o, w_kw): + if w_self is None: + w_self = self.w_self + func = self.ml.c_ml_meth + return generic_cpy_call(space, func, w_self, w_o) class W_PyCMethodObject(W_PyCFunctionObject): w_self = None @@ -192,6 +204,25 @@ space.setitem(w_kw, space.newtext(key), w_obj) return self.call(space, w_self, w_args, w_kw) +def cfunction_descr_call_noargs(space, w_self, __args__): + # special case for calling with flags METH_NOARGS + self = space.interp_w(W_PyCFunctionObject, w_self) + length = len(__args__.arguments_w) + if length != 0: + raise oefmt(space.w_TypeError, + "%s() takes no arguments", self.name) + return self.call(space, None, None, None) + +def cfunction_descr_call_single_object(space, w_self, __args__): + # special case for calling with flags METH_O + self = space.interp_w(W_PyCFunctionObjectSingleObject, w_self) + length = len(__args__.arguments_w) + if length != 1: + raise oefmt(space.w_TypeError, + "%s() takes exactly one argument (%d given)", + self.name, length) + o_w = __args__.firstarg() + return self.call(space, None, o_w, None) def cfunction_descr_call(space, w_self, __args__): # specialize depending on the W_PyCFunctionObject @@ -241,6 +272,26 @@ ) W_PyCFunctionObject.typedef.acceptable_as_base_class = False +W_PyCFunctionObjectNoArgs.typedef = TypeDef( + 'builtin_function_or_method', + __call__ = interp2app(cfunction_descr_call_noargs), + __doc__ = GetSetProperty(W_PyCFunctionObjectNoArgs.get_doc), + __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObjectNoArgs), + __name__ = interp_attrproperty('name', cls=W_PyCFunctionObjectNoArgs, + wrapfn="newtext_or_none"), + ) +W_PyCFunctionObjectNoArgs.typedef.acceptable_as_base_class = False + +W_PyCFunctionObjectSingleObject.typedef = TypeDef( + 'builtin_function_or_method', + __call__ = interp2app(cfunction_descr_call_single_object), + __doc__ = GetSetProperty(W_PyCFunctionObjectSingleObject.get_doc), + __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObjectSingleObject), + __name__ = interp_attrproperty('name', cls=W_PyCFunctionObjectSingleObject, + wrapfn="newtext_or_none"), + ) +W_PyCFunctionObjectSingleObject.typedef.acceptable_as_base_class = False + W_PyCMethodObject.typedef = TypeDef( 'method', __get__ = interp2app(cmethod_descr_get), diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -1,11 +1,13 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ - METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING + METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING, \ + METH_NOARGS, METH_O from pypy.module.cpyext.pyobject import PyObject, as_pyobj from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, - PyMethodDef, PyDescr_NewClassMethod, PyStaticMethod_New) + PyMethodDef, PyDescr_NewClassMethod, PyStaticMethod_New, + W_PyCFunctionObjectNoArgs, W_PyCFunctionObjectSingleObject) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.state import State from pypy.interpreter.error import oefmt @@ -71,6 +73,12 @@ space.newtext(rffi.charp2str(doc))) return w_mod # borrowed result kept alive in PyImport_AddModule() +def _create_pyc_function_object(space, method, w_self, w_name, flags): + if flags == METH_NOARGS: + return W_PyCFunctionObjectNoArgs(method, w_self, w_name) + if flags == METH_O: + return W_PyCFunctionObjectSingleObject(method, w_self, w_name) + return W_PyCFunctionObject(method, w_self, w_name) def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None): w_name = space.newtext_or_none(name) @@ -90,7 +98,8 @@ raise oefmt(space.w_ValueError, "module functions cannot set METH_CLASS or " "METH_STATIC") - w_obj = W_PyCFunctionObject(space, method, w_self, w_name) + w_obj = _create_pyc_function_object(space, method, w_self, + w_name, flags) else: if methodname in dict_w and not (flags & METH_COEXIST): continue From pypy.commits at gmail.com Thu Mar 2 10:15:32 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 02 Mar 2017 07:15:32 -0800 (PST) Subject: [pypy-commit] pypy cpyext-callopt: (antonio, plan_rich) more simplifiactions, W_CPyFunctionObjectSingleObject.typedef uses W_CPyFunctionObject.typedef as __base parameter Message-ID: <58b83714.4f5c190a.cb3de.7dee@mx.google.com> Author: Richard Plangger Branch: cpyext-callopt Changeset: r90490:9cbc8bd76297 Date: 2017-03-02 16:14 +0100 http://bitbucket.org/pypy/pypy/changeset/9cbc8bd76297/ Log: (antonio, plan_rich) more simplifiactions, W_CPyFunctionObjectSingleObject.typedef uses W_CPyFunctionObject.typedef as __base parameter diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -204,25 +204,15 @@ space.setitem(w_kw, space.newtext(key), w_obj) return self.call(space, w_self, w_args, w_kw) -def cfunction_descr_call_noargs(space, w_self, __args__): +def cfunction_descr_call_noargs(space, w_self): # special case for calling with flags METH_NOARGS - self = space.interp_w(W_PyCFunctionObject, w_self) - length = len(__args__.arguments_w) - if length != 0: - raise oefmt(space.w_TypeError, - "%s() takes no arguments", self.name) + self = space.interp_w(W_PyCFunctionObjectNoArgs, w_self) return self.call(space, None, None, None) -def cfunction_descr_call_single_object(space, w_self, __args__): +def cfunction_descr_call_single_object(space, w_self, w_o): # special case for calling with flags METH_O self = space.interp_w(W_PyCFunctionObjectSingleObject, w_self) - length = len(__args__.arguments_w) - if length != 1: - raise oefmt(space.w_TypeError, - "%s() takes exactly one argument (%d given)", - self.name, length) - o_w = __args__.firstarg() - return self.call(space, None, o_w, None) + return self.call(space, None, w_o, None) def cfunction_descr_call(space, w_self, __args__): # specialize depending on the W_PyCFunctionObject @@ -273,7 +263,7 @@ W_PyCFunctionObject.typedef.acceptable_as_base_class = False W_PyCFunctionObjectNoArgs.typedef = TypeDef( - 'builtin_function_or_method', + 'builtin_function_or_method', W_PyCFunctionObject.typedef, __call__ = interp2app(cfunction_descr_call_noargs), __doc__ = GetSetProperty(W_PyCFunctionObjectNoArgs.get_doc), __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObjectNoArgs), @@ -283,7 +273,7 @@ W_PyCFunctionObjectNoArgs.typedef.acceptable_as_base_class = False W_PyCFunctionObjectSingleObject.typedef = TypeDef( - 'builtin_function_or_method', + 'builtin_function_or_method', W_PyCFunctionObject.typedef, __call__ = interp2app(cfunction_descr_call_single_object), __doc__ = GetSetProperty(W_PyCFunctionObjectSingleObject.get_doc), __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObjectSingleObject), diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -74,11 +74,12 @@ return w_mod # borrowed result kept alive in PyImport_AddModule() def _create_pyc_function_object(space, method, w_self, w_name, flags): + flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST) if flags == METH_NOARGS: - return W_PyCFunctionObjectNoArgs(method, w_self, w_name) + return W_PyCFunctionObjectNoArgs(space, method, w_self, w_name) if flags == METH_O: - return W_PyCFunctionObjectSingleObject(method, w_self, w_name) - return W_PyCFunctionObject(method, w_self, w_name) + return W_PyCFunctionObjectSingleObject(space, method, w_self, w_name) + return W_PyCFunctionObject(space, method, w_self, w_name) def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None): w_name = space.newtext_or_none(name) From pypy.commits at gmail.com Thu Mar 2 10:25:00 2017 From: pypy.commits at gmail.com (Raemi) Date: Thu, 02 Mar 2017 07:25:00 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) make exception data thread-local Message-ID: <58b8394c.18532e0a.37a74.9d21@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90491:e5602c37c2a3 Date: 2017-03-02 16:24 +0100 http://bitbucket.org/pypy/pypy/changeset/e5602c37c2a3/ Log: (arigo, remi) make exception data thread-local diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py --- a/rpython/memory/gctypelayout.py +++ b/rpython/memory/gctypelayout.py @@ -432,6 +432,12 @@ appendto = self.addresses_of_static_ptrs else: return + elif hasattr(TYPE, "_hints") and TYPE._hints.get('is_excdata'): + # The exception data's value object is skipped: it's a thread- + # local data structure. We assume that objects are stored only + # temporarily there, so it is always cleared at the point where we + # collect the roots. + return else: appendto = self.addresses_of_static_ptrs_in_nongc for a in gc_pointers_inside(value, adr, mutable_only=True): diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1442,7 +1442,7 @@ def bootstrap(): rthread.gc_thread_start() x = None - for i in range(1000000): + for i in range(100000): x = X(x) state.lock.acquire(True) @@ -1458,7 +1458,7 @@ def entry_point(argv): os.write(1, "hello world\n") # start 5 new threads - TS = 30 + TS = 100 state.lock = rthread.allocate_lock() state.counter = TS @@ -1466,7 +1466,7 @@ new_thread() while True: - time.sleep(0.5) + time.sleep(0.1) gc.collect() if state.counter == 0: break diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -452,7 +452,8 @@ def setup_excdata(self): EXCDATA = lltype.Struct('ExcData', ('exc_type', self.lltype_of_exception_type), - ('exc_value', self.lltype_of_exception_value)) + ('exc_value', self.lltype_of_exception_value), + hints={'thread_local': True, 'is_excdata': True}) self.EXCDATA = EXCDATA exc_data = lltype.malloc(EXCDATA, immortal=True) From pypy.commits at gmail.com Thu Mar 2 11:17:05 2017 From: pypy.commits at gmail.com (Raemi) Date: Thu, 02 Mar 2017 08:17:05 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) 'fix' for potential crash Message-ID: <58b84581.09512e0a.56eb0.9f8c@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90492:e32f17803824 Date: 2017-03-02 17:15 +0100 http://bitbucket.org/pypy/pypy/changeset/e32f17803824/ Log: (arigo, remi) 'fix' for potential crash diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -94,6 +94,12 @@ # XXX: only visit if nursery_free was not NULL base = (tl + tl_shadowstack._offset).address[0] top = (tl + tl_shadowstack_top._offset).address[0] + if base == llmemory.NULL or top == llmemory.NULL: + # gctransform/shadowstack.py does not set these two fields + # atomically. Hence, if one is still NULL, we don't need to + # walk that new thread's shadowstack (XXX: compiler may reorder + # without barriers) + return self.rootstackhook(collect_stack_root, base, top) self._walk_thread_stack = walk_thread_stack From pypy.commits at gmail.com Thu Mar 2 12:11:45 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Mar 2017 09:11:45 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Skip test that fails untranslated (someone please kill ll2ctypes in a fire) Message-ID: <58b85251.12ce190a.95f91.9df4@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90493:e05f93a7addd Date: 2017-03-02 18:10 +0100 http://bitbucket.org/pypy/pypy/changeset/e05f93a7addd/ Log: Skip test that fails untranslated (someone please kill ll2ctypes in a fire) diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -63,8 +63,10 @@ a = array.array('c', 'text') b = buffer(a) assert module.roundtrip(b) == 'text' - + def test_releasebuffer(self): + if not self.runappdirect: + skip("Fails due to ll2ctypes nonsense") module = self.import_extension('foo', [ ("create_test", "METH_NOARGS", """ @@ -104,10 +106,10 @@ return 0; } - void releasebuffer(PyObject *obj, Py_buffer *view) { + void releasebuffer(PyObject *obj, Py_buffer *view) { cnt --; } - """, more_init=""" + """, more_init=""" type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); type->ht_type.tp_name = "Test"; From pypy.commits at gmail.com Thu Mar 2 12:22:16 2017 From: pypy.commits at gmail.com (Raemi) Date: Thu, 02 Mar 2017 09:22:16 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) deal with new threads popping up randomly Message-ID: <58b854c8.09512e0a.56eb0.a277@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90494:ea1fd293ee24 Date: 2017-03-02 18:21 +0100 http://bitbucket.org/pypy/pypy/changeset/ea1fd293ee24/ Log: (arigo, remi) deal with new threads popping up randomly if a new thread shows up and all others are waiting in a safepoint, it should also wait (by entering the slowpath of acquire GIL) diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h --- a/rpython/translator/c/src/thread.h +++ b/rpython/translator/c/src/thread.h @@ -52,18 +52,43 @@ //RPY_EXTERN long rpy_fastgil; #include "threadlocal.h" -#define _RPyGilAcquire() do { \ - if (!__sync_bool_compare_and_swap( \ - &RPY_THREADLOCALREF_GET(synclock), 0L, 1L)) \ - RPyGilAcquireSlowPath(); \ - } while (0) +/* + synclock is 3 bits: + bit 0: GIL acquired + bit 1: safepoint requested + bit 2: thread known and initialised (old) -#define _RPyGilRelease() do { \ - assert(RPY_THREADLOCALREF_GET(synclock) != 0L); \ - if (!__sync_bool_compare_and_swap( \ - &RPY_THREADLOCALREF_GET(synclock), 1L, 0L)) \ - RPyGilReleaseSlowPath(); \ - } while (0) + synclock possible values: + 000: new thread; GIL released; safepoint requested + 010: INVALID VALUE + 0?1: INVALID VALUE + 100: old thread; GIL released + 101: old thread; GIL acquired + 110: old thread; GIL released; safepoint requested + 111: old thread; GIL acquired; safepoint requested + + synclock transitions: + acquire: (possible values: ??0) + FASTPATH: 100 -> 101 + SLOWPATH: check safepoint; initialise new thread; ??? -> 101 + release: (possible values: 1?1): + FASTPATH: 101 -> 100 + SLOWPATH: signal "now at safepoint"; 111 -> 110 + */ + +#define _RPyGilAcquire() do { \ + assert((RPY_THREADLOCALREF_GET(synclock) & 0b001) == 0b0); \ + if (!__sync_bool_compare_and_swap( \ + &RPY_THREADLOCALREF_GET(synclock), 0b100L, 0b101L)) \ + RPyGilAcquireSlowPath(); \ + } while (0) + +#define _RPyGilRelease() do { \ + assert((RPY_THREADLOCALREF_GET(synclock) & 0b101) == 0b101); \ + if (!__sync_bool_compare_and_swap( \ + &RPY_THREADLOCALREF_GET(synclock), 0b101L, 0b100L)) \ + RPyGilReleaseSlowPath(); \ + } while (0) static inline long *_RPyFetchFastGil(void) { abort(); @@ -72,7 +97,7 @@ #define RPyGilYieldThread() do { \ assert(RPY_THREADLOCALREF_GET(synclock) & 1L); \ - if (RPY_THREADLOCALREF_GET(synclock) == 3L) { \ + if (RPY_THREADLOCALREF_GET(synclock) == 0b111L) { \ RPyGilYieldThreadSlowPath(); \ } \ } while (0) diff --git a/rpython/translator/c/src/thread_gil.c b/rpython/translator/c/src/thread_gil.c --- a/rpython/translator/c/src/thread_gil.c +++ b/rpython/translator/c/src/thread_gil.c @@ -1,13 +1,14 @@ #include #include #include +#include #include "threadlocal.h" static pthread_mutex_t master_mutex; static pthread_mutex_t sync_mutex; static pthread_cond_t sync_cond; -static long counter_of_threes = 0; +static long counter_of_sevens = 0; static long rpy_initialize = -42; @@ -26,7 +27,7 @@ if (err) abort(); - counter_of_threes = 0; // XXX: fork? + counter_of_sevens = 0; // XXX: fork? rpy_initialize = 0; } @@ -44,29 +45,35 @@ void RPyGilAcquireSlowPath(void) { - assert(RPY_THREADLOCALREF_GET(synclock) == 2); - /* wait until the master leaves the safe point */ pthread_mutex_lock(&master_mutex); - RPY_THREADLOCALREF_GET(synclock) = 1; + + long synclock = RPY_THREADLOCALREF_GET(synclock); + assert(synclock == 0b110 || synclock == 0b000); + + bool is_new_thread = synclock == 000; + if (is_new_thread) { + // TODO + } + + RPY_THREADLOCALREF_GET(synclock) = 0b101L; pthread_mutex_unlock(&master_mutex); } void RPyGilReleaseSlowPath(void) { - assert(RPY_THREADLOCALREF_GET(synclock) == 3); + pthread_mutex_lock(&sync_mutex); + assert(RPY_THREADLOCALREF_GET(synclock) == 0b111L); - pthread_mutex_lock(&sync_mutex); - - /* we are one of the THREES that the master is waiting for. Decrease the + /* we are one of the SEVENs that the master is waiting for. Decrease the * counter and signal the master if we are the last. */ - counter_of_threes--; - if (counter_of_threes == 0) + counter_of_sevens--; + if (counter_of_sevens == 0) pthread_cond_signal(&sync_cond); - /* set to TWO, so that Acquire above will wait until the master is finished + /* set to 110, so that Acquire above will wait until the master is finished * with its safe point */ - RPY_THREADLOCALREF_GET(synclock) = 2; + RPY_THREADLOCALREF_GET(synclock) = 0b110L; pthread_mutex_unlock(&sync_mutex); // continue without GIL } @@ -92,7 +99,7 @@ void RPyGilMasterRequestSafepoint(void) { pthread_mutex_lock(&sync_mutex); - assert(counter_of_threes == 0); + assert(counter_of_sevens == 0); /* signal all threads to enter safepoints */ OP_THREADLOCALREF_ACQUIRE(/* */); @@ -105,22 +112,25 @@ retry: switch (t->synclock) { - case 3: - assert(!"unexpected synclock=3 found"); + default: + fprintf(stderr, "ERROR: found synclock=%ld\n", t->synclock); abort(); - case 2: + case 0b000L: + /* new thread, no need to explicitly request safepoint */ + break; + case 0b110L: /* thread running in C code, already knows we want a safepoint */ break; - case 0: + case 0b100L: /* thread running in C code, make sure it checks for and enters * the safepoint before acquiring the "gil" again */ - if (__sync_bool_compare_and_swap(&t->synclock, 0, 2)) + if (__sync_bool_compare_and_swap(&t->synclock, 0b100L, 0b110L)) break; goto retry; - case 1: + case 0b101L: /* thread running normally, place request to enter safepoint */ - if (__sync_bool_compare_and_swap(&t->synclock, 1, 3)) { - counter_of_threes++; + if (__sync_bool_compare_and_swap(&t->synclock, 0b101L, 0b111L)) { + counter_of_sevens++; t->nursery_top = NULL; break; } @@ -129,8 +139,8 @@ } OP_THREADLOCALREF_RELEASE(/* */); - /* wait until all THREES entered their safepoints */ - while (counter_of_threes > 0) { + /* wait until all SEVENs entered their safepoints */ + while (counter_of_sevens > 0) { pthread_cond_wait(&sync_cond, &sync_mutex); } From pypy.commits at gmail.com Thu Mar 2 12:31:53 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 02 Mar 2017 09:31:53 -0800 (PST) Subject: [pypy-commit] pypy cpyext-callopt: (arigato, plan_rich) some structural changes, cannot have different typeobjects for such a function object, does not work currently. need a way to pass over the gateway without __args__ for 3 specializations Message-ID: <58b85709.6911190a.9b8e7.a0f0@mx.google.com> Author: Richard Plangger Branch: cpyext-callopt Changeset: r90495:dbba78b270fd Date: 2017-03-02 18:31 +0100 http://bitbucket.org/pypy/pypy/changeset/dbba78b270fd/ Log: (arigato, plan_rich) some structural changes, cannot have different typeobjects for such a function object, does not work currently. need a way to pass over the gateway without __args__ for 3 specializations diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -500,6 +500,8 @@ self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),)) def make_fastfunc(unwrap_spec, func): + if hasattr(func, '__cpyext_dispatch_hack'): + raise FastFuncNotSupported unwrap_info = UnwrapSpec_FastFunc_Unwrap() unwrap_info.apply_over(unwrap_spec) narg = unwrap_info.n @@ -692,6 +694,9 @@ arity, fastfunc = UnwrapSpec_FastFunc_Unwrap.make_fastfunc( unwrap_spec, func) except FastFuncNotSupported: + if hasattr(func, '__cpyext_dispatch_hack'): + self.__class__ = BuiltinCPyExtCallHack + self._func__args__ = [] if unwrap_spec == [ObjSpace, Arguments]: self.__class__ = BuiltinCodePassThroughArguments0 self.func__args__ = func @@ -769,6 +774,21 @@ # (verbose) performance hack below +class BuiltinCPyExtCallHack(BuiltinCode): + _immutable_ = True + + def fastcall_0(self, space, w_func): + # METH_NOARGS + try: + w_result = self.fastfunc_0(space) + except DescrMismatch: + raise oefmt(space.w_SystemError, "unexpected DescrMismatch error") + except Exception as e: + self.handle_exception(space, e) + w_result = None + if w_result is None: + w_result = space.w_None + return w_result class BuiltinCodePassThroughArguments0(BuiltinCode): _immutable_ = True diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -3,7 +3,7 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.function import ClassMethod, Method, StaticMethod -from pypy.interpreter.gateway import interp2app +from pypy.interpreter.gateway import interp2app, interpindirect2app from pypy.interpreter.typedef import ( GetSetProperty, TypeDef, interp_attrproperty, interp_attrproperty_w) from pypy.objspace.std.typeobject import W_TypeObject @@ -43,17 +43,28 @@ _dealloc(space, py_obj) class W_PyCFunctionObject(W_Root): - # TODO create a slightly different class depending on the c_ml_flags def __init__(self, space, ml, w_self, w_module=None): self.ml = ml self.name = rffi.charp2str(rffi.cast(rffi.CCHARP,self.ml.c_ml_name)) self.w_self = w_self self.w_module = w_module - def call(self, space, w_self, w_args, w_kw): + def descr_call(self, space, __args__): + # specialize depending on the W_PyCFunctionObject + args_w, kw_w = __args__.unpack() + + w_args = space.newtuple(args_w) + if len(kw_w) != 0: + # XXX __args__.unpack is slow + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.newtext(key), w_obj) + else: + w_kw = None + # Call the C function - if w_self is None: - w_self = self.w_self + # generic version if either METH_KEYWORDS or METH_OLDARGS is + # specified flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags) flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST) if not flags & METH_KEYWORDS and space.is_true(w_kw): @@ -64,21 +75,7 @@ length = space.int_w(space.len(w_args)) if flags & METH_KEYWORDS: func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) - return generic_cpy_call(space, func, w_self, w_args, w_kw) - elif flags & METH_NOARGS: - if length == 0: - return generic_cpy_call(space, func, w_self, None) - raise oefmt(space.w_TypeError, - "%s() takes no arguments", self.name) - elif flags & METH_O: - if length != 1: - raise oefmt(space.w_TypeError, - "%s() takes exactly one argument (%d given)", - self.name, length) - w_arg = space.getitem(w_args, space.newint(0)) - return generic_cpy_call(space, func, w_self, w_arg) - elif flags & METH_VARARGS: - return generic_cpy_call(space, func, w_self, w_args) + return generic_cpy_call(space, func, self, w_args, w_kw) else: # METH_OLDARGS, the really old style size = length if size == 1: @@ -87,7 +84,9 @@ w_arg = None else: w_arg = w_args - return generic_cpy_call(space, func, w_self, w_arg) + return generic_cpy_call(space, func, self, w_arg) + + descr_call.__cpyext_dispatch_hack = True def get_doc(self, space): doc = self.ml.c_ml_doc @@ -96,20 +95,23 @@ else: return space.w_None +class W_PyCFunctionObjectVarArgsOnly(W_PyCFunctionObject): + def descr_call(self, space, __args__): + # METH_VARARGS only + func = self.ml.c_ml_meth + return generic_cpy_call(space, func, self, space.newtuple(args_w)) + class W_PyCFunctionObjectNoArgs(W_PyCFunctionObject): - def call(self, space, w_self, w_args, w_kw): - # Call the C function - if w_self is None: - w_self = self.w_self + def descr_call(self, space): + # METH_NOARGS func = self.ml.c_ml_meth - return generic_cpy_call(space, func, w_self, None) + return generic_cpy_call(space, func, self, None) class W_PyCFunctionObjectSingleObject(W_PyCFunctionObject): - def call(self, space, w_self, w_o, w_kw): - if w_self is None: - w_self = self.w_self + def descr_call(self, space, w_o): + # special case for calling with flags METH_O func = self.ml.c_ml_meth - return generic_cpy_call(space, func, w_self, w_o) + return generic_cpy_call(space, func, self, w_o) class W_PyCMethodObject(W_PyCFunctionObject): w_self = None @@ -204,28 +206,6 @@ space.setitem(w_kw, space.newtext(key), w_obj) return self.call(space, w_self, w_args, w_kw) -def cfunction_descr_call_noargs(space, w_self): - # special case for calling with flags METH_NOARGS - self = space.interp_w(W_PyCFunctionObjectNoArgs, w_self) - return self.call(space, None, None, None) - -def cfunction_descr_call_single_object(space, w_self, w_o): - # special case for calling with flags METH_O - self = space.interp_w(W_PyCFunctionObjectSingleObject, w_self) - return self.call(space, None, w_o, None) - -def cfunction_descr_call(space, w_self, __args__): - # specialize depending on the W_PyCFunctionObject - self = space.interp_w(W_PyCFunctionObject, w_self) - args_w, kw_w = __args__.unpack() - # XXX __args__.unpack is slow - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - ret = self.call(space, None, w_args, w_kw) - return ret - def cmethod_descr_call(space, w_self, __args__): self = space.interp_w(W_PyCFunctionObject, w_self) args_w, kw_w = __args__.unpack() @@ -251,10 +231,9 @@ w_cls = space.type(w_obj) return Method(space, w_function, w_cls, space.w_None) - W_PyCFunctionObject.typedef = TypeDef( 'builtin_function_or_method', - __call__ = interp2app(cfunction_descr_call), + __call__ = interp2app(W_PyCFunctionObject.descr_call), __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc), __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject), __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject, @@ -262,26 +241,6 @@ ) W_PyCFunctionObject.typedef.acceptable_as_base_class = False -W_PyCFunctionObjectNoArgs.typedef = TypeDef( - 'builtin_function_or_method', W_PyCFunctionObject.typedef, - __call__ = interp2app(cfunction_descr_call_noargs), - __doc__ = GetSetProperty(W_PyCFunctionObjectNoArgs.get_doc), - __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObjectNoArgs), - __name__ = interp_attrproperty('name', cls=W_PyCFunctionObjectNoArgs, - wrapfn="newtext_or_none"), - ) -W_PyCFunctionObjectNoArgs.typedef.acceptable_as_base_class = False - -W_PyCFunctionObjectSingleObject.typedef = TypeDef( - 'builtin_function_or_method', W_PyCFunctionObject.typedef, - __call__ = interp2app(cfunction_descr_call_single_object), - __doc__ = GetSetProperty(W_PyCFunctionObjectSingleObject.get_doc), - __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObjectSingleObject), - __name__ = interp_attrproperty('name', cls=W_PyCFunctionObjectSingleObject, - wrapfn="newtext_or_none"), - ) -W_PyCFunctionObjectSingleObject.typedef.acceptable_as_base_class = False - W_PyCMethodObject.typedef = TypeDef( 'method', __get__ = interp2app(cmethod_descr_get), diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -1,13 +1,14 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING, \ - METH_NOARGS, METH_O + METH_NOARGS, METH_O, METH_VARARGS from pypy.module.cpyext.pyobject import PyObject, as_pyobj from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, PyMethodDef, PyDescr_NewClassMethod, PyStaticMethod_New, - W_PyCFunctionObjectNoArgs, W_PyCFunctionObjectSingleObject) + W_PyCFunctionObjectNoArgs, W_PyCFunctionObjectSingleObject, + W_PyCFunctionObjectVarArgsOnly) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.state import State from pypy.interpreter.error import oefmt @@ -79,6 +80,8 @@ return W_PyCFunctionObjectNoArgs(space, method, w_self, w_name) if flags == METH_O: return W_PyCFunctionObjectSingleObject(space, method, w_self, w_name) + if flags == METH_VARARGS: + return W_PyCFunctionObjectVarArgsOnly(space, method, w_self, w_name) return W_PyCFunctionObject(space, method, w_self, w_name) def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None): diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py --- a/pypy/module/cpyext/test/test_methodobject.py +++ b/pypy/module/cpyext/test/test_methodobject.py @@ -40,6 +40,14 @@ } ''' ), + ('getarg_VARARGS', 'METH_VARARGS', + ''' + PyObject * i; + i = PyLong_FromLong((long)PyObject_Length(args)); + Py_INCREF(i); + return i; + ''' + ), ('isCFunction', 'METH_O', ''' if(PyCFunction_Check(args)) { @@ -83,6 +91,10 @@ raises(TypeError, mod.getarg_NO, 1) raises(TypeError, mod.getarg_NO, 1, 1) + assert mod.getarg_VARARGS() == 0 + assert mod.getarg_VARARGS(1) == 1 + raises(TypeError, mod.getarg_VARARGS, k=1) + assert mod.getarg_OLD(1) == 1 assert mod.getarg_OLD() is None assert mod.getarg_OLD(1, 2) == (1, 2) From pypy.commits at gmail.com Thu Mar 2 12:33:13 2017 From: pypy.commits at gmail.com (Raemi) Date: Thu, 02 Mar 2017 09:33:13 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) add todo after discussion Message-ID: <58b85759.4d29190a.1ff9b.a1de@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90496:fd450c9ca2ce Date: 2017-03-02 18:31 +0100 http://bitbucket.org/pypy/pypy/changeset/fd450c9ca2ce/ Log: (arigo, remi) add todo after discussion diff --git a/rpython/translator/c/src/thread_gil.c b/rpython/translator/c/src/thread_gil.c --- a/rpython/translator/c/src/thread_gil.c +++ b/rpython/translator/c/src/thread_gil.c @@ -53,7 +53,8 @@ bool is_new_thread = synclock == 000; if (is_new_thread) { - // TODO + // TODO: do the shadowstack.py:allocate_shadow_stack() here, then the + // walk_thread_stack() does not need to check for ss_top==NULL anymore. } RPY_THREADLOCALREF_GET(synclock) = 0b101L; From pypy.commits at gmail.com Thu Mar 2 15:49:14 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 12:49:14 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Fix. Message-ID: <58b8854a.14a9190a.c1163.a924@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90497:5f9ea0599935 Date: 2017-03-02 21:38 +0100 http://bitbucket.org/pypy/pypy/changeset/5f9ea0599935/ Log: Fix. diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -35,7 +35,7 @@ w_uni = space.wrap(u'abcd') assert space.text_w(w_uni) == 'abcd' w_uni = space.wrap(unichr(0xd921) + unichr(0xdddd)) - raises(UnicodeEncodeError, space.text_w, w_uni) + space.raises_w(space.w_UnicodeEncodeError, space.text_w, w_uni) class AppTestUnicodeStringStdOnly: From pypy.commits at gmail.com Thu Mar 2 15:49:17 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 12:49:17 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Raise ValueError instead of TypeError here. Message-ID: <58b8854d.1f002e0a.a2e24.59d8@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90498:ce30e89d7f14 Date: 2017-03-02 21:48 +0100 http://bitbucket.org/pypy/pypy/changeset/ce30e89d7f14/ Log: Raise ValueError instead of TypeError here. diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -500,6 +500,7 @@ assert u'x\ty'.expandtabs(-42) == u'xy' def test_translate(self): + import sys assert 'bbbc' == 'abababc'.translate({ord('a'):None}) assert 'iiic' == 'abababc'.translate({ord('a'):None, ord('b'):ord('i')}) assert 'iiix' == 'abababc'.translate({ord('a'):None, ord('b'):ord('i'), ord('c'):'x'}) @@ -511,6 +512,7 @@ assert 'abababc'.translate({ord('a'): ''}) == 'bbbc' raises(TypeError, 'hello'.translate) + raises(ValueError, "\xff".translate, {0xff: sys.maxunicode+1}) def test_maketrans(self): assert 'abababc' == 'abababc'.translate({'b': ''}) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -429,7 +429,7 @@ elif space.isinstance_w(w_newval, space.w_int): newval = space.int_w(w_newval) if newval < 0 or newval > maxunicode: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_ValueError, "character mapping must be in range(%s)", hex(maxunicode + 1)) result.append(unichr(newval)) From pypy.commits at gmail.com Thu Mar 2 16:08:02 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 13:08:02 -0800 (PST) Subject: [pypy-commit] pypy py3.5: (mjacob, stevie) Try to fix pickle tests in lib-python/3/test/test_descr.py. Message-ID: <58b889b2.55502e0a.2ec48.a36e@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90499:c402f2a419e6 Date: 2017-03-02 22:07 +0100 http://bitbucket.org/pypy/pypy/changeset/c402f2a419e6/ Log: (mjacob, stevie) Try to fix pickle tests in lib- python/3/test/test_descr.py. diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py --- a/pypy/objspace/std/objectobject.py +++ b/pypy/objspace/std/objectobject.py @@ -27,6 +27,9 @@ # and raises a TypeError if the condition holds true, this is done # just before reduce_2 is called in pypy state = getattr(obj, "__dict__", None) + # CPython returns None if the dict is empty + if state is not None and len(state) == 0: + state = None names = slotnames(cls) # not checking for list if names is not None: slots = {} diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -89,6 +89,11 @@ assert '__getnewargs__' not in seen assert '__getnewargs_ex__' not in seen + def test_reduce_state_empty_dict(self): + class X(object): + pass + assert X().__reduce_ex__(2)[2] is None + def test_default_format(self): class x(object): def __str__(self): From pypy.commits at gmail.com Thu Mar 2 16:23:16 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 13:23:16 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Change recursive repr test in lib-python/3/test/test_functools.py. Message-ID: <58b88d44.0d50190a.d3d5c.ac52@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90500:02636c123ae1 Date: 2017-03-02 22:22 +0100 http://bitbucket.org/pypy/pypy/changeset/02636c123ae1/ Log: Change recursive repr test in lib-python/3/test/test_functools.py. Our implementation has a slightly different __repr__, but since CPython changed it in commit 457fc9a69e7918e3c10319604903915c2f2ecd9a to be more similar to ours, I decided to change the test instead of our implementation. diff --git a/lib-python/3/test/test_functools.py b/lib-python/3/test/test_functools.py --- a/lib-python/3/test/test_functools.py +++ b/lib-python/3/test/test_functools.py @@ -233,21 +233,21 @@ f = self.partial(capture) f.__setstate__((f, (), {}, {})) try: - self.assertEqual(repr(f), '%s(%s(...))' % (name, name)) + self.assertEqual(repr(f), '%s(...)' % (name)) finally: f.__setstate__((capture, (), {}, {})) f = self.partial(capture) f.__setstate__((capture, (f,), {}, {})) try: - self.assertEqual(repr(f), '%s(%r, %s(...))' % (name, capture, name)) + self.assertEqual(repr(f), '%s(%r, ...)' % (name, capture)) finally: f.__setstate__((capture, (), {}, {})) f = self.partial(capture) f.__setstate__((capture, (), {'a': f}, {})) try: - self.assertEqual(repr(f), '%s(%r, a=%s(...))' % (name, capture, name)) + self.assertEqual(repr(f), '%s(%r, a=...)' % (name, capture)) finally: f.__setstate__((capture, (), {}, {})) From pypy.commits at gmail.com Thu Mar 2 16:54:40 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Mar 2017 13:54:40 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Move memoryview test from test_bufferobject to test_memoryobject Message-ID: <58b894a0.5d532e0a.d82cc.aeca@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90501:13effce84dd7 Date: 2017-03-02 18:20 +0100 http://bitbucket.org/pypy/pypy/changeset/13effce84dd7/ Log: Move memoryview test from test_bufferobject to test_memoryobject diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -63,75 +63,3 @@ a = array.array('c', 'text') b = buffer(a) assert module.roundtrip(b) == 'text' - - def test_releasebuffer(self): - if not self.runappdirect: - skip("Fails due to ll2ctypes nonsense") - module = self.import_extension('foo', [ - ("create_test", "METH_NOARGS", - """ - PyObject *obj; - obj = PyObject_New(PyObject, (PyTypeObject*)type); - return obj; - """), - ("get_cnt", "METH_NOARGS", - 'return PyLong_FromLong(cnt);'), - ("get_dealloc_cnt", "METH_NOARGS", - 'return PyLong_FromLong(dealloc_cnt);'), - ], - prologue=""" - static float test_data = 42.f; - static int cnt=0; - static int dealloc_cnt=0; - static PyHeapTypeObject * type=NULL; - - void dealloc(PyObject *self) { - dealloc_cnt++; - } - int getbuffer(PyObject *obj, Py_buffer *view, int flags) { - - cnt ++; - memset(view, 0, sizeof(Py_buffer)); - view->obj = obj; - /* see the CPython docs for why we need this incref: - https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer */ - Py_INCREF(obj); - view->ndim = 0; - view->buf = (void *) &test_data; - view->itemsize = sizeof(float); - view->len = 1; - view->strides = NULL; - view->shape = NULL; - view->format = "f"; - return 0; - } - - void releasebuffer(PyObject *obj, Py_buffer *view) { - cnt --; - } - """, more_init=""" - type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); - - type->ht_type.tp_name = "Test"; - type->ht_type.tp_basicsize = sizeof(PyObject); - type->ht_name = PyString_FromString("Test"); - type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_NEWBUFFER; - type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; - - type->ht_type.tp_dealloc = dealloc; - type->ht_type.tp_as_buffer = &type->as_buffer; - type->as_buffer.bf_getbuffer = getbuffer; - type->as_buffer.bf_releasebuffer = releasebuffer; - - if (PyType_Ready(&type->ht_type) < 0) INITERROR; - """, ) - import gc - assert module.get_cnt() == 0 - a = memoryview(module.create_test()) - assert module.get_cnt() == 1 - assert module.get_dealloc_cnt() == 0 - del a - self.debug_collect() - assert module.get_cnt() == 0 - assert module.get_dealloc_cnt() == 1 diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -7,7 +7,7 @@ from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.memoryobject import PyMemoryViewObject -only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" class TestMemoryViewObject(BaseApiTest): def test_fromobject(self, space, api): @@ -35,10 +35,10 @@ assert o == 'hello' ref = api.PyMemoryView_FromBuffer(view) w_mv = from_ref(space, ref) - for f in ('format', 'itemsize', 'ndim', 'readonly', + for f in ('format', 'itemsize', 'ndim', 'readonly', 'shape', 'strides', 'suboffsets'): w_f = space.wrap(f) - assert space.eq_w(space.getattr(w_mv, w_f), + assert space.eq_w(space.getattr(w_mv, w_f), space.getattr(w_memoryview, w_f)) api.Py_DecRef(ref) @@ -137,15 +137,15 @@ shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS']) assert strides[-1] == 8 dt1 = np.dtype( - [('a', 'b'), ('b', 'i'), - ('sub0', np.dtype('b,i')), - ('sub1', np.dtype('b,i')), - ('sub2', np.dtype('b,i')), - ('sub3', np.dtype('b,i')), - ('sub4', np.dtype('b,i')), - ('sub5', np.dtype('b,i')), - ('sub6', np.dtype('b,i')), - ('sub7', np.dtype('b,i')), + [('a', 'b'), ('b', 'i'), + ('sub0', np.dtype('b,i')), + ('sub1', np.dtype('b,i')), + ('sub2', np.dtype('b,i')), + ('sub3', np.dtype('b,i')), + ('sub4', np.dtype('b,i')), + ('sub5', np.dtype('b,i')), + ('sub6', np.dtype('b,i')), + ('sub7', np.dtype('b,i')), ('c', 'i')], ) x = np.arange(dt1.itemsize, dtype='int8').view(dt1) @@ -162,3 +162,75 @@ " on too long format string" finally: warnings.resetwarnings() + + def test_releasebuffer(self): + if not self.runappdirect: + skip("Fails due to ll2ctypes nonsense") + module = self.import_extension('foo', [ + ("create_test", "METH_NOARGS", + """ + PyObject *obj; + obj = PyObject_New(PyObject, (PyTypeObject*)type); + return obj; + """), + ("get_cnt", "METH_NOARGS", + 'return PyLong_FromLong(cnt);'), + ("get_dealloc_cnt", "METH_NOARGS", + 'return PyLong_FromLong(dealloc_cnt);'), + ], + prologue=""" + static float test_data = 42.f; + static int cnt=0; + static int dealloc_cnt=0; + static PyHeapTypeObject * type=NULL; + + void dealloc(PyObject *self) { + dealloc_cnt++; + } + int getbuffer(PyObject *obj, Py_buffer *view, int flags) { + + cnt ++; + memset(view, 0, sizeof(Py_buffer)); + view->obj = obj; + /* see the CPython docs for why we need this incref: + https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer */ + Py_INCREF(obj); + view->ndim = 0; + view->buf = (void *) &test_data; + view->itemsize = sizeof(float); + view->len = 1; + view->strides = NULL; + view->shape = NULL; + view->format = "f"; + return 0; + } + + void releasebuffer(PyObject *obj, Py_buffer *view) { + cnt --; + } + """, more_init=""" + type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + + type->ht_type.tp_name = "Test"; + type->ht_type.tp_basicsize = sizeof(PyObject); + type->ht_name = PyString_FromString("Test"); + type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_NEWBUFFER; + type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; + + type->ht_type.tp_dealloc = dealloc; + type->ht_type.tp_as_buffer = &type->as_buffer; + type->as_buffer.bf_getbuffer = getbuffer; + type->as_buffer.bf_releasebuffer = releasebuffer; + + if (PyType_Ready(&type->ht_type) < 0) INITERROR; + """, ) + import gc + assert module.get_cnt() == 0 + a = memoryview(module.create_test()) + assert module.get_cnt() == 1 + assert module.get_dealloc_cnt() == 0 + del a + self.debug_collect() + assert module.get_cnt() == 0 + assert module.get_dealloc_cnt() == 1 From pypy.commits at gmail.com Thu Mar 2 16:54:42 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 02 Mar 2017 13:54:42 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Add buffer_w based bf_getbuffer implementation to support numpypy Message-ID: <58b894a2.4f5c190a.cb3de.8e62@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90502:ca20597155fa Date: 2017-03-02 22:53 +0100 http://bitbucket.org/pypy/pypy/changeset/ca20597155fa/ Log: Add buffer_w based bf_getbuffer implementation to support numpypy diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -723,31 +723,12 @@ slot_func = slot_tp_new elif name == 'tp_as_buffer.c_bf_getbuffer': buff_fn = w_type.getdictvalue(space, '__buffer__') - if buff_fn is None: + if buff_fn is not None: + buff_w = slot_from___buffer__(space, typedef, buff_fn) + elif typedef.buffer: + buff_w = slot_from_buffer_w(space, typedef, buff_fn) + else: return - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def buff_w(space, w_self, view, flags): - args = Arguments(space, [space.newint(flags)]) - w_obj = space.call_args(space.get(buff_fn, w_self), args) - if view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - view.c_obj = make_ref(space, w_s) - view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, view) - return ret - return 0 slot_func = buff_w else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce @@ -757,6 +738,60 @@ return slot_func + +def slot_from___buffer__(space, typedef, buff_fn): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, view, flags): + args = Arguments(space, [space.newint(flags)]) + w_obj = space.call_args(space.get(buff_fn, w_self), args) + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + return buff_w + +def slot_from_buffer_w(space, typedef, buff_fn): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, view, flags): + w_obj = w_self + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + return buff_w + PyWrapperFlag_KEYWORDS = 1 class TypeSlot: diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -1532,7 +1532,7 @@ return res """, filename=__file__).interphook('ptp') -W_NDimArray.typedef = TypeDef("numpy.ndarray", +W_NDimArray.typedef = TypeDef("numpy.ndarray", None, None, 'read-write', __new__ = interp2app(descr_new_array), __len__ = interp2app(W_NDimArray.descr_len), From pypy.commits at gmail.com Thu Mar 2 18:11:40 2017 From: pypy.commits at gmail.com (wlav) Date: Thu, 02 Mar 2017 15:11:40 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hopefully fix the str/bytes mismatches on py3.5 Message-ID: <58b8a6ac.4aa7190a.3a00e.ac2f@mx.google.com> Author: Wim Lavrijsen Branch: py3.5 Changeset: r90503:6c8f0848fcf2 Date: 2017-03-02 15:00 -0800 http://bitbucket.org/pypy/pypy/changeset/6c8f0848fcf2/ Log: hopefully fix the str/bytes mismatches on py3.5 diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -355,7 +355,7 @@ return rffi.cast(rffi.UCHAR, space.c_uint_w(call_capi(space, 'call_b', args))) def c_call_c(space, cppmethod, cppobject, nargs, cargs): args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)] - return rffi.cast(rffi.CHAR, space.text_w(call_capi(space, 'call_c', args))[0]) + return rffi.cast(rffi.CHAR, space.bytes_w(call_capi(space, 'call_c', args))[0]) def c_call_h(space, cppmethod, cppobject, nargs, cargs): args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)] return rffi.cast(rffi.SHORT, space.int_w(call_capi(space, 'call_h', args))) diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py --- a/pypy/module/cppyy/executor.py +++ b/pypy/module/cppyy/executor.py @@ -132,7 +132,11 @@ if ccpresult == rffi.cast(rffi.CCHARP, 0): return space.newbytes("") result = rffi.charp2str(ccpresult) # TODO: make it a choice to free - return space.newbytes(result) + # debatable whether this should be newtext or newbytes; they are bytes + # but will be more likely used as text after binding ... probably need + # to make this configurable on a per-function bases (same as the old + # char* v.s. byte* problem) + return space.newtext(result) class ConstructorExecutor(FunctionExecutor): diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py --- a/pypy/module/cppyy/ffitypes.py +++ b/pypy/module/cppyy/ffitypes.py @@ -81,13 +81,15 @@ raise oefmt(space.w_ValueError, "char arg not in range(256)") value = rffi.cast(rffi.CHAR, space.c_int_w(w_value)) + elif space.isinstance_w(w_value, space.w_text): + value = space.text_w(w_value) else: value = space.bytes_w(w_value) if len(value) != 1: raise oefmt(space.w_ValueError, "char expected, got string of size %d", len(value)) - return value[0] # turn it into a "char" to the annotator + return rffi.cast(rffi.CHAR, value[0]) # help the annotator def cffi_type(self, space): state = space.fromcache(State) diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py --- a/pypy/module/cppyy/test/test_datatypes.py +++ b/pypy/module/cppyy/test/test_datatypes.py @@ -38,9 +38,9 @@ assert not c.get_bool(); assert not c.get_bool_cr(); assert not c.get_bool_r() # reading char types - assert c.m_char == 'a' - assert c.m_schar == 'b' - assert c.m_uchar == 'c' + assert c.m_char == b'a' + assert c.m_schar == b'b' + assert c.m_uchar == b'c' # reading integer types assert c.m_short == -11; assert c.get_short_cr() == -11; assert c.get_short_r() == -11 @@ -133,18 +133,20 @@ raises(ValueError, 'c.set_bool(10)') # char types through functions - c.set_char('c'); assert c.get_char() == 'c' - c.set_uchar('e'); assert c.get_uchar() == 'e' + c.set_char('c'); assert c.get_char() == b'c' + c.set_char(b'b'); assert c.get_char() == b'b' + c.set_uchar('e'); assert c.get_uchar() == b'e' + c.set_uchar(b'd'); assert c.get_uchar() == b'd' # char types through data members - c.m_char = 'b'; assert c.get_char() == 'b' - c.m_char = 40; assert c.get_char() == chr(40) - c.set_char('c'); assert c.m_char == 'c' - c.set_char(41); assert c.m_char == chr(41) - c.m_uchar = 'd'; assert c.get_uchar() == 'd' - c.m_uchar = 42; assert c.get_uchar() == chr(42) - c.set_uchar('e'); assert c.m_uchar == 'e' - c.set_uchar(43); assert c.m_uchar == chr(43) + c.m_char = 'b'; assert c.get_char() == b'b' + c.m_char = 40; assert c.get_char() == str.encode(chr(40)) + c.set_char('c'); assert c.m_char == b'c' + c.set_char(41); assert c.m_char == str.encode(chr(41)) + c.m_uchar = 'd'; assert c.get_uchar() == b'd' + c.m_uchar = 42; assert c.get_uchar() == str.encode(chr(42)) + c.set_uchar('e'); assert c.m_uchar == b'e' + c.set_uchar(43); assert c.m_uchar == str.encode(chr(43)) raises(ValueError, 'c.set_char("string")') raises(ValueError, 'c.set_char(500)') @@ -246,10 +248,10 @@ assert isinstance(c, CppyyTestData) # char types - assert CppyyTestData.s_char == 'c' - assert c.s_char == 'c' - assert c.s_uchar == 'u' - assert CppyyTestData.s_uchar == 'u' + assert CppyyTestData.s_char == b'c' + assert c.s_char == b'c' + assert c.s_uchar == b'u' + assert CppyyTestData.s_uchar == b'u' # integer types assert CppyyTestData.s_short == -101 @@ -259,15 +261,15 @@ assert CppyyTestData.s_int == -202 assert c.s_int == -202 assert c.s_uint == 202 - assert CppyyTestData.s_uint == 202 - assert CppyyTestData.s_long == -303 + assert CppyyTestData.s_uint == 202 + assert CppyyTestData.s_long == -303 assert c.s_long == -303 assert c.s_ulong == 303 - assert CppyyTestData.s_ulong == 303 - assert CppyyTestData.s_llong == -404 + assert CppyyTestData.s_ulong == 303 + assert CppyyTestData.s_llong == -404 assert c.s_llong == -404 - assert c.s_ullong == 505 - assert CppyyTestData.s_ullong == 505 + assert c.s_ullong == 404 + assert CppyyTestData.s_ullong == 404 # floating point types assert round(CppyyTestData.s_float + 606., 5) == 0 @@ -287,57 +289,57 @@ assert isinstance(c, CppyyTestData) # char types - CppyyTestData.s_char = 'a' - assert c.s_char == 'a' - c.s_char = 'b' - assert CppyyTestData.s_char == 'b' - CppyyTestData.s_uchar = 'c' - assert c.s_uchar == 'c' - c.s_uchar = 'd' - assert CppyyTestData.s_uchar == 'd' + CppyyTestData.s_char = 'a' + assert c.s_char == b'a' + c.s_char = 'b' + assert CppyyTestData.s_char == b'b' + CppyyTestData.s_uchar = 'c' + assert c.s_uchar == b'c' + c.s_uchar = 'd' + assert CppyyTestData.s_uchar == b'd' raises(ValueError, setattr, CppyyTestData, 's_uchar', -1) - raises(ValueError, setattr, c, 's_uchar', -1) + raises(ValueError, setattr, c, 's_uchar', -1) # integer types c.s_short = -102 - assert CppyyTestData.s_short == -102 - CppyyTestData.s_short = -203 + assert CppyyTestData.s_short == -102 + CppyyTestData.s_short = -203 assert c.s_short == -203 c.s_ushort = 127 - assert CppyyTestData.s_ushort == 127 - CppyyTestData.s_ushort = 227 + assert CppyyTestData.s_ushort == 127 + CppyyTestData.s_ushort = 227 assert c.s_ushort == 227 - CppyyTestData.s_int = -234 + CppyyTestData.s_int = -234 assert c.s_int == -234 c.s_int = -321 - assert CppyyTestData.s_int == -321 - CppyyTestData.s_uint = 1234 + assert CppyyTestData.s_int == -321 + CppyyTestData.s_uint = 1234 assert c.s_uint == 1234 c.s_uint = 4321 - assert CppyyTestData.s_uint == 4321 + assert CppyyTestData.s_uint == 4321 raises(ValueError, setattr, c, 's_uint', -1) raises(ValueError, setattr, CppyyTestData, 's_uint', -1) - CppyyTestData.s_long = -87 + CppyyTestData.s_long = -87 assert c.s_long == -87 c.s_long = 876 - assert CppyyTestData.s_long == 876 - CppyyTestData.s_ulong = 876 + assert CppyyTestData.s_long == 876 + CppyyTestData.s_ulong = 876 assert c.s_ulong == 876 c.s_ulong = 678 - assert CppyyTestData.s_ulong == 678 + assert CppyyTestData.s_ulong == 678 raises(ValueError, setattr, CppyyTestData, 's_ulong', -1) - raises(ValueError, setattr, c, 's_ulong', -1) + raises(ValueError, setattr, c, 's_ulong', -1) # floating point types CppyyTestData.s_float = -3.1415 - assert round(c.s_float, 5 ) == -3.1415 - c.s_float = 3.1415 + assert round(c.s_float, 5 ) == -3.1415 + c.s_float = 3.1415 assert round(CppyyTestData.s_float, 5 ) == 3.1415 import math - c.s_double = -math.pi + c.s_double = -math.pi assert CppyyTestData.s_double == -math.pi CppyyTestData.s_double = math.pi - assert c.s_double == math.pi + assert c.s_double == math.pi c.destruct() From pypy.commits at gmail.com Thu Mar 2 19:47:56 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 02 Mar 2017 16:47:56 -0800 (PST) Subject: [pypy-commit] pypy gcc-lto: A branch to test the performance implications of GCC's LTO support. Message-ID: <58b8bd3c.5ed7190a.9b22.9aba@mx.google.com> Author: Manuel Jacob Branch: gcc-lto Changeset: r90504:381e11f5f483 Date: 2017-03-03 01:47 +0100 http://bitbucket.org/pypy/pypy/changeset/381e11f5f483/ Log: A branch to test the performance implications of GCC's LTO support. Start by passing -flto to the compiler on Linux. diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py --- a/rpython/translator/platform/linux.py +++ b/rpython/translator/platform/linux.py @@ -13,7 +13,7 @@ + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) cflags = tuple( - ['-O3', '-pthread', '-fomit-frame-pointer', + ['-O3', '-pthread', '-fomit-frame-pointer', '-flto', '-Wall', '-Wno-unused'] + os.environ.get('CFLAGS', '').split()) standalone_only = () From pypy.commits at gmail.com Fri Mar 3 05:53:27 2017 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Mar 2017 02:53:27 -0800 (PST) Subject: [pypy-commit] pypy fix-cpyext-releasebuffer: Close branch fix-cpyext-releasebuffer Message-ID: <58b94b27.1a4b2e0a.95e02.c9b0@mx.google.com> Author: Ronan Lamy Branch: fix-cpyext-releasebuffer Changeset: r90505:7c40ce697921 Date: 2017-03-03 11:53 +0100 http://bitbucket.org/pypy/pypy/changeset/7c40ce697921/ Log: Close branch fix-cpyext-releasebuffer From pypy.commits at gmail.com Fri Mar 3 05:53:46 2017 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Mar 2017 02:53:46 -0800 (PST) Subject: [pypy-commit] pypy default: Merged fix-cpyext-releasebuffer into default Message-ID: <58b94b3a.04052e0a.4aba2.c4e4@mx.google.com> Author: Ronan Lamy Branch: Changeset: r90506:37e26597bd07 Date: 2017-03-03 11:53 +0100 http://bitbucket.org/pypy/pypy/changeset/37e26597bd07/ Log: Merged fix-cpyext-releasebuffer into default diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -2,7 +2,7 @@ from pypy.interpreter.error import oefmt from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts, Py_buffer, - Py_ssize_t, Py_ssize_tP, + Py_ssize_t, Py_ssize_tP, generic_cpy_call, PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES) from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef @@ -13,10 +13,6 @@ flags = pyobj.c_ob_type.c_tp_flags if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer): return 1 - name = rffi.charp2str(cts.cast('char*', pyobj.c_ob_type.c_tp_name)) - if name in ('str', 'bytes'): - # XXX remove once wrapper of __buffer__ -> bf_getbuffer works - return 1 return 0 @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t, @@ -60,6 +56,15 @@ Release the buffer view. This should be called when the buffer is no longer being used as it may free memory from it """ - Py_DecRef(space, view.c_obj) + obj = view.c_obj + if not obj: + return + assert obj.c_ob_type + as_buffer = obj.c_ob_type.c_tp_as_buffer + if as_buffer: + func = as_buffer.c_bf_releasebuffer + if func: + generic_cpy_call(space, func, obj, view) + Py_DecRef(space, obj) view.c_obj = lltype.nullptr(PyObject.TO) # XXX do other fields leak memory? diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -1,15 +1,18 @@ +from rpython.rlib.objectmodel import keepalive_until_here +from pypy.interpreter.error import oefmt from pypy.module.cpyext.api import ( cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct, - bootstrap_function, Py_bufferP, slot_function) + bootstrap_function, Py_bufferP, slot_function, generic_cpy_call) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr, + PyObject, make_ref, as_pyobj, decref, from_ref, make_typedescr, get_typedescr, track_reference) from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import widen from pypy.objspace.std.memoryobject import W_MemoryView from pypy.module.cpyext.object import _dealloc from pypy.module.cpyext.import_ import PyImport_Import +from pypy.module.cpyext.buffer import PyObject_CheckBuffer PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView") @@ -114,18 +117,13 @@ raise an error if the object can't support a simpler view of its memory. 0 is returned on success and -1 on error.""" - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - except ValueError: - if not space.isinstance_w(w_obj, space.w_bytes): - # XXX Python 3? - raise BufferError("could not create buffer from object") - view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.bytes_w(w_obj), track_allocation=False)) - rffi.setintfield(view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, view) - view.c_obj = make_ref(space, w_obj) + if not PyObject_CheckBuffer(space, w_obj): + raise oefmt(space.w_TypeError, + "'%T' does not have the buffer interface", w_obj) + py_obj = as_pyobj(space, w_obj) + func = py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer + ret = generic_cpy_call(space, func, py_obj, view, flags) + keepalive_until_here(w_obj) return ret def fill_Py_buffer(space, buf, view): @@ -259,4 +257,3 @@ py_mem.c_view.c_shape = view.c_shape # XXX ignore suboffsets? return py_obj - diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -7,14 +7,14 @@ from rpython.rlib import rgc # Force registration of gc.collect from pypy.module.cpyext.api import ( slot_function, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES, - pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr) + pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr, cts) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, ternaryfunc, binaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, readbufferproc, getbufferproc, releasebufferproc, ssizessizeobjargproc) -from pypy.module.cpyext.pyobject import make_ref, decref +from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.memoryobject import fill_Py_buffer from pypy.module.cpyext.state import State @@ -327,13 +327,14 @@ _immutable_ = True def __init__(self, space, ptr, size, w_obj, format='B', shape=None, - strides=None, ndim=1, itemsize=1, readonly=True, - releasebufferproc=rffi.cast(rffi.VOIDP, 0)): + strides=None, ndim=1, itemsize=1, readonly=True, + needs_decref=False, + releasebufferproc=rffi.cast(rffi.VOIDP, 0)): self.space = space self.ptr = ptr self.size = size self.w_obj = w_obj # kept alive - self.pyobj = make_ref(space, w_obj) + self.pyobj = as_pyobj(space, w_obj) self.format = format if not shape: self.shape = [size] @@ -346,30 +347,33 @@ self.ndim = ndim self.itemsize = itemsize self.readonly = readonly + self.needs_decref = needs_decref self.releasebufferproc = releasebufferproc def releasebuffer(self): if self.pyobj: - decref(self.space, self.pyobj) + if self.needs_decref: + if self.releasebufferproc: + func_target = rffi.cast(releasebufferproc, self.releasebufferproc) + with lltype.scoped_alloc(Py_buffer) as pybuf: + pybuf.c_buf = self.ptr + pybuf.c_len = self.size + pybuf.c_ndim = cts.cast('int', self.ndim) + pybuf.c_shape = cts.cast('Py_ssize_t*', pybuf.c__shape) + pybuf.c_strides = cts.cast('Py_ssize_t*', pybuf.c__strides) + for i in range(self.ndim): + pybuf.c_shape[i] = self.shape[i] + pybuf.c_strides[i] = self.strides[i] + if self.format: + pybuf.c_format = rffi.str2charp(self.format) + else: + pybuf.c_format = rffi.str2charp("B") + generic_cpy_call(self.space, func_target, self.pyobj, pybuf) + decref(self.space, self.pyobj) self.pyobj = lltype.nullptr(PyObject.TO) else: #do not call twice return - if self.releasebufferproc: - func_target = rffi.cast(releasebufferproc, self.releasebufferproc) - with lltype.scoped_alloc(Py_buffer) as pybuf: - pybuf.c_buf = self.ptr - pybuf.c_len = self.size - pybuf.c_ndim = rffi.cast(rffi.INT_real, self.ndim) - for i in range(self.ndim): - pybuf.c_shape[i] = self.shape[i] - pybuf.c_strides[i] = self.strides[i] - if self.format: - pybuf.c_format = rffi.str2charp(self.format) - else: - pybuf.c_format = rffi.str2charp("B") - generic_cpy_call(self.space, func_target, self.pyobj, pybuf) - self.releasebufferproc = rffi.cast(rffi.VOIDP, 0) def getlength(self): return self.size @@ -465,19 +469,25 @@ ptr = pybuf.c_buf size = pybuf.c_len ndim = widen(pybuf.c_ndim) - shape = [pybuf.c_shape[i] for i in range(ndim)] + shape = None + if pybuf.c_shape: + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = None if pybuf.c_strides: strides = [pybuf.c_strides[i] for i in range(ndim)] - else: - strides = [1] if pybuf.c_format: format = rffi.charp2str(pybuf.c_format) else: format = 'B' + # the CPython docs mandates that you do an incref whenever you call + # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't + # leak we release the buffer: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer buf = CPyBuffer(space, ptr, size, w_self, format=format, ndim=ndim, shape=shape, strides=strides, itemsize=pybuf.c_itemsize, readonly=widen(pybuf.c_readonly), + needs_decref=True, releasebufferproc = rbp) fq.register_finalizer(buf) return space.newbuffer(buf) @@ -713,33 +723,12 @@ slot_func = slot_tp_new elif name == 'tp_as_buffer.c_bf_getbuffer': buff_fn = w_type.getdictvalue(space, '__buffer__') - if buff_fn is None: + if buff_fn is not None: + buff_w = slot_from___buffer__(space, typedef, buff_fn) + elif typedef.buffer: + buff_w = slot_from_buffer_w(space, typedef, buff_fn) + else: return - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def buff_w(space, w_self, view, flags): - args = Arguments(space, [space.newint(flags)]) - w_obj = space.call_args(space.get(buff_fn, w_self), args) - if view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - view.c_obj = make_ref(space, w_s) - view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, view) - return ret - return 0 - # XXX remove this when it no longer crashes a translated PyPy - return slot_func = buff_w else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce @@ -749,6 +738,60 @@ return slot_func + +def slot_from___buffer__(space, typedef, buff_fn): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, view, flags): + args = Arguments(space, [space.newint(flags)]) + w_obj = space.call_args(space.get(buff_fn, w_self), args) + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + return buff_w + +def slot_from_buffer_w(space, typedef, buff_fn): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, view, flags): + w_obj = w_self + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + return buff_w + PyWrapperFlag_KEYWORDS = 1 class TypeSlot: diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -1893,10 +1893,10 @@ } else if(obj1->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); else if(obj2->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -1947,10 +1947,10 @@ } else if(obj1->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); else if(obj2->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -2006,6 +2006,29 @@ return 1; } +static int +array_getbuffer(PyObject* obj, Py_buffer* view, int flags) +{ + arrayobject* self = (arrayobject*)obj; + return PyBuffer_FillInfo(view, obj, self->ob_item, + Py_SIZE(self)*self->ob_descr->itemsize, 0, flags); +} + +static long releasebuffer_cnt = 0; + +static PyObject * +get_releasebuffer_cnt(void) +{ + return PyLong_FromLong(releasebuffer_cnt); +} + +static void +array_releasebuffer(arrayobject* self, Py_buffer* view) +{ + releasebuffer_cnt++; + return; +} + static PySequenceMethods array_as_sequence = { (lenfunc)array_length, /*sq_length*/ (binaryfunc)array_concat, /*sq_concat*/ @@ -2024,6 +2047,8 @@ (writebufferproc)array_buffer_getwritebuf, (segcountproc)array_buffer_getsegcount, NULL, + (getbufferproc)array_getbuffer, + (releasebufferproc)array_releasebuffer }; static PyObject * @@ -2237,7 +2262,7 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &array_as_buffer, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ arraytype_doc, /* tp_doc */ 0, /* tp_traverse */ @@ -2280,8 +2305,9 @@ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &array_as_buffer, /* tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_CHECKTYPES | + Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */ arraytype_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ @@ -2409,6 +2435,16 @@ return PyString_FromStringAndSize((char*)ptr, size); } +static PyObject * +create_and_release_buffer(PyObject *self, PyObject *obj) +{ + Py_buffer view; + int res = PyObject_GetBuffer(obj, &view, 0); + if (res < 0) + return NULL; + PyBuffer_Release(&view); + Py_RETURN_NONE; +} /*********************** Install Module **************************/ @@ -2417,6 +2453,8 @@ {"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL}, {"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL}, {"readbuffer_as_string", (PyCFunction)readbuffer_as_string, METH_VARARGS, NULL}, + {"get_releasebuffer_cnt", (PyCFunction)get_releasebuffer_cnt, METH_NOARGS, NULL}, + {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -78,6 +78,23 @@ '\x03\0\0\0' '\x04\0\0\0') + def test_releasebuffer(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert module.get_releasebuffer_cnt() == 0 + module.create_and_release_buffer(arr) + assert module.get_releasebuffer_cnt() == 1 + + def test_Py_buffer(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert module.get_releasebuffer_cnt() == 0 + m = memoryview(arr) + assert module.get_releasebuffer_cnt() == 0 + del m + self.debug_collect() + assert module.get_releasebuffer_cnt() == 1 + def test_pickle(self): import pickle module = self.import_module(name='array') @@ -107,7 +124,7 @@ arr = Sub('i', [2]) res = [1, 2, 3] * arr assert res == [1, 2, 3, 1, 2, 3] - + val = module.readbuffer_as_string(arr) assert val == struct.pack('i', 2) diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -63,59 +63,3 @@ a = array.array('c', 'text') b = buffer(a) assert module.roundtrip(b) == 'text' - - def test_releasebuffer(self): - module = self.import_extension('foo', [ - ("create_test", "METH_NOARGS", - """ - PyObject *obj; - obj = PyObject_New(PyObject, (PyTypeObject*)type); - return obj; - """), - ("get_cnt", "METH_NOARGS", - 'return PyLong_FromLong(cnt);')], prologue=""" - static float test_data = 42.f; - static int cnt=0; - static PyHeapTypeObject * type=NULL; - - int getbuffer(PyObject *obj, Py_buffer *view, int flags) { - - cnt ++; - memset(view, 0, sizeof(Py_buffer)); - view->obj = obj; - view->ndim = 0; - view->buf = (void *) &test_data; - view->itemsize = sizeof(float); - view->len = 1; - view->strides = NULL; - view->shape = NULL; - view->format = "f"; - return 0; - } - - void releasebuffer(PyObject *obj, Py_buffer *view) { - cnt --; - } - """, more_init=""" - type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); - - type->ht_type.tp_name = "Test"; - type->ht_type.tp_basicsize = sizeof(PyObject); - type->ht_name = PyString_FromString("Test"); - type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_NEWBUFFER; - type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; - - type->ht_type.tp_as_buffer = &type->as_buffer; - type->as_buffer.bf_getbuffer = getbuffer; - type->as_buffer.bf_releasebuffer = releasebuffer; - - if (PyType_Ready(&type->ht_type) < 0) INITERROR; - """, ) - import gc - assert module.get_cnt() == 0 - a = memoryview(module.create_test()) - assert module.get_cnt() == 1 - del a - gc.collect(); gc.collect(); gc.collect() - assert module.get_cnt() == 0 diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -7,7 +7,7 @@ from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.memoryobject import PyMemoryViewObject -only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" class TestMemoryViewObject(BaseApiTest): def test_fromobject(self, space, api): @@ -35,10 +35,10 @@ assert o == 'hello' ref = api.PyMemoryView_FromBuffer(view) w_mv = from_ref(space, ref) - for f in ('format', 'itemsize', 'ndim', 'readonly', + for f in ('format', 'itemsize', 'ndim', 'readonly', 'shape', 'strides', 'suboffsets'): w_f = space.wrap(f) - assert space.eq_w(space.getattr(w_mv, w_f), + assert space.eq_w(space.getattr(w_mv, w_f), space.getattr(w_memoryview, w_f)) api.Py_DecRef(ref) @@ -137,15 +137,15 @@ shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS']) assert strides[-1] == 8 dt1 = np.dtype( - [('a', 'b'), ('b', 'i'), - ('sub0', np.dtype('b,i')), - ('sub1', np.dtype('b,i')), - ('sub2', np.dtype('b,i')), - ('sub3', np.dtype('b,i')), - ('sub4', np.dtype('b,i')), - ('sub5', np.dtype('b,i')), - ('sub6', np.dtype('b,i')), - ('sub7', np.dtype('b,i')), + [('a', 'b'), ('b', 'i'), + ('sub0', np.dtype('b,i')), + ('sub1', np.dtype('b,i')), + ('sub2', np.dtype('b,i')), + ('sub3', np.dtype('b,i')), + ('sub4', np.dtype('b,i')), + ('sub5', np.dtype('b,i')), + ('sub6', np.dtype('b,i')), + ('sub7', np.dtype('b,i')), ('c', 'i')], ) x = np.arange(dt1.itemsize, dtype='int8').view(dt1) @@ -162,3 +162,75 @@ " on too long format string" finally: warnings.resetwarnings() + + def test_releasebuffer(self): + if not self.runappdirect: + skip("Fails due to ll2ctypes nonsense") + module = self.import_extension('foo', [ + ("create_test", "METH_NOARGS", + """ + PyObject *obj; + obj = PyObject_New(PyObject, (PyTypeObject*)type); + return obj; + """), + ("get_cnt", "METH_NOARGS", + 'return PyLong_FromLong(cnt);'), + ("get_dealloc_cnt", "METH_NOARGS", + 'return PyLong_FromLong(dealloc_cnt);'), + ], + prologue=""" + static float test_data = 42.f; + static int cnt=0; + static int dealloc_cnt=0; + static PyHeapTypeObject * type=NULL; + + void dealloc(PyObject *self) { + dealloc_cnt++; + } + int getbuffer(PyObject *obj, Py_buffer *view, int flags) { + + cnt ++; + memset(view, 0, sizeof(Py_buffer)); + view->obj = obj; + /* see the CPython docs for why we need this incref: + https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer */ + Py_INCREF(obj); + view->ndim = 0; + view->buf = (void *) &test_data; + view->itemsize = sizeof(float); + view->len = 1; + view->strides = NULL; + view->shape = NULL; + view->format = "f"; + return 0; + } + + void releasebuffer(PyObject *obj, Py_buffer *view) { + cnt --; + } + """, more_init=""" + type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + + type->ht_type.tp_name = "Test"; + type->ht_type.tp_basicsize = sizeof(PyObject); + type->ht_name = PyString_FromString("Test"); + type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_NEWBUFFER; + type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; + + type->ht_type.tp_dealloc = dealloc; + type->ht_type.tp_as_buffer = &type->as_buffer; + type->as_buffer.bf_getbuffer = getbuffer; + type->as_buffer.bf_releasebuffer = releasebuffer; + + if (PyType_Ready(&type->ht_type) < 0) INITERROR; + """, ) + import gc + assert module.get_cnt() == 0 + a = memoryview(module.create_test()) + assert module.get_cnt() == 1 + assert module.get_dealloc_cnt() == 0 + del a + self.debug_collect() + assert module.get_cnt() == 0 + assert module.get_dealloc_cnt() == 1 diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -1532,7 +1532,7 @@ return res """, filename=__file__).interphook('ptp') -W_NDimArray.typedef = TypeDef("numpy.ndarray", +W_NDimArray.typedef = TypeDef("numpy.ndarray", None, None, 'read-write', __new__ = interp2app(descr_new_array), __len__ = interp2app(W_NDimArray.descr_len), From pypy.commits at gmail.com Fri Mar 3 06:01:35 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Fri, 03 Mar 2017 03:01:35 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Introduce general data structure for timing event workloads Message-ID: <58b94d0f.8173190a.ad43f.128f@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2018:7c3284e554f1 Date: 2017-03-01 21:56 +0100 http://bitbucket.org/pypy/stmgc/changeset/7c3284e554f1/ Log: Introduce general data structure for timing event workloads diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -574,6 +574,13 @@ STM_GC_MAJOR_START, STM_GC_MAJOR_DONE, + /* execution duration profiling events */ + STM_DURATION_WRITE_BARRIER, + STM_DURATION_VALIDATION, + STM_DURATION_COMMIT, + STM_DURATION_MINOR_GC, + STM_DURATION_MAJOR_GC, + _STM_EVENT_N }; @@ -596,6 +603,22 @@ uintptr_t odd_number; /* marker odd number, or 0 if marker is missing */ object_t *object; /* marker object, or NULL if marker is missing */ } stm_loc_marker_t; +/* Allow any kind of payload to be attached to a timing event. */ +enum stm_payload_type_e { + STM_EVENT_PAYLOAD_MARKER, + STM_EVENT_PAYLOAD_DURATION, + + _STM_EVENT_PAYLOAD_N +}; +typedef union { + stm_loc_marker_t *loc_marker; + uint32_t duration; +} stm_timing_event_payload_data_t; +/* Wrapper for payload holding data type and data. */ +typedef struct { + enum stm_payload_type_e payload_type; + stm_timing_event_payload_data_t payload_data; +} stm_timing_event_payload_t; extern void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ enum stm_event_e event, stm_loc_marker_t *marker); From pypy.commits at gmail.com Fri Mar 3 06:01:40 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Fri, 03 Mar 2017 03:01:40 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Adapt stmgc API in support.py to timing event function interface change Message-ID: <58b94d14.0d50190a.d3d5c.c786@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2020:58cddde10089 Date: 2017-03-03 11:42 +0100 http://bitbucket.org/pypy/stmgc/changeset/58cddde10089/ Log: Adapt stmgc API in support.py to timing event function interface change diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -173,10 +173,23 @@ uintptr_t odd_number; object_t *object; } stm_loc_marker_t; - +enum stm_payload_type_e { + STM_EVENT_PAYLOAD_MARKER, + STM_EVENT_PAYLOAD_DURATION, + ... +}; +typedef union { + stm_loc_marker_t *loc_marker; + uint32_t duration; +} stm_timing_event_payload_data_t; +/* Wrapper for payload holding data type and data. */ +typedef struct { + enum stm_payload_type_e type; + stm_timing_event_payload_data_t data; +} stm_timing_event_payload_t; typedef void (*stmcb_timing_event_fn)(stm_thread_local_t *tl, enum stm_event_e event, - stm_loc_marker_t *markers); + stm_timing_event_payload_t *payload); stmcb_timing_event_fn stmcb_timing_event; typedef int (*stm_expand_marker_fn)(char *seg_base, stm_loc_marker_t *marker, @@ -224,7 +237,7 @@ object_t *nvalue, stm_thread_local_t *tl); bool _check_hashtable_write_entry(object_t *, stm_hashtable_entry_t *, object_t *nvalue); - + stm_hashtable_entry_t *stm_hashtable_lookup(object_t *hashtableobj, stm_hashtable_t *hashtable, uintptr_t index); From pypy.commits at gmail.com Fri Mar 3 06:01:42 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Fri, 03 Mar 2017 03:01:42 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Fix marker tests Message-ID: <58b94d16.c35c2e0a.b3faa.c0ab@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2021:32769a7365e2 Date: 2017-03-03 11:53 +0100 http://bitbucket.org/pypy/stmgc/changeset/32769a7365e2/ Log: Fix marker tests diff --git a/c8/test/test_marker.py b/c8/test/test_marker.py --- a/c8/test/test_marker.py +++ b/c8/test/test_marker.py @@ -7,10 +7,11 @@ def recording(self, *kinds): seen = [] @ffi.callback("stmcb_timing_event_fn") - def timing_event(tl, event, marker): + def timing_event(tl, event, payload): if len(kinds) > 0 and event not in kinds: return - if marker: + if payload and payload.type == lib.STM_EVENT_PAYLOAD_MARKER: + marker = payload.data.loc_marker expanded = (marker.odd_number, marker.object) else: expanded = None From pypy.commits at gmail.com Fri Mar 3 06:01:38 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Fri, 03 Mar 2017 03:01:38 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Refactor timing event function to accept general payloads Message-ID: <58b94d12.d4d9190a.ce329.1134@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2019:87f8a688298c Date: 2017-03-02 16:44 +0100 http://bitbucket.org/pypy/stmgc/changeset/87f8a688298c/ Log: Refactor timing event function to accept general payloads diff --git a/c8/stm/marker.c b/c8/stm/marker.c --- a/c8/stm/marker.c +++ b/c8/stm/marker.c @@ -3,6 +3,9 @@ # include "core.h" // silence flymake #endif +#define payload(marker) stm_timing_event_payload_data_t data = { &marker }; \ + stm_timing_event_payload_t payload = { \ + STM_EVENT_PAYLOAD_MARKER, data }; static bool marker_fetch(stm_thread_local_t *tl, stm_loc_marker_t *out_marker) { @@ -92,19 +95,16 @@ stm_loc_marker_t marker; marker_fetch_obj_write(start, contention, &marker); + payload(marker) stmcb_timing_event(STM_SEGMENT->running_thread, - STM_CONTENTION_WRITE_READ, &marker); + STM_CONTENTION_WRITE_READ, &payload); } static void _timing_become_inevitable(void) { stm_loc_marker_t marker; marker_fetch(STM_SEGMENT->running_thread, &marker); + payload(marker) stmcb_timing_event(STM_SEGMENT->running_thread, - STM_BECOME_INEVITABLE, &marker); + STM_BECOME_INEVITABLE, &payload); } - - -void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ - enum stm_event_e event, - stm_loc_marker_t *marker); diff --git a/c8/stm/prof.c b/c8/stm/prof.c --- a/c8/stm/prof.c +++ b/c8/stm/prof.c @@ -6,37 +6,45 @@ static char *profiling_basefn = NULL; static stm_expand_marker_fn profiling_expand_marker; -#define MARKER_LEN_MAX 160 +#define EXTRA_LEN_MAX 160 static bool close_timing_log(void); /* forward */ static void _stm_profiling_event(stm_thread_local_t *tl, enum stm_event_e event, - stm_loc_marker_t *marker) + stm_timing_event_payload_t *payload) { struct buf_s { uint32_t tv_sec; uint32_t tv_nsec; uint32_t thread_num; uint8_t event; - uint8_t marker_length; - char extra[MARKER_LEN_MAX+1]; + uint8_t extra_length; + char extra[EXTRA_LEN_MAX+1]; } __attribute__((packed)); struct buf_s buf; struct timespec t; buf.thread_num = tl->thread_local_counter; buf.event = event; - buf.marker_length = 0; + buf.extra_length = 0; - if (marker != NULL && marker->odd_number != 0) { - buf.marker_length = profiling_expand_marker(get_segment_base(0), - marker, - buf.extra, MARKER_LEN_MAX); + if (payload != NULL) { + if (payload->type == STM_EVENT_PAYLOAD_MARKER) { + stm_loc_marker_t *marker = payload->data.loc_marker; + if (marker != NULL && marker->odd_number != 0) { + buf.extra_length = profiling_expand_marker(get_segment_base(0), + marker, + buf.extra, EXTRA_LEN_MAX); + } + } else if (payload->type == STM_EVENT_PAYLOAD_DURATION) { + uint32_t duration = payload->data.duration; + buf.extra_length = sprintf(buf.extra, "%u", duration); + } } - size_t result, outsize = offsetof(struct buf_s, extra) + buf.marker_length; + size_t result, outsize = offsetof(struct buf_s, extra) + buf.extra_length; FILE *f = profiling_file; if (f == NULL) return; @@ -146,3 +154,7 @@ profiling_basefn = strdup(profiling_file_name); return 0; } + +void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ + enum stm_event_e event, + stm_timing_event_payload_t *payload); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -596,6 +596,7 @@ "gc minor done", \ "gc major start", \ "gc major done" + /* TODO names for new duration events */ /* The markers pushed in the shadowstack are an odd number followed by a regular object pointer. */ @@ -616,12 +617,12 @@ } stm_timing_event_payload_data_t; /* Wrapper for payload holding data type and data. */ typedef struct { - enum stm_payload_type_e payload_type; - stm_timing_event_payload_data_t payload_data; + enum stm_payload_type_e type; + stm_timing_event_payload_data_t data; } stm_timing_event_payload_t; extern void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ enum stm_event_e event, - stm_loc_marker_t *marker); + stm_timing_event_payload_t *payload); /* Calling this sets up a stmcb_timing_event callback that will produce a binary file called 'profiling_file_name'. Call it with @@ -634,6 +635,7 @@ the given position and with the given maximum length. */ typedef int (*stm_expand_marker_fn)(char *seg_base, stm_loc_marker_t *marker, char *output, int output_size); +/* TODO generalize expand function */ int stm_set_timing_log(const char *profiling_file_name, int fork_mode, stm_expand_marker_fn expand_marker); From pypy.commits at gmail.com Fri Mar 3 06:53:54 2017 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Mar 2017 03:53:54 -0800 (PST) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <58b95952.4f5c190a.cb3de.ac2c@mx.google.com> Author: Ronan Lamy Branch: Changeset: r90507:64e48b7abe6f Date: 2017-03-03 12:53 +0100 http://bitbucket.org/pypy/pypy/changeset/64e48b7abe6f/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -163,3 +163,9 @@ .. branch: fix_bool_restype Fix for ``ctypes.c_bool``-returning ctypes functions + +.. branch: fix-cpyext-releasebuffer + +Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews +keeping objects alive forever (missing decref), and make sure that +bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. From pypy.commits at gmail.com Fri Mar 3 08:42:45 2017 From: pypy.commits at gmail.com (Raemi) Date: Fri, 03 Mar 2017 05:42:45 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: change test so we don't spend most of the time doing minor GC Message-ID: <58b972d5.02482e0a.c64b0.ca79@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90508:0dd8add9e63b Date: 2017-03-03 14:42 +0100 http://bitbucket.org/pypy/pypy/changeset/0dd8add9e63b/ Log: change test so we don't spend most of the time doing minor GC before, we were spending most of the time making the huge chains of X-objects old. there was very little time spent by threads actually running in parallel. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -279,8 +279,8 @@ # allocated nursery size is 2 times "tl_block_size". # "cache_line_min" is used to round the actual thread-local # blocks to a cache line, to avoid pointless cache conflicts. - "tl_block_size": 65536, - "cache_line_min": 256, + "tl_block_size": 131072, + "cache_line_min": 256, # why not 64b? } def __init__(self, config, @@ -900,6 +900,11 @@ self.set_nursery_free(self.nursery_barriers.popleft()) self.set_nursery_top(self.nursery_barriers.popleft()) else: + # XXX: if we have more threads than nursery_size/block_size, + # the thread(s) not getting any blocks will immediately request + # a minor collection, even if the blocks of other threads are + # still mostly empty. + rgil.master_request_safepoint() # we are the only thread here; all others are in gc-safepoints diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1442,8 +1442,10 @@ def bootstrap(): rthread.gc_thread_start() x = None - for i in range(100000): + for i in range(10000000): x = X(x) + if i % 10001 == 0: + x = None state.lock.acquire(True) os.write(1, "counter=%d\n" % state.counter) @@ -1458,7 +1460,7 @@ def entry_point(argv): os.write(1, "hello world\n") # start 5 new threads - TS = 100 + TS = int(argv[1]) state.lock = rthread.allocate_lock() state.counter = TS @@ -1467,7 +1469,7 @@ while True: time.sleep(0.1) - gc.collect() + #gc.collect() if state.counter == 0: break os.write(1, "all threads done\n") @@ -1475,7 +1477,7 @@ def runme(no__thread): t, cbuilder = self.compile(entry_point, no__thread=no__thread) - data = cbuilder.cmdexec('') + data = cbuilder.cmdexec('5') assert data.splitlines() == ['hello world', '1 ok', '2 ok', From pypy.commits at gmail.com Fri Mar 3 09:22:33 2017 From: pypy.commits at gmail.com (Raemi) Date: Fri, 03 Mar 2017 06:22:33 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: add colourful, thread-prefixed debug print Message-ID: <58b97c29.18532e0a.37a74.cfb5@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90509:3fe01091f755 Date: 2017-03-03 15:21 +0100 http://bitbucket.org/pypy/pypy/changeset/3fe01091f755/ Log: add colourful, thread-prefixed debug print (imported from stmgc-c8 branch) diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c --- a/rpython/translator/c/src/debug_print.c +++ b/rpython/translator/c/src/debug_print.c @@ -12,15 +12,16 @@ #include #endif #include "common_header.h" +#include "structdef.h" #include "src/profiling.h" #include "src/debug_print.h" -long pypy_have_debug_prints = -1; FILE *pypy_debug_file = NULL; static unsigned char debug_ready = 0; static unsigned char debug_profile = 0; -static char *debug_start_colors_1 = ""; -static char *debug_start_colors_2 = ""; +static __thread char debug_start_colors_1[32]; +static __thread char debug_start_colors_2[28]; +__thread char pypy_debug_threadid[16] = {0}; static char *debug_stop_colors = ""; static char *debug_prefix = NULL; @@ -90,8 +91,6 @@ pypy_debug_file = stderr; if (isatty(2)) { - debug_start_colors_1 = "\033[1m\033[31m"; - debug_start_colors_2 = "\033[31m"; debug_stop_colors = "\033[0m"; } } @@ -210,6 +209,40 @@ debug_stop_colors); } + +#define bool_cas __sync_bool_compare_and_swap +static Signed threadcounter = 0; + +static void _prepare_display_colors(void) +{ + Signed counter; + char *p; + while (1) { + counter = threadcounter; + if (bool_cas(&threadcounter, counter, counter + 1)) + break; + } + if (debug_stop_colors[0] == 0) { + /* not a tty output: no colors */ + sprintf(debug_start_colors_1, "%d# ", (int)counter); + sprintf(debug_start_colors_2, "%d# ", (int)counter); + sprintf(pypy_debug_threadid, "%d#", (int)counter); + } + else { + /* tty output */ + int color = 0; + color = 31 + (int)(counter % 7); + sprintf(debug_start_colors_1, "\033[%dm%d# \033[1m", + color, (int)counter); + sprintf(debug_start_colors_2, "\033[%dm%d# ", + color, (int)counter); +#ifdef RPY_STM + sprintf(pypy_debug_threadid, "\033[%dm%d#\033[0m", + color, (int)counter); +#endif + } +} + void pypy_debug_start(const char *category) { pypy_debug_ensure_opened(); @@ -229,6 +262,8 @@ /* else make this subsection active */ pypy_have_debug_prints |= 1; } + if (!debug_start_colors_1[0]) + _prepare_display_colors(); display_startstop("{", "", category, debug_start_colors_1); } diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -27,6 +27,10 @@ removed from the environment and not passed to subprocesses. */ +RPY_EXTERN __thread struct pypy_ExcData0 pypy_g_ExcData; +#define pypy_have_debug_prints pypy_g_ExcData.ed_have_debug_prints + + /* macros used by the generated code */ #define PYPY_HAVE_DEBUG_PRINTS (pypy_have_debug_prints & 1 ? \ (pypy_debug_ensure_opened(), 1) : 0) @@ -48,7 +52,7 @@ RPY_EXTERN void pypy_debug_forked(long original_offset); RPY_EXTERN long pypy_have_debug_prints_for(const char *category_prefix); -RPY_EXTERN long pypy_have_debug_prints; +RPY_EXTERN __thread char pypy_debug_threadid[]; RPY_EXPORTED FILE *pypy_debug_file; #define OP_LL_READ_TIMESTAMP(val) READ_TIMESTAMP(val) diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1442,9 +1442,9 @@ def bootstrap(): rthread.gc_thread_start() x = None - for i in range(10000000): + for i in range(100000000): x = X(x) - if i % 10001 == 0: + if i % 5001 == 0: x = None state.lock.acquire(True) diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -453,10 +453,12 @@ EXCDATA = lltype.Struct('ExcData', ('exc_type', self.lltype_of_exception_type), ('exc_value', self.lltype_of_exception_value), + ('have_debug_prints', lltype.Signed), hints={'thread_local': True, 'is_excdata': True}) self.EXCDATA = EXCDATA exc_data = lltype.malloc(EXCDATA, immortal=True) + exc_data.have_debug_prints = -1 null_type = lltype.nullptr(self.lltype_of_exception_type.TO) null_value = lltype.nullptr(self.lltype_of_exception_value.TO) From pypy.commits at gmail.com Fri Mar 3 09:47:52 2017 From: pypy.commits at gmail.com (Raemi) Date: Fri, 03 Mar 2017 06:47:52 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: add missing pieces for debug prints Message-ID: <58b98218.0a482e0a.32272.c7a4@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90510:8137acf8c43a Date: 2017-03-03 15:47 +0100 http://bitbucket.org/pypy/pypy/changeset/8137acf8c43a/ Log: add missing pieces for debug prints diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -1,6 +1,6 @@ import sys import time - +import thread from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rarithmetic import is_valid_int @@ -44,10 +44,16 @@ _log = None # patched from tests to be an object of class DebugLog # or compatible +_thread_numbering = {} +def _get_thread_num(): + thid = thread.get_ident() + if thid not in _thread_numbering: + _thread_numbering[thid] = len(_thread_numbering) + return _thread_numbering[thid] + def debug_print(*args): - for arg in args: - print >> sys.stderr, arg, - print >> sys.stderr + msg = " ".join(map(str, args)) + sys.stderr.write("%s# %s\n" % (_get_thread_num(), msg)) if _log is not None: _log.debug_print(*args) @@ -76,15 +82,17 @@ def debug_start(category): c = int(time.clock() * 100) - print >> sys.stderr, '%s[%x] {%s%s' % (_start_colors_1, c, - category, _stop_colors) + sys.stderr.write('%s%s# [%x] {%s%s\n' % (_start_colors_1, + _get_thread_num(), c, + category, _stop_colors)) if _log is not None: _log.debug_start(category) def debug_stop(category): c = int(time.clock() * 100) - print >> sys.stderr, '%s[%x] %s}%s' % (_start_colors_2, c, - category, _stop_colors) + sys.stderr.write('%s%s# [%x] %s}%s\n' % (_start_colors_2, + _get_thread_num(), c, + category, _stop_colors)) if _log is not None: _log.debug_stop(category) @@ -441,7 +449,7 @@ except OSError as e: os.write(2, "Could not start GDB: %s" % ( os.strerror(e.errno))) - os._exit(1) + raise SystemExit else: time.sleep(1) # give the GDB time to attach diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -737,8 +737,8 @@ def OP_DEBUG_PRINT(self, op): # XXX from rpython.rtyper.lltypesystem.rstr import STR - format = [] - argv = [] + format = ['%s'] + argv = ['pypy_debug_threadid'] free_line = "" for arg in op.args: T = arg.concretetype diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c --- a/rpython/translator/c/src/debug_print.c +++ b/rpython/translator/c/src/debug_print.c @@ -236,10 +236,9 @@ color, (int)counter); sprintf(debug_start_colors_2, "\033[%dm%d# ", color, (int)counter); -#ifdef RPY_STM + sprintf(pypy_debug_threadid, "\033[%dm%d#\033[0m", color, (int)counter); -#endif } } From pypy.commits at gmail.com Fri Mar 3 10:37:09 2017 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Mar 2017 07:37:09 -0800 (PST) Subject: [pypy-commit] pypy default: Reimplement PyObject_CheckBuffer, PyObject_GetBuffer and PyBuffer_Release in C Message-ID: <58b98da5.4aa7190a.3a00e.d1ee@mx.google.com> Author: Ronan Lamy Branch: Changeset: r90511:d84bd32a26e4 Date: 2017-03-03 16:36 +0100 http://bitbucket.org/pypy/pypy/changeset/d84bd32a26e4/ Log: Reimplement PyObject_CheckBuffer, PyObject_GetBuffer and PyBuffer_Release in C diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -574,6 +574,7 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', + 'PyObject_GetBuffer', 'PyBuffer_Release', 'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', 'PyBuffer_FromObject', 'PyBuffer_FromReadWriteObject', 'PyBuffer_New', 'PyBuffer_Type', '_Py_get_buffer_type', diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -1,19 +1,10 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.interpreter.error import oefmt from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts, Py_buffer, + cpython_api, CANNOT_FAIL, cts, Py_buffer, Py_ssize_t, Py_ssize_tP, generic_cpy_call, PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES) -from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyObject_CheckBuffer(space, pyobj): - """Return 1 if obj supports the buffer interface otherwise 0.""" - as_buffer = pyobj.c_ob_type.c_tp_as_buffer - flags = pyobj.c_ob_type.c_tp_flags - if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer): - return 1 - return 0 +from pypy.module.cpyext.pyobject import PyObject, Py_IncRef @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t, lltype.Signed, lltype.Signed], rffi.INT, error=-1) @@ -48,23 +39,3 @@ view.c_internal = lltype.nullptr(rffi.VOIDP.TO) return 0 - - - at cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL) -def PyBuffer_Release(space, view): - """ - Release the buffer view. This should be called when the buffer is - no longer being used as it may free memory from it - """ - obj = view.c_obj - if not obj: - return - assert obj.c_ob_type - as_buffer = obj.c_ob_type.c_tp_as_buffer - if as_buffer: - func = as_buffer.c_bf_releasebuffer - if func: - generic_cpy_call(space, func, obj, view) - Py_DecRef(space, obj) - view.c_obj = lltype.nullptr(PyObject.TO) - # XXX do other fields leak memory? diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -79,6 +79,7 @@ #include "pyconfig.h" #include "object.h" +#include "abstract.h" #include "pymath.h" #include "pyport.h" #include "warnings.h" diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -1,1 +1,35 @@ -/* empty */ +#ifndef Py_ABSTRACTOBJECT_H +#define Py_ABSTRACTOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + + /* new buffer API */ + +#define PyObject_CheckBuffer(obj) \ + (((obj)->ob_type->tp_as_buffer != NULL) && \ + (PyType_HasFeature((obj)->ob_type, Py_TPFLAGS_HAVE_NEWBUFFER)) && \ + ((obj)->ob_type->tp_as_buffer->bf_getbuffer != NULL)) + + /* Return 1 if the getbuffer function is available, otherwise + return 0 */ + + PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view, + int flags); + + /* This is a C-API version of the getbuffer function call. It checks + to make sure object has the required function pointer and issues the + call. Returns -1 and raises an error on failure and returns 0 on + success + */ + + PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); + + /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. + */ + + +#ifdef __cplusplus +} +#endif +#endif /* Py_ABSTRACTOBJECT_H */ diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -97,35 +97,6 @@ _dealloc(space, py_obj) - at cpython_api([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) -def PyObject_GetBuffer(space, w_obj, view, flags): - """Export obj into a Py_buffer, view. These arguments must - never be NULL. The flags argument is a bit field indicating what - kind of buffer the caller is prepared to deal with and therefore what - kind of buffer the exporter is allowed to return. The buffer interface - allows for complicated memory sharing possibilities, but some caller may - not be able to handle all the complexity but may want to see if the - exporter will let them take a simpler view to its memory. - - Some exporters may not be able to share memory in every possible way and - may need to raise errors to signal to some consumers that something is - just not possible. These errors should be a BufferError unless - there is another error that is actually causing the problem. The - exporter can use flags information to simplify how much of the - Py_buffer structure is filled in with non-default values and/or - raise an error if the object can't support a simpler view of its memory. - - 0 is returned on success and -1 on error.""" - if not PyObject_CheckBuffer(space, w_obj): - raise oefmt(space.w_TypeError, - "'%T' does not have the buffer interface", w_obj) - py_obj = as_pyobj(space, w_obj) - func = py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer - ret = generic_cpy_call(space, func, py_obj, view, flags) - keepalive_until_here(w_obj) - return ret - def fill_Py_buffer(space, buf, view): # c_buf, c_obj have been filled in ndim = buf.getndim() diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -101,6 +101,30 @@ return 0; } +/* Buffer C-API for Python 3.0 */ + +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + +void +PyBuffer_Release(Py_buffer *view) +{ + PyObject *obj = view->obj; + if (obj && Py_TYPE(obj)->tp_as_buffer && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer) + Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer(obj, view); + Py_XDECREF(obj); + view->obj = NULL; +} + /* Operations on callable objects */ static PyObject* diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -10,15 +10,6 @@ only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" class TestMemoryViewObject(BaseApiTest): - def test_fromobject(self, space, api): - w_hello = space.newbytes("hello") - assert api.PyObject_CheckBuffer(w_hello) - w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello)) - w_char = space.call_method(w_view, '__getitem__', space.wrap(0)) - assert space.eq_w(w_char, space.wrap('h')) - w_bytes = space.call_method(w_view, "tobytes") - assert space.unwrap(w_bytes) == "hello" - def test_frombuffer(self, space, api): w_buf = space.newbuffer(StringBuffer("hello")) c_memoryview = rffi.cast( @@ -74,6 +65,19 @@ del result class AppTestBufferProtocol(AppTestCpythonExtensionBase): + def test_fromobject(self): + foo = self.import_extension('foo', [ + ("make_view", "METH_O", + """ + if (!PyObject_CheckBuffer(args)) + return Py_None; + return PyMemoryView_FromObject(args); + """)]) + hello = b'hello' + mview = foo.make_view(hello) + assert mview[0] == hello[0] + assert mview.tobytes() == hello + def test_buffer_protocol_app(self): import struct module = self.import_module(name='buffer_test') From pypy.commits at gmail.com Fri Mar 3 11:23:33 2017 From: pypy.commits at gmail.com (mjacob) Date: Fri, 03 Mar 2017 08:23:33 -0800 (PST) Subject: [pypy-commit] pypy default: Call setup for each test method in this class instead of only once. Fix wrong test that would fail otherwise because the tests weren't independent before. Message-ID: <58b99885.c35c2e0a.b3faa.d12c@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90512:085da6954f2c Date: 2017-03-03 17:21 +0100 http://bitbucket.org/pypy/pypy/changeset/085da6954f2c/ Log: Call setup for each test method in this class instead of only once. Fix wrong test that would fail otherwise because the tests weren't independent before. diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -5,12 +5,12 @@ class AppTestFileIO: spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else [])) - def setup_class(cls): + def setup_method(self, meth): tmpfile = udir.join('tmpfile') tmpfile.write("a\nb\nc", mode='wb') - cls.w_tmpfile = cls.space.wrap(str(tmpfile)) - cls.w_tmpdir = cls.space.wrap(str(udir)) - cls.w_posix = cls.space.appexec([], """(): + self.w_tmpfile = self.space.wrap(str(tmpfile)) + self.w_tmpdir = self.space.wrap(str(udir)) + self.w_posix = self.space.appexec([], """(): import %s as m; return m""" % os.name) @@ -135,7 +135,7 @@ import _io a = bytearray('x' * 10) f = _io.FileIO(self.tmpfile, 'r+') - assert f.readinto(a) == 10 + assert f.readinto(a) == 5 exc = raises(TypeError, f.readinto, u"hello") assert str(exc.value) == "cannot use unicode as modifiable buffer" exc = raises(TypeError, f.readinto, buffer(b"hello")) @@ -145,7 +145,7 @@ exc = raises(TypeError, f.readinto, memoryview(b"hello")) assert str(exc.value) == "must be read-write buffer, not memoryview" f.close() - assert a == 'a\nb\nc\0\0\0\0\0' + assert a == 'a\nb\ncxxxxx' # a = bytearray('x' * 10) f = _io.FileIO(self.tmpfile, 'r+') From pypy.commits at gmail.com Fri Mar 3 11:23:36 2017 From: pypy.commits at gmail.com (mjacob) Date: Fri, 03 Mar 2017 08:23:36 -0800 (PST) Subject: [pypy-commit] pypy default: hg merge Message-ID: <58b99888.c4052e0a.55298.d11d@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90513:53fee63aea78 Date: 2017-03-03 17:22 +0100 http://bitbucket.org/pypy/pypy/changeset/53fee63aea78/ Log: hg merge diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -574,6 +574,7 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', + 'PyObject_GetBuffer', 'PyBuffer_Release', 'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', 'PyBuffer_FromObject', 'PyBuffer_FromReadWriteObject', 'PyBuffer_New', 'PyBuffer_Type', '_Py_get_buffer_type', diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -1,19 +1,10 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.interpreter.error import oefmt from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER, cts, Py_buffer, + cpython_api, CANNOT_FAIL, cts, Py_buffer, Py_ssize_t, Py_ssize_tP, generic_cpy_call, PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES) -from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyObject_CheckBuffer(space, pyobj): - """Return 1 if obj supports the buffer interface otherwise 0.""" - as_buffer = pyobj.c_ob_type.c_tp_as_buffer - flags = pyobj.c_ob_type.c_tp_flags - if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer): - return 1 - return 0 +from pypy.module.cpyext.pyobject import PyObject, Py_IncRef @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t, lltype.Signed, lltype.Signed], rffi.INT, error=-1) @@ -48,23 +39,3 @@ view.c_internal = lltype.nullptr(rffi.VOIDP.TO) return 0 - - - at cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL) -def PyBuffer_Release(space, view): - """ - Release the buffer view. This should be called when the buffer is - no longer being used as it may free memory from it - """ - obj = view.c_obj - if not obj: - return - assert obj.c_ob_type - as_buffer = obj.c_ob_type.c_tp_as_buffer - if as_buffer: - func = as_buffer.c_bf_releasebuffer - if func: - generic_cpy_call(space, func, obj, view) - Py_DecRef(space, obj) - view.c_obj = lltype.nullptr(PyObject.TO) - # XXX do other fields leak memory? diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -79,6 +79,7 @@ #include "pyconfig.h" #include "object.h" +#include "abstract.h" #include "pymath.h" #include "pyport.h" #include "warnings.h" diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -1,1 +1,35 @@ -/* empty */ +#ifndef Py_ABSTRACTOBJECT_H +#define Py_ABSTRACTOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + + /* new buffer API */ + +#define PyObject_CheckBuffer(obj) \ + (((obj)->ob_type->tp_as_buffer != NULL) && \ + (PyType_HasFeature((obj)->ob_type, Py_TPFLAGS_HAVE_NEWBUFFER)) && \ + ((obj)->ob_type->tp_as_buffer->bf_getbuffer != NULL)) + + /* Return 1 if the getbuffer function is available, otherwise + return 0 */ + + PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view, + int flags); + + /* This is a C-API version of the getbuffer function call. It checks + to make sure object has the required function pointer and issues the + call. Returns -1 and raises an error on failure and returns 0 on + success + */ + + PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); + + /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. + */ + + +#ifdef __cplusplus +} +#endif +#endif /* Py_ABSTRACTOBJECT_H */ diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -97,35 +97,6 @@ _dealloc(space, py_obj) - at cpython_api([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) -def PyObject_GetBuffer(space, w_obj, view, flags): - """Export obj into a Py_buffer, view. These arguments must - never be NULL. The flags argument is a bit field indicating what - kind of buffer the caller is prepared to deal with and therefore what - kind of buffer the exporter is allowed to return. The buffer interface - allows for complicated memory sharing possibilities, but some caller may - not be able to handle all the complexity but may want to see if the - exporter will let them take a simpler view to its memory. - - Some exporters may not be able to share memory in every possible way and - may need to raise errors to signal to some consumers that something is - just not possible. These errors should be a BufferError unless - there is another error that is actually causing the problem. The - exporter can use flags information to simplify how much of the - Py_buffer structure is filled in with non-default values and/or - raise an error if the object can't support a simpler view of its memory. - - 0 is returned on success and -1 on error.""" - if not PyObject_CheckBuffer(space, w_obj): - raise oefmt(space.w_TypeError, - "'%T' does not have the buffer interface", w_obj) - py_obj = as_pyobj(space, w_obj) - func = py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer - ret = generic_cpy_call(space, func, py_obj, view, flags) - keepalive_until_here(w_obj) - return ret - def fill_Py_buffer(space, buf, view): # c_buf, c_obj have been filled in ndim = buf.getndim() diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -101,6 +101,30 @@ return 0; } +/* Buffer C-API for Python 3.0 */ + +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + +void +PyBuffer_Release(Py_buffer *view) +{ + PyObject *obj = view->obj; + if (obj && Py_TYPE(obj)->tp_as_buffer && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer) + Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer(obj, view); + Py_XDECREF(obj); + view->obj = NULL; +} + /* Operations on callable objects */ static PyObject* diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -10,15 +10,6 @@ only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" class TestMemoryViewObject(BaseApiTest): - def test_fromobject(self, space, api): - w_hello = space.newbytes("hello") - assert api.PyObject_CheckBuffer(w_hello) - w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello)) - w_char = space.call_method(w_view, '__getitem__', space.wrap(0)) - assert space.eq_w(w_char, space.wrap('h')) - w_bytes = space.call_method(w_view, "tobytes") - assert space.unwrap(w_bytes) == "hello" - def test_frombuffer(self, space, api): w_buf = space.newbuffer(StringBuffer("hello")) c_memoryview = rffi.cast( @@ -74,6 +65,19 @@ del result class AppTestBufferProtocol(AppTestCpythonExtensionBase): + def test_fromobject(self): + foo = self.import_extension('foo', [ + ("make_view", "METH_O", + """ + if (!PyObject_CheckBuffer(args)) + return Py_None; + return PyMemoryView_FromObject(args); + """)]) + hello = b'hello' + mview = foo.make_view(hello) + assert mview[0] == hello[0] + assert mview.tobytes() == hello + def test_buffer_protocol_app(self): import struct module = self.import_module(name='buffer_test') From pypy.commits at gmail.com Fri Mar 3 12:44:13 2017 From: pypy.commits at gmail.com (Raemi) Date: Fri, 03 Mar 2017 09:44:13 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) WIP with failing test Message-ID: <58b9ab6d.d3d3190a.366be.a31f@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90514:9d54936470c4 Date: 2017-03-03 18:43 +0100 http://bitbucket.org/pypy/pypy/changeset/9d54936470c4/ Log: (arigo, remi) WIP with failing test diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -90,6 +90,7 @@ callback(gc, addr) self.rootstackhook = walk_stack_root + from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop def walk_thread_stack(collect_stack_root, tl): # XXX: only visit if nursery_free was not NULL base = (tl + tl_shadowstack._offset).address[0] @@ -100,6 +101,7 @@ # walk that new thread's shadowstack (XXX: compiler may reorder # without barriers) return + debug_print("walk_stack", base, top) self.rootstackhook(collect_stack_root, base, top) self._walk_thread_stack = walk_thread_stack diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -13,7 +13,7 @@ const char *funcname; }; -static struct pypy_debug_alloc_s *pypy_debug_alloc_list = NULL; +static struct pypy_debug_alloc_s *pypy_debug_alloc_list = NULL; static rpy_spinlock_t pypy_debug_alloc_lock = 0; @@ -35,7 +35,7 @@ { struct pypy_debug_alloc_s **p; if (!addr) - return; + return; rpy_spinlock_acquire(&pypy_debug_alloc_lock); for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next)) if ((*p)->addr == addr) diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1432,8 +1432,9 @@ from rpython.rlib import rthread, rposix class X: - def __init__(self, prev): + def __init__(self, prev, i): self.prev = prev + self.i = i class State: pass @@ -1443,7 +1444,13 @@ rthread.gc_thread_start() x = None for i in range(100000000): - x = X(x) + prev_x = x + + x = X(x, i) + + if prev_x is not None: + assert prev_x.i == i - 1 + if i % 5001 == 0: x = None @@ -1467,11 +1474,15 @@ for _ in range(TS): new_thread() + i = 0 while True: + x = X(None, i) time.sleep(0.1) + assert x.i == i #gc.collect() if state.counter == 0: break + i += 1 os.write(1, "all threads done\n") return 0 From pypy.commits at gmail.com Fri Mar 3 12:54:38 2017 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Mar 2017 09:54:38 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58b9adde.a9212e0a.e4b11.dd28@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90515:531a0caf2bb4 Date: 2017-03-03 18:53 +0100 http://bitbucket.org/pypy/pypy/changeset/531a0caf2bb4/ Log: hg merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -176,3 +176,9 @@ .. branch: py3.5-text-utf8 space.text_w now encodes to utf-8 not preserving surrogates. + +.. branch: fix-cpyext-releasebuffer + +Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews +keeping objects alive forever (missing decref), and make sure that +bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -6,12 +6,12 @@ class AppTestFileIO: spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else [])) - def setup_class(cls): + def setup_method(self, meth): tmpfile = udir.join('tmpfile') tmpfile.write("a\nb\nc", mode='wb') - cls.w_tmpfile = cls.space.wrap(str(tmpfile)) - cls.w_tmpdir = cls.space.wrap(str(udir)) - cls.w_posix = cls.space.appexec([], """(): + self.w_tmpfile = self.space.wrap(str(tmpfile)) + self.w_tmpdir = self.space.wrap(str(udir)) + self.w_posix = self.space.appexec([], """(): import %s as m; return m""" % os.name) @@ -154,7 +154,7 @@ f.seek(5) f.write(b'\x00' * 5) f.seek(0) - assert f.readinto(a) == 10 + assert f.readinto(a) == 5 f.seek(0) m = memoryview(bytearray(b"helloworld")) assert f.readinto(m) == 10 @@ -170,7 +170,7 @@ assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() - assert a == b'a\nb\nc\0\0\0\0\0' + assert a == b'a\nb\ncxxxxx' # a = bytearray(b'x' * 10) f = _io.FileIO(self.tmpfile, 'r+') diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -579,7 +579,7 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', - 'PyBuffer_Release', + 'PyObject_GetBuffer', 'PyBuffer_Release', 'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr', 'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr', diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -1,5 +1,8 @@ #ifndef Py_ABSTRACTOBJECT_H #define Py_ABSTRACTOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif /* new buffer API */ @@ -10,9 +13,22 @@ /* Return 1 if the getbuffer function is available, otherwise return 0 */ + PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view, + int flags); + + /* This is a C-API version of the getbuffer function call. It checks + to make sure object has the required function pointer and issues the + call. Returns -1 and raises an error on failure and returns 0 on + success + */ + PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. */ + +#ifdef __cplusplus +} +#endif #endif /* Py_ABSTRACTOBJECT_H */ diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -1,9 +1,11 @@ +from rpython.rlib.objectmodel import keepalive_until_here +from pypy.interpreter.error import oefmt from pypy.module.cpyext.api import ( cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct, - bootstrap_function, Py_bufferP, generic_cpy_call, slot_function) + bootstrap_function, Py_bufferP, slot_function, generic_cpy_call) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr, + PyObject, make_ref, as_pyobj, decref, from_ref, make_typedescr, get_typedescr, track_reference) from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import widen @@ -94,30 +96,6 @@ mem_obj.c_view.c_obj = rffi.cast(PyObject, 0) _dealloc(space, py_obj) - at cpython_api([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) -def PyObject_GetBuffer(space, exporter, view, flags): - """Send a request to exporter to fill in view as specified by flags. If the - exporter cannot provide a buffer of the exact type, it MUST raise - PyExc_BufferError, set view->obj to NULL and return -1. - - On success, fill in view, set view->obj to a new reference to exporter and - return 0. In the case of chained buffer providers that redirect requests - to a single object, view->obj MAY refer to this object instead of exporter. - - Successful calls to PyObject_GetBuffer() must be paired with calls to - PyBuffer_Release(), similar to malloc() and free(). Thus, after the - consumer is done with the buffer, PyBuffer_Release() must be called exactly - once. - """ - # XXX compare this implementation with PyPy2's XXX - pb = exporter.c_ob_type.c_tp_as_buffer - if not pb or not pb.c_bf_getbuffer: - w_exporter = from_ref(space, exporter) - raise oefmt(space.w_TypeError, - "a bytes-like object is required, not '%T'", w_exporter) - return generic_cpy_call(space, pb.c_bf_getbuffer, exporter, view, flags) - def fill_Py_buffer(space, buf, view): # c_buf, c_obj have been filled in ndim = buf.getndim() diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -7,14 +7,14 @@ from rpython.rlib import rgc # Force registration of gc.collect from pypy.module.cpyext.api import ( slot_function, generic_cpy_call, PyObject, Py_ssize_t, - pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr) + pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr, cts) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, ternaryfunc, binaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, getbufferproc, releasebufferproc, ssizessizeobjargproc) -from pypy.module.cpyext.pyobject import make_ref, decref +from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.memoryobject import fill_Py_buffer from pypy.module.cpyext.state import State @@ -318,13 +318,14 @@ _immutable_ = True def __init__(self, space, ptr, size, w_obj, format='B', shape=None, - strides=None, ndim=1, itemsize=1, readonly=True, - releasebufferproc=rffi.cast(rffi.VOIDP, 0)): + strides=None, ndim=1, itemsize=1, readonly=True, + needs_decref=False, + releasebufferproc=rffi.cast(rffi.VOIDP, 0)): self.space = space self.ptr = ptr self.size = size self.w_obj = w_obj # kept alive - self.pyobj = make_ref(space, w_obj) + self.pyobj = as_pyobj(space, w_obj) self.format = format self.ndim = ndim self.itemsize = itemsize @@ -348,30 +349,33 @@ # XXX: missing init_strides_from_shape self.strides = strides self.readonly = readonly + self.needs_decref = needs_decref self.releasebufferproc = releasebufferproc def releasebuffer(self): if self.pyobj: - decref(self.space, self.pyobj) + if self.needs_decref: + if self.releasebufferproc: + func_target = rffi.cast(releasebufferproc, self.releasebufferproc) + with lltype.scoped_alloc(Py_buffer) as pybuf: + pybuf.c_buf = self.ptr + pybuf.c_len = self.size + pybuf.c_ndim = cts.cast('int', self.ndim) + pybuf.c_shape = cts.cast('Py_ssize_t*', pybuf.c__shape) + pybuf.c_strides = cts.cast('Py_ssize_t*', pybuf.c__strides) + for i in range(self.ndim): + pybuf.c_shape[i] = self.shape[i] + pybuf.c_strides[i] = self.strides[i] + if self.format: + pybuf.c_format = rffi.str2charp(self.format) + else: + pybuf.c_format = rffi.str2charp("B") + generic_cpy_call(self.space, func_target, self.pyobj, pybuf) + decref(self.space, self.pyobj) self.pyobj = lltype.nullptr(PyObject.TO) else: #do not call twice return - if self.releasebufferproc: - func_target = rffi.cast(releasebufferproc, self.releasebufferproc) - with lltype.scoped_alloc(Py_buffer) as pybuf: - pybuf.c_buf = self.ptr - pybuf.c_len = self.size - pybuf.c_ndim = rffi.cast(rffi.INT_real, self.ndim) - for i in range(self.ndim): - pybuf.c_shape[i] = self.shape[i] - pybuf.c_strides[i] = self.strides[i] - if self.format: - pybuf.c_format = rffi.str2charp(self.format) - else: - pybuf.c_format = rffi.str2charp("B") - generic_cpy_call(self.space, func_target, self.pyobj, pybuf) - self.releasebufferproc = rffi.cast(rffi.VOIDP, 0) def getlength(self): return self.size @@ -467,22 +471,25 @@ ptr = pybuf.c_buf size = pybuf.c_len ndim = widen(pybuf.c_ndim) + shape = None if pybuf.c_shape: - shape = [pybuf.c_shape[i] for i in range(ndim)] - else: - shape = None + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = None if pybuf.c_strides: strides = [pybuf.c_strides[i] for i in range(ndim)] - else: - strides = [1] if pybuf.c_format: format = rffi.charp2str(pybuf.c_format) else: format = 'B' + # the CPython docs mandates that you do an incref whenever you call + # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't + # leak we release the buffer: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer buf = CPyBuffer(space, ptr, size, w_self, format=format, ndim=ndim, shape=shape, strides=strides, itemsize=pybuf.c_itemsize, readonly=widen(pybuf.c_readonly), + needs_decref=True, releasebufferproc = rbp) fq.register_finalizer(buf) return space.newbuffer(buf, itemsize=buf.itemsize) @@ -717,33 +724,12 @@ slot_func = slot_tp_new elif name == 'tp_as_buffer.c_bf_getbuffer': buff_fn = w_type.getdictvalue(space, '__buffer__') - if buff_fn is None: + if buff_fn is not None: + buff_w = slot_from___buffer__(space, typedef, buff_fn) + elif typedef.buffer: + buff_w = slot_from_buffer_w(space, typedef, buff_fn) + else: return - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def buff_w(space, w_self, view, flags): - args = Arguments(space, [space.newint(flags)]) - w_obj = space.call_args(space.get(buff_fn, w_self), args) - if view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - view.c_obj = make_ref(space, w_s) - view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, view) - return ret - return 0 - # XXX remove this when it no longer crashes a translated PyPy - return slot_func = buff_w else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce @@ -753,6 +739,60 @@ return slot_func + +def slot_from___buffer__(space, typedef, buff_fn): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, view, flags): + args = Arguments(space, [space.newint(flags)]) + w_obj = space.call_args(space.get(buff_fn, w_self), args) + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + return buff_w + +def slot_from_buffer_w(space, typedef, buff_fn): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, view, flags): + w_obj = w_self + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + return buff_w + PyWrapperFlag_KEYWORDS = 1 class TypeSlot: diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -98,6 +98,18 @@ /* Buffer C-API for Python 3.0 */ +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + void PyBuffer_Release(Py_buffer *view) { diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2351,10 +2351,10 @@ } else if(obj1->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); else if(obj2->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -2401,10 +2401,10 @@ } else if(obj1->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); else if(obj2->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -2426,7 +2426,7 @@ static int -array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) +array_getbuffer(arrayobject *self, Py_buffer *view, int flags) { if (view==NULL) goto finish; @@ -2463,10 +2463,19 @@ return 0; } +static long releasebuffer_cnt = 0; + +static PyObject * +get_releasebuffer_cnt(void) +{ + return PyLong_FromLong(releasebuffer_cnt); +} + static void -array_buffer_relbuf(arrayobject *self, Py_buffer *view) +array_releasebuffer(arrayobject *self, Py_buffer *view) { self->ob_exports--; + releasebuffer_cnt++; } static PySequenceMethods array_as_sequence = { @@ -2483,8 +2492,8 @@ }; static PyBufferProcs array_as_buffer = { - (getbufferproc)array_buffer_getbuf, - (releasebufferproc)array_buffer_relbuf + (getbufferproc)array_getbuffer, + (releasebufferproc)array_releasebuffer }; static PyObject * @@ -2872,6 +2881,16 @@ } static PyObject * +create_and_release_buffer(PyObject *self, PyObject *obj) +{ + Py_buffer view; + int res = PyObject_GetBuffer(obj, &view, 0); + if (res < 0) + return NULL; + PyBuffer_Release(&view); + Py_RETURN_NONE; +} +static PyObject * write_buffer_len(PyObject * self, PyObject * obj) { void* buf; @@ -2890,6 +2909,8 @@ PyDoc_STR("Internal. Used for pickling support.")}, {"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL}, {"readbuffer_as_string", (PyCFunction)readbuffer_as_string, METH_VARARGS, NULL}, + {"get_releasebuffer_cnt", (PyCFunction)get_releasebuffer_cnt, METH_NOARGS, NULL}, + {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {"write_buffer_len", write_buffer_len, METH_O, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -69,6 +69,23 @@ b'\x03\0\0\0' b'\x04\0\0\0') + def test_releasebuffer(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert module.get_releasebuffer_cnt() == 0 + module.create_and_release_buffer(arr) + assert module.get_releasebuffer_cnt() == 1 + + def test_Py_buffer(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert module.get_releasebuffer_cnt() == 0 + m = memoryview(arr) + assert module.get_releasebuffer_cnt() == 0 + del m + self.debug_collect() + assert module.get_releasebuffer_cnt() == 1 + def test_0d_view(self): module = self.import_module(name='array') arr = module.array('B', b'\0\0\0\x01') diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -10,15 +10,6 @@ only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" class TestMemoryViewObject(BaseApiTest): - def test_fromobject(self, space, api): - w_hello = space.newbytes("hello") - #assert api.PyObject_CheckBuffer(w_hello) - w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello)) - w_byte = space.call_method(w_view, '__getitem__', space.wrap(0)) - assert space.eq_w(w_byte, space.wrap(ord('h'))) - w_bytes = space.call_method(w_view, "tobytes") - assert space.unwrap(w_bytes) == "hello" - def test_frombuffer(self, space, api): w_buf = space.newbuffer(StringBuffer("hello")) c_memoryview = rffi.cast( @@ -96,6 +87,19 @@ assert result.tobytes() == b'hello, world.' class AppTestBufferProtocol(AppTestCpythonExtensionBase): + def test_fromobject(self): + foo = self.import_extension('foo', [ + ("make_view", "METH_O", + """ + if (!PyObject_CheckBuffer(args)) + return Py_None; + return PyMemoryView_FromObject(args); + """)]) + hello = b'hello' + mview = foo.make_view(hello) + assert mview[0] == hello[0] + assert mview.tobytes() == hello + def test_buffer_protocol_app(self): module = self.import_module(name='buffer_test') arr = module.PyMyArray(10) @@ -183,3 +187,75 @@ " on too long format string" finally: warnings.resetwarnings() + + def test_releasebuffer(self): + if not self.runappdirect: + skip("Fails due to ll2ctypes nonsense") + module = self.import_extension('foo', [ + ("create_test", "METH_NOARGS", + """ + PyObject *obj; + obj = PyObject_New(PyObject, (PyTypeObject*)type); + return obj; + """), + ("get_cnt", "METH_NOARGS", + 'return PyLong_FromLong(cnt);'), + ("get_dealloc_cnt", "METH_NOARGS", + 'return PyLong_FromLong(dealloc_cnt);'), + ], + prologue=""" + static float test_data = 42.f; + static int cnt=0; + static int dealloc_cnt=0; + static PyHeapTypeObject * type=NULL; + + void dealloc(PyObject *self) { + dealloc_cnt++; + } + int getbuffer(PyObject *obj, Py_buffer *view, int flags) { + + cnt ++; + memset(view, 0, sizeof(Py_buffer)); + view->obj = obj; + /* see the CPython docs for why we need this incref: + https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer */ + Py_INCREF(obj); + view->ndim = 0; + view->buf = (void *) &test_data; + view->itemsize = sizeof(float); + view->len = 1; + view->strides = NULL; + view->shape = NULL; + view->format = "f"; + return 0; + } + + void releasebuffer(PyObject *obj, Py_buffer *view) { + cnt --; + } + """, more_init=""" + type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + + type->ht_type.tp_name = "Test"; + type->ht_type.tp_basicsize = sizeof(PyObject); + type->ht_name = PyString_FromString("Test"); + type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_NEWBUFFER; + type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; + + type->ht_type.tp_dealloc = dealloc; + type->ht_type.tp_as_buffer = &type->as_buffer; + type->as_buffer.bf_getbuffer = getbuffer; + type->as_buffer.bf_releasebuffer = releasebuffer; + + if (PyType_Ready(&type->ht_type) < 0) INITERROR; + """, ) + import gc + assert module.get_cnt() == 0 + a = memoryview(module.create_test()) + assert module.get_cnt() == 1 + assert module.get_dealloc_cnt() == 0 + del a + self.debug_collect() + assert module.get_cnt() == 0 + assert module.get_dealloc_cnt() == 1 diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -1513,7 +1513,7 @@ return res """, filename=__file__).interphook('ptp') -W_NDimArray.typedef = TypeDef("numpy.ndarray", +W_NDimArray.typedef = TypeDef("numpy.ndarray", None, None, 'read-write', __new__ = interp2app(descr_new_array), __len__ = interp2app(W_NDimArray.descr_len), diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -524,7 +524,8 @@ ll_attach = rffi.llexternal("AttachToVS", [], lltype.Void, compilation_info=make_vs_attach_eci()) def impl_attach_gdb(): - ll_attach() + #ll_attach() + print "AttachToVS is disabled at the moment (compilation failure)" register_external(attach_gdb, [], result=None, export_name="impl_attach_gdb", llimpl=impl_attach_gdb) From pypy.commits at gmail.com Fri Mar 3 12:57:11 2017 From: pypy.commits at gmail.com (rlamy) Date: Fri, 03 Mar 2017 09:57:11 -0800 (PST) Subject: [pypy-commit] pypy default: Remove wrong import Message-ID: <58b9ae77.8d63190a.e021b.d909@mx.google.com> Author: Ronan Lamy Branch: Changeset: r90516:f29f4899fafc Date: 2017-03-03 18:56 +0100 http://bitbucket.org/pypy/pypy/changeset/f29f4899fafc/ Log: Remove wrong import diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -12,7 +12,6 @@ from pypy.objspace.std.memoryobject import W_MemoryView from pypy.module.cpyext.object import _dealloc from pypy.module.cpyext.import_ import PyImport_Import -from pypy.module.cpyext.buffer import PyObject_CheckBuffer PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView") From pypy.commits at gmail.com Fri Mar 3 17:22:04 2017 From: pypy.commits at gmail.com (mjacob) Date: Fri, 03 Mar 2017 14:22:04 -0800 (PST) Subject: [pypy-commit] pypy default: Add optimized "zero-copy" path for io.FileIO.readinto(). Message-ID: <58b9ec8c.8c2b190a.1f257.e4f5@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90517:2d9e1b4a2e61 Date: 2017-03-03 23:21 +0100 http://bitbucket.org/pypy/pypy/changeset/2d9e1b4a2e61/ Log: Add optimized "zero-copy" path for io.FileIO.readinto(). This path is taken if the passed buffer has a raw address and is bigger than 64 bytes. diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py --- a/pypy/interpreter/error.py +++ b/pypy/interpreter/error.py @@ -504,14 +504,16 @@ exception_name=exception_name, w_exception_class=w_exception_class) -def exception_from_saved_errno(space, w_type): - from rpython.rlib.rposix import get_saved_errno - - errno = get_saved_errno() +def exception_from_errno(space, w_type, errno): msg = os.strerror(errno) w_error = space.call_function(w_type, space.newint(errno), space.newtext(msg)) return OperationError(w_type, w_error) +def exception_from_saved_errno(space, w_type): + from rpython.rlib.rposix import get_saved_errno + errno = get_saved_errno() + return exception_from_errno(space, w_type, errno) + def new_exception_class(space, name, w_bases=None, w_dict=None): """Create a new exception type. @param name: the name of the type. diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -4,7 +4,8 @@ from pypy.interpreter.typedef import ( TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w) from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault -from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list +from rpython.rlib.rgc import ( + nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr) from rpython.rlib.buffer import Buffer from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import r_longlong, intmask @@ -162,7 +163,8 @@ raise oefmt(space.w_ValueError, "buffer size must be strictly positive") - self.buffer = ['\0'] * self.buffer_size + self.buffer = resizable_list_supporting_raw_ptr(['\0'] * + self.buffer_size) self.lock = TryLock(space) @@ -561,7 +563,7 @@ if n <= current_size: return self._read_fast(n) - result_buffer = ['\0'] * n + result_buffer = resizable_list_supporting_raw_ptr(['\0'] * n) remaining = n written = 0 if current_size: diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -1,9 +1,11 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import ( - OperationError, oefmt, wrap_oserror, wrap_oserror2) + OperationError, oefmt, wrap_oserror, wrap_oserror2, exception_from_errno) from rpython.rlib.rarithmetic import r_longlong +from rpython.rlib.rposix import get_saved_errno from rpython.rlib.rstring import StringBuilder +from rpython.rtyper.lltypesystem import lltype, rffi from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC import sys, os, stat, errno from pypy.module._io.interp_iobase import W_RawIOBase, convert_size @@ -112,6 +114,15 @@ return currentsize + BIGCHUNK return currentsize + SMALLCHUNK + +_WIN32 = sys.platform.startswith('win') +UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' + +os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], + rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) + + class W_FileIO(W_RawIOBase): def __init__(self, space): W_RawIOBase.__init__(self, space) @@ -368,15 +379,38 @@ self._check_readable(space) rwbuffer = space.getarg_w('w*', w_buffer) length = rwbuffer.getlength() - try: - buf = os.read(self.fd, length) - except OSError as e: - if e.errno == errno.EAGAIN: - return space.w_None - raise wrap_oserror(space, e, - exception_name='w_IOError') - rwbuffer.setslice(0, buf) - return space.newint(len(buf)) + + target_address = lltype.nullptr(rffi.CCHARP.TO) + if length > 64: + try: + target_address = rwbuffer.get_raw_address() + except ValueError: + pass + + if not target_address: + # unoptimized case + try: + buf = os.read(self.fd, length) + except OSError as e: + if e.errno == errno.EAGAIN: + return space.w_None + raise wrap_oserror(space, e, + exception_name='w_IOError') + rwbuffer.setslice(0, buf) + return space.newint(len(buf)) + else: + # optimized case: reading more than 64 bytes into a rwbuffer + # with a valid raw address + got = os_read(self.fd, target_address, length) + got = rffi.cast(lltype.Signed, got) + if got >= 0: + return space.newint(got) + else: + err = get_saved_errno() + if err == errno.EAGAIN: + return space.w_None + raise exception_from_errno(space, space.w_IOError, err) + keepalive_until_here(rwbuffer) def readall_w(self, space): self._check_closed(space) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -1,3 +1,4 @@ +from pypy.interpreter.gateway import interp2app from rpython.tool.udir import udir import os @@ -13,6 +14,11 @@ self.w_posix = self.space.appexec([], """(): import %s as m; return m""" % os.name) + def create_bigfile_w(): + bigfile = udir.join('bigfile') + bigfile.write('a' * 1000, mode='wb') + return self.space.wrap(str(bigfile)) + self.w_create_bigfile = self.space.wrap(interp2app(create_bigfile_w)) def test_constructor(self): import _io @@ -154,6 +160,13 @@ f.close() assert a == 'a\nbxxxxxxx' + def test_readinto_optimized(self): + import _io + a = bytearray('x' * 1024) + f = _io.FileIO(self.create_bigfile(), 'r+') + assert f.readinto(a) == 1000 + assert a == 'a' * 1000 + 'x' * 24 + def test_nonblocking_read(self): try: import os, fcntl @@ -169,6 +182,8 @@ assert f.read(10) is None a = bytearray('x' * 10) assert f.readinto(a) is None + a2 = bytearray('x' * 1024) + assert f.readinto(a2) is None def test_repr(self): import _io From pypy.commits at gmail.com Fri Mar 3 17:37:02 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 03 Mar 2017 14:37:02 -0800 (PST) Subject: [pypy-commit] pypy default: Move keepalive_until_here() to some place where it is reached Message-ID: <58b9f00e.4f5c190a.cb3de.c605@mx.google.com> Author: Armin Rigo Branch: Changeset: r90518:237c12abc97e Date: 2017-03-03 23:36 +0100 http://bitbucket.org/pypy/pypy/changeset/237c12abc97e/ Log: Move keepalive_until_here() to some place where it is reached diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -402,6 +402,7 @@ # optimized case: reading more than 64 bytes into a rwbuffer # with a valid raw address got = os_read(self.fd, target_address, length) + keepalive_until_here(rwbuffer) got = rffi.cast(lltype.Signed, got) if got >= 0: return space.newint(got) @@ -410,7 +411,6 @@ if err == errno.EAGAIN: return space.w_None raise exception_from_errno(space, space.w_IOError, err) - keepalive_until_here(rwbuffer) def readall_w(self, space): self._check_closed(space) From pypy.commits at gmail.com Fri Mar 3 18:43:39 2017 From: pypy.commits at gmail.com (mjacob) Date: Fri, 03 Mar 2017 15:43:39 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58b9ffab.16502e0a.d6639.e3c2@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90519:5adc6b27b53a Date: 2017-03-04 00:43 +0100 http://bitbucket.org/pypy/pypy/changeset/5adc6b27b53a/ Log: hg merge default diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py --- a/pypy/interpreter/error.py +++ b/pypy/interpreter/error.py @@ -672,15 +672,17 @@ w_filename2=w_filename2, eintr_retry=eintr_retry) -def exception_from_saved_errno(space, w_type): - from rpython.rlib.rposix import get_saved_errno - - errno = get_saved_errno() +def exception_from_errno(space, w_type, errno): msg = strerror(errno) w_error = space.call_function(w_type, space.newint(errno), space.newunicode(msg)) return OperationError(w_type, w_error) +def exception_from_saved_errno(space, w_type): + from rpython.rlib.rposix import get_saved_errno + errno = get_saved_errno() + return exception_from_errno(space, w_type, errno) + def new_exception_class(space, name, w_bases=None, w_dict=None): """Create a new exception type. @param name: the name of the type. diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -5,6 +5,8 @@ TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w) from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from rpython.rlib.buffer import Buffer, SubBuffer +from rpython.rlib.rgc import ( + nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr) from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import r_longlong, intmask from rpython.rlib import rposix @@ -159,7 +161,7 @@ def __init__(self, n): self.length = n - self.buf = ['\0'] * n + self.buf = resizable_list_supporting_raw_ptr(['\0'] * n) self.readonly = False def getlength(self): diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -1,11 +1,13 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import ( - OperationError, oefmt, wrap_oserror, wrap_oserror2) + OperationError, oefmt, wrap_oserror, wrap_oserror2, exception_from_errno) from rpython.rlib.rarithmetic import r_longlong +from rpython.rlib.rposix import get_saved_errno from rpython.rlib.rstring import StringBuilder from rpython.rlib import rposix from rpython.rlib.rposix_stat import STAT_FIELD_TYPES +from rpython.rtyper.lltypesystem import lltype, rffi from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL import sys, os, stat, errno from pypy.module._io.interp_iobase import ( @@ -126,6 +128,15 @@ return currentsize + BIGCHUNK return currentsize + SMALLCHUNK + +_WIN32 = sys.platform.startswith('win') +UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' + +os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], + rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) + + class W_FileIO(W_RawIOBase): def __init__(self, space): W_RawIOBase.__init__(self, space) @@ -447,18 +458,42 @@ self._check_readable(space) rwbuffer = space.getarg_w('w*', w_buffer) length = rwbuffer.getlength() - while True: + + target_address = lltype.nullptr(rffi.CCHARP.TO) + if length > 64: try: - buf = os.read(self.fd, length) - break - except OSError as e: - if e.errno == errno.EAGAIN: + target_address = rwbuffer.get_raw_address() + except ValueError: + pass + + if not target_address: + # unoptimized case + while True: + try: + buf = os.read(self.fd, length) + break + except OSError as e: + if e.errno == errno.EAGAIN: + return space.w_None + wrap_oserror(space, e, + exception_name='w_IOError', + eintr_retry=True) + rwbuffer.setslice(0, buf) + return space.newint(len(buf)) + else: + # optimized case: reading more than 64 bytes into a rwbuffer + # with a valid raw address + # XXX TODO(mjacob): implement PEP 475 here! + got = os_read(self.fd, target_address, length) + got = rffi.cast(lltype.Signed, got) + if got >= 0: + return space.newint(got) + else: + err = get_saved_errno() + if err == errno.EAGAIN: return space.w_None - wrap_oserror(space, e, - exception_name='w_IOError', - eintr_retry=True) - rwbuffer.setslice(0, buf) - return space.newint(len(buf)) + raise exception_from_errno(space, space.w_IOError, err) + keepalive_until_here(rwbuffer) def readall_w(self, space): self._check_closed(space) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -1,4 +1,5 @@ # encoding: utf-8 +from pypy.interpreter.gateway import interp2app from rpython.tool.udir import udir import os @@ -14,6 +15,11 @@ self.w_posix = self.space.appexec([], """(): import %s as m; return m""" % os.name) + def create_bigfile_w(): + bigfile = udir.join('bigfile') + bigfile.write('a' * 1000, mode='wb') + return self.space.wrap(str(bigfile)) + self.w_create_bigfile = self.space.wrap(interp2app(create_bigfile_w)) def test_constructor(self): import _io @@ -151,13 +157,10 @@ import _io a = bytearray(b'x' * 10) f = _io.FileIO(self.tmpfile, 'r+') - f.seek(5) - f.write(b'\x00' * 5) - f.seek(0) assert f.readinto(a) == 5 f.seek(0) m = memoryview(bytearray(b"helloworld")) - assert f.readinto(m) == 10 + assert f.readinto(m) == 5 # exc = raises(TypeError, f.readinto, u"hello") msg = str(exc.value) @@ -179,6 +182,13 @@ f.close() assert a == b'a\nbxxxxxxx' + def test_readinto_optimized(self): + import _io + a = bytearray(b'x' * 1024) + f = _io.FileIO(self.create_bigfile(), 'r+') + assert f.readinto(a) == 1000 + assert a == b'a' * 1000 + b'x' * 24 + def test_nonblocking_read(self): try: import os, fcntl @@ -194,6 +204,8 @@ assert f.read(10) is None a = bytearray(b'x' * 10) assert f.readinto(a) is None + a2 = bytearray(b'x' * 1024) + assert f.readinto(a2) is None def test_repr(self): import _io From pypy.commits at gmail.com Fri Mar 3 18:48:24 2017 From: pypy.commits at gmail.com (mjacob) Date: Fri, 03 Mar 2017 15:48:24 -0800 (PST) Subject: [pypy-commit] pypy default: Fix translation by adding import. Message-ID: <58ba00c8.0528190a.a556c.e1c4@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90520:ef6200437274 Date: 2017-03-04 00:47 +0100 http://bitbucket.org/pypy/pypy/changeset/ef6200437274/ Log: Fix translation by adding import. diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -2,6 +2,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import ( OperationError, oefmt, wrap_oserror, wrap_oserror2, exception_from_errno) +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.rposix import get_saved_errno from rpython.rlib.rstring import StringBuilder From pypy.commits at gmail.com Fri Mar 3 18:49:31 2017 From: pypy.commits at gmail.com (mjacob) Date: Fri, 03 Mar 2017 15:49:31 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58ba010b.5591190a.49f2a.e528@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90521:2b8db666aa18 Date: 2017-03-04 00:48 +0100 http://bitbucket.org/pypy/pypy/changeset/2b8db666aa18/ Log: hg merge default diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -2,6 +2,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import ( OperationError, oefmt, wrap_oserror, wrap_oserror2, exception_from_errno) +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.rposix import get_saved_errno from rpython.rlib.rstring import StringBuilder @@ -485,6 +486,7 @@ # with a valid raw address # XXX TODO(mjacob): implement PEP 475 here! got = os_read(self.fd, target_address, length) + keepalive_until_here(rwbuffer) got = rffi.cast(lltype.Signed, got) if got >= 0: return space.newint(got) @@ -493,7 +495,6 @@ if err == errno.EAGAIN: return space.w_None raise exception_from_errno(space, space.w_IOError, err) - keepalive_until_here(rwbuffer) def readall_w(self, space): self._check_closed(space) From pypy.commits at gmail.com Fri Mar 3 23:27:40 2017 From: pypy.commits at gmail.com (pjenvey) Date: Fri, 03 Mar 2017 20:27:40 -0800 (PST) Subject: [pypy-commit] pypy py3.5: idna needs unicodedata Message-ID: <58ba423c.4395190a.a6c86.e81b@mx.google.com> Author: Philip Jenvey Branch: py3.5 Changeset: r90522:35f5281b39f7 Date: 2017-03-03 20:04 -0800 http://bitbucket.org/pypy/pypy/changeset/35f5281b39f7/ Log: idna needs unicodedata diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -296,7 +296,8 @@ class AppTestSocket: - spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct', 'select']) + spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct', 'select', + 'unicodedata']) def setup_class(cls): cls.space = space @@ -667,8 +668,6 @@ def test_hostname_unicode(self): import _socket domain = u"испытание.pythontest.net" - # XXX figure out why the idna encoding is sometimes missing in - # tests, notably if we run all tests instead of just this one _socket.gethostbyname(domain) _socket.gethostbyname_ex(domain) _socket.getaddrinfo(domain, 0, _socket.AF_UNSPEC, _socket.SOCK_STREAM) From pypy.commits at gmail.com Fri Mar 3 23:27:44 2017 From: pypy.commits at gmail.com (pjenvey) Date: Fri, 03 Mar 2017 20:27:44 -0800 (PST) Subject: [pypy-commit] pypy py3.5: handle surrogates in filenames Message-ID: <58ba4240.90202e0a.ae30f.f1b6@mx.google.com> Author: Philip Jenvey Branch: py3.5 Changeset: r90524:28c92faba249 Date: 2017-03-03 20:26 -0800 http://bitbucket.org/pypy/pypy/changeset/28c92faba249/ Log: handle surrogates in filenames diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py --- a/pypy/module/_warnings/interp_warnings.py +++ b/pypy/module/_warnings/interp_warnings.py @@ -226,14 +226,16 @@ return False def normalize_module(space, w_filename): - filename = space.text_w(w_filename) + # XXX: could be more efficient (doesn't necessarily need + # fsencoding/redecoding) + filename = space.fsencode_w(w_filename) if len(filename) == 0: return space.newtext("") if filename.endswith(".py"): n = len(filename) - 3 assert n >= 0 filename = filename[:n] - return space.newtext(filename) + return space.newfilename(filename) return w_filename def show_warning(space, w_filename, lineno, w_text, w_category, From pypy.commits at gmail.com Fri Mar 3 23:27:42 2017 From: pypy.commits at gmail.com (pjenvey) Date: Fri, 03 Mar 2017 20:27:42 -0800 (PST) Subject: [pypy-commit] pypy py3.5: lost this in the d4951c9a2236 merge Message-ID: <58ba423e.1e142e0a.abb50.ed90@mx.google.com> Author: Philip Jenvey Branch: py3.5 Changeset: r90523:766232ac3729 Date: 2017-03-03 20:26 -0800 http://bitbucket.org/pypy/pypy/changeset/766232ac3729/ Log: lost this in the d4951c9a2236 merge diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -565,6 +565,8 @@ raise oefmt(space.w_ValueError, "negative buffersize in recv_into") if nbytes == 0: nbytes = lgt + if lgt < nbytes: + raise oefmt(space.w_ValueError, "buffer too small for requested bytes") while True: try: nbytes_read = self.sock.recvinto(rwbuffer, nbytes, flags) From pypy.commits at gmail.com Sat Mar 4 04:32:30 2017 From: pypy.commits at gmail.com (mjacob) Date: Sat, 04 Mar 2017 01:32:30 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: Experiment: add -flto to CFLAGS. This breaks asmgcc! Message-ID: <58ba89ae.1bd8190a.b4963.438d@mx.google.com> Author: Manuel Jacob Branch: shadowstack-perf-2 Changeset: r90525:7331592bdd2b Date: 2017-03-04 10:29 +0100 http://bitbucket.org/pypy/pypy/changeset/7331592bdd2b/ Log: Experiment: add -flto to CFLAGS. This breaks asmgcc! diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py --- a/rpython/translator/platform/linux.py +++ b/rpython/translator/platform/linux.py @@ -13,7 +13,7 @@ + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) cflags = tuple( - ['-O3', '-pthread', '-fomit-frame-pointer', + ['-O3', '-pthread', '-fomit-frame-pointer', '-flto', '-Wall', '-Wno-unused'] + os.environ.get('CFLAGS', '').split()) standalone_only = () From pypy.commits at gmail.com Sat Mar 4 05:07:57 2017 From: pypy.commits at gmail.com (rlamy) Date: Sat, 04 Mar 2017 02:07:57 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Fix test_suboffsets Message-ID: <58ba91fd.4c142e0a.4f67f.f363@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90526:c17b3acf5242 Date: 2017-03-04 11:07 +0100 http://bitbucket.org/pypy/pypy/changeset/c17b3acf5242/ Log: Fix test_suboffsets diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -214,6 +214,15 @@ print(module.fmt(b'd:%d', 10)) assert module.fmt(b'd:%d', 10) == b'd:10' + def test_suboffsets(self): + module = self.import_extension('foo', [ + ("check_suboffsets", "METH_O", + """ + Py_buffer view; + PyObject_GetBuffer(args, &view, 0); + return PyLong_FromLong(view.suboffsets == NULL); + """)]) + assert module.check_suboffsets(b'1234') == 1 class TestBytes(BaseApiTest): def test_bytes_resize(self, space, api): @@ -304,9 +313,3 @@ assert api.PyBytes_FromObject(w_obj) is None api.PyErr_Clear() - def test_suboffsets(self, space, api): - w_bytes = space.newbytes('1234') - view = lltype.malloc(Py_buffer, flavor='raw', zero=True) - flags = rffi.cast(rffi.INT_real, 0) - api.PyObject_GetBuffer(w_bytes, view, flags) - assert not view.c_suboffsets From pypy.commits at gmail.com Sat Mar 4 05:32:27 2017 From: pypy.commits at gmail.com (rlamy) Date: Sat, 04 Mar 2017 02:32:27 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Fix test_releasebuffer Message-ID: <58ba97bb.01462e0a.765d5.f915@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90527:422601c23f30 Date: 2017-03-04 11:31 +0100 http://bitbucket.org/pypy/pypy/changeset/422601c23f30/ Log: Fix test_releasebuffer diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -238,9 +238,9 @@ type->ht_type.tp_name = "Test"; type->ht_type.tp_basicsize = sizeof(PyObject); - type->ht_name = PyString_FromString("Test"); + type->ht_name = PyUnicode_FromString("Test"); type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_NEWBUFFER; + Py_TPFLAGS_HEAPTYPE; type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; type->ht_type.tp_dealloc = dealloc; From pypy.commits at gmail.com Sat Mar 4 06:03:39 2017 From: pypy.commits at gmail.com (mjacob) Date: Sat, 04 Mar 2017 03:03:39 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Fix direct app-level test. Message-ID: <58ba9f0b.12a3190a.51bd6.f691@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90528:740e021e095b Date: 2017-03-04 12:03 +0100 http://bitbucket.org/pypy/pypy/changeset/740e021e095b/ Log: Fix direct app-level test. diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -15,11 +15,10 @@ self.w_posix = self.space.appexec([], """(): import %s as m; return m""" % os.name) - def create_bigfile_w(): + if meth == self.test_readinto_optimized: bigfile = udir.join('bigfile') bigfile.write('a' * 1000, mode='wb') - return self.space.wrap(str(bigfile)) - self.w_create_bigfile = self.space.wrap(interp2app(create_bigfile_w)) + self.w_bigfile = self.space.wrap(self.space.wrap(str(bigfile))) def test_constructor(self): import _io @@ -185,7 +184,7 @@ def test_readinto_optimized(self): import _io a = bytearray(b'x' * 1024) - f = _io.FileIO(self.create_bigfile(), 'r+') + f = _io.FileIO(self.bigfile, 'r+') assert f.readinto(a) == 1000 assert a == b'a' * 1000 + b'x' * 24 From pypy.commits at gmail.com Sat Mar 4 06:06:39 2017 From: pypy.commits at gmail.com (mjacob) Date: Sat, 04 Mar 2017 03:06:39 -0800 (PST) Subject: [pypy-commit] pypy default: Fix direct app-level test. Message-ID: <58ba9fbf.548d190a.b77b5.816f@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90529:bdf68ecd0bac Date: 2017-03-04 12:03 +0100 http://bitbucket.org/pypy/pypy/changeset/bdf68ecd0bac/ Log: Fix direct app-level test. diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -14,11 +14,10 @@ self.w_posix = self.space.appexec([], """(): import %s as m; return m""" % os.name) - def create_bigfile_w(): + if meth == self.test_readinto_optimized: bigfile = udir.join('bigfile') bigfile.write('a' * 1000, mode='wb') - return self.space.wrap(str(bigfile)) - self.w_create_bigfile = self.space.wrap(interp2app(create_bigfile_w)) + self.w_bigfile = self.space.wrap(self.space.wrap(str(bigfile))) def test_constructor(self): import _io @@ -163,7 +162,7 @@ def test_readinto_optimized(self): import _io a = bytearray('x' * 1024) - f = _io.FileIO(self.create_bigfile(), 'r+') + f = _io.FileIO(self.bigfile, 'r+') assert f.readinto(a) == 1000 assert a == 'a' * 1000 + 'x' * 24 From pypy.commits at gmail.com Sat Mar 4 06:37:12 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Mar 2017 03:37:12 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: (remi, arigo around) fix for a "corner case" of having a startblock Message-ID: <58baa6e8.4aa7190a.3a00e.f547@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90530:f2ae43e31f49 Date: 2017-03-04 12:35 +0100 http://bitbucket.org/pypy/pypy/changeset/f2ae43e31f49/ Log: (remi, arigo around) fix for a "corner case" of having a startblock that is also reached from the rest of the graph diff --git a/rpython/memory/gctransform/shadowcolor.py b/rpython/memory/gctransform/shadowcolor.py --- a/rpython/memory/gctransform/shadowcolor.py +++ b/rpython/memory/gctransform/shadowcolor.py @@ -233,6 +233,7 @@ if not regalloc: return + insert_empty_startblock(graph) entrymap = mkentrymap(graph) inputvars = {} # {inputvar: (its block, its index in inputargs)} for block in graph.iterblocks(): diff --git a/rpython/memory/gctransform/test/test_shadowcolor.py b/rpython/memory/gctransform/test/test_shadowcolor.py --- a/rpython/memory/gctransform/test/test_shadowcolor.py +++ b/rpython/memory/gctransform/test/test_shadowcolor.py @@ -671,6 +671,26 @@ add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) postprocess_double_check(graph) +def test_bug_2(): + def f(w_tup): + while True: + llop.gc_push_roots(lltype.Void, w_tup) + llop.gc_pop_roots(lltype.Void, w_tup) + + graph = make_graph(f, [llmemory.GCREF]) + assert not graph.startblock.operations + # this test is about what occurs if the startblock of the graph + # is also reached from another block. None of the 'simplify' + # functions actually remove that, but the JIT transformation can... + graph.startblock = graph.startblock.exits[0].target + + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + def test_add_enter_roots_frame_remove_empty(): class W: pass From pypy.commits at gmail.com Sat Mar 4 06:59:37 2017 From: pypy.commits at gmail.com (Raemi) Date: Sat, 04 Mar 2017 03:59:37 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) add lock to slowpath of write barrier Message-ID: <58baac29.8fdb190a.35b46.f4c3@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90531:27908e5859b8 Date: 2017-03-04 12:59 +0100 http://bitbucket.org/pypy/pypy/changeset/27908e5859b8/ Log: (arigo, remi) add lock to slowpath of write barrier diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -438,11 +438,11 @@ self.old_objects_pointing_to_pinned = self.AddressStack() self.updated_old_objects_pointing_to_pinned = False # - # # Allocate lock(s) - # ll_lock = lltype.malloc(rthread.TLOCKP.TO, flavor='raw', - # track_allocation=False) - # rthread.c_thread_lock_init(ll_lock) - # self.ll_lock = ll_lock + # Allocate lock(s) + wb_slowpath_lock = lltype.malloc(rthread.TLOCKP.TO, flavor='raw', + track_allocation=False) + rthread.c_thread_lock_init(wb_slowpath_lock) + self.wb_slowpath_lock = wb_slowpath_lock # # Allocate a nursery. In case of auto_nursery_size, start by # allocating a very small nursery, enough to do things like look @@ -860,7 +860,7 @@ major collection, and finally reserve totalsize bytes. """ - # rthread.acquire_NOAUTO(self.ll_lock, 1) + # rthread.acquire_NOAUTO(self.wb_slowpath_lock, 1) rgil.enter_master_section() minor_collection_count = 0 @@ -947,7 +947,7 @@ self.debug_tiny_nursery) # rgil.leave_master_section() - # rthread.release_NOAUTO(self.ll_lock) + # rthread.release_NOAUTO(self.wb_slowpath_lock) return result collect_and_reserve._dont_inline_ = True @@ -1476,6 +1476,12 @@ # make the code in write_barrier() marginally smaller # (which is important because it is inlined *everywhere*). def remember_young_pointer(addr_struct): + rthread.acquire_NOAUTO(self.wb_slowpath_lock, True) + if not llop.gc_bit(lltype.Signed, self.header(addr_struct), + GCFLAG_TRACK_YOUNG_PTRS): + rthread.release_NOAUTO(self.wb_slowpath_lock) + return + # # 'addr_struct' is the address of the object in which we write. # We know that 'addr_struct' has GCFLAG_TRACK_YOUNG_PTRS so far. # @@ -1507,6 +1513,8 @@ if objhdr.tid & GCFLAG_NO_HEAP_PTRS: objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.prebuilt_root_objects.append(addr_struct) + # + rthread.release_NOAUTO(self.wb_slowpath_lock) remember_young_pointer._dont_inline_ = True self.remember_young_pointer = remember_young_pointer @@ -1518,6 +1526,12 @@ def _init_writebarrier_with_card_marker(self): DEBUG = self.DEBUG def remember_young_pointer_from_array2(addr_array, index): + rthread.acquire_NOAUTO(self.wb_slowpath_lock, True) + if not llop.gc_bit(lltype.Signed, self.header(addr_array), + GCFLAG_TRACK_YOUNG_PTRS): + rthread.release_NOAUTO(self.wb_slowpath_lock) + return + # # 'addr_array' is the address of the object in which we write, # which must have an array part; 'index' is the index of the # item that is (or contains) the pointer that we write. @@ -1536,6 +1550,7 @@ if objhdr.tid & GCFLAG_NO_HEAP_PTRS: objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.prebuilt_root_objects.append(addr_array) + rthread.release_NOAUTO(self.wb_slowpath_lock) return # # 'addr_array' is a raw_malloc'ed array with card markers @@ -1548,6 +1563,7 @@ addr_byte = self.get_card(addr_array, byteindex) byte = ord(addr_byte.char[0]) if byte & bitmask: + rthread.release_NOAUTO(self.wb_slowpath_lock) return # # We set the flag (even if the newly written address does not @@ -1559,6 +1575,7 @@ if objhdr.tid & GCFLAG_CARDS_SET == 0: self.old_objects_with_cards_set.append(addr_array) objhdr.tid |= GCFLAG_CARDS_SET + rthread.release_NOAUTO(self.wb_slowpath_lock) remember_young_pointer_from_array2._dont_inline_ = True ll_assert(self.card_page_indices > 0, diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -4,6 +4,7 @@ import py, sys from rpython.rlib import jit, rgc from rpython.rlib.debug import ll_assert +from rpython.rlib.objectmodel import enforceargs from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.rlib.objectmodel import CDefinedIntSymbolic, not_rpython from rpython.rtyper.lltypesystem.lloperation import llop @@ -253,6 +254,7 @@ c_thread_lock_dealloc_NOAUTO(ll_lock) lltype.free(ll_lock, flavor='raw', track_allocation=False) + at enforceargs(None, bool) def acquire_NOAUTO(ll_lock, flag): flag = rffi.cast(rffi.INT, int(flag)) res = c_thread_acquirelock_NOAUTO(ll_lock, flag) From pypy.commits at gmail.com Sat Mar 4 07:03:38 2017 From: pypy.commits at gmail.com (Raemi) Date: Sat, 04 Mar 2017 04:03:38 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (arigo, remi) add lock to yet another place Message-ID: <58baad1a.d426190a.f039.4c23@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90532:79fcf2f97451 Date: 2017-03-04 13:03 +0100 http://bitbucket.org/pypy/pypy/changeset/79fcf2f97451/ Log: (arigo, remi) add lock to yet another place diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -1607,6 +1607,13 @@ def writebarrier_before_copy(self, source_addr, dest_addr, source_start, dest_start, length): + rthread.acquire_NOAUTO(self.wb_slowpath_lock, True) + res = self._writebarrier_before_copy(source_addr, dest_addr, source_start, dest_start, length) + rthread.release_NOAUTO(self.wb_slowpath_lock) + return res + + def _writebarrier_before_copy(self, source_addr, dest_addr, + source_start, dest_start, length): """ This has the same effect as calling writebarrier over each element in dest copied from source, except it might reset one of the following flags a bit too eagerly, which means we'll have From pypy.commits at gmail.com Sat Mar 4 07:31:35 2017 From: pypy.commits at gmail.com (mjacob) Date: Sat, 04 Mar 2017 04:31:35 -0800 (PST) Subject: [pypy-commit] pypy default: Use wrap_oserror with a just-built OSError here. It will make things easier on py3.5. Message-ID: <58bab3a7.8d202e0a.119d3.01fd@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90533:734597bfaaec Date: 2017-03-04 12:17 +0100 http://bitbucket.org/pypy/pypy/changeset/734597bfaaec/ Log: Use wrap_oserror with a just-built OSError here. It will make things easier on py3.5. diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -1,7 +1,7 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import ( - OperationError, oefmt, wrap_oserror, wrap_oserror2, exception_from_errno) + OperationError, oefmt, wrap_oserror, wrap_oserror2) from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.rposix import get_saved_errno @@ -411,7 +411,8 @@ err = get_saved_errno() if err == errno.EAGAIN: return space.w_None - raise exception_from_errno(space, space.w_IOError, err) + e = OSError(err, "read failed") + raise wrap_oserror(space, e, exception_name='w_IOError') def readall_w(self, space): self._check_closed(space) From pypy.commits at gmail.com Sat Mar 4 07:31:38 2017 From: pypy.commits at gmail.com (mjacob) Date: Sat, 04 Mar 2017 04:31:38 -0800 (PST) Subject: [pypy-commit] pypy default: Reuse rposix.c_read in these two places instead of redefining an almost identical llexternal. Message-ID: <58bab3aa.55502e0a.2ec48.ed8f@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90534:0251b34cb8ee Date: 2017-03-04 13:30 +0100 http://bitbucket.org/pypy/pypy/changeset/0251b34cb8ee/ Log: Reuse rposix.c_read in these two places instead of redefining an almost identical llexternal. diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py --- a/pypy/module/_file/readinto.py +++ b/pypy/module/_file/readinto.py @@ -1,16 +1,10 @@ -import sys, errno +import errno from rpython.rlib import rposix from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rlib.rposix import c_read from rpython.rtyper.lltypesystem import lltype, rffi from pypy.module._file.interp_file import is_wouldblock_error, signal_checker -_WIN32 = sys.platform.startswith('win') -UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' - -os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read', - [rffi.INT, rffi.CCHARP, rffi.SIZE_T], - rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) - def direct_readinto(self, w_rwbuffer): rwbuffer = self.space.writebuf_w(w_rwbuffer) @@ -56,11 +50,11 @@ target_pos += len(data) size -= len(data) - # then call os_read() to get the rest + # then call c_read() to get the rest if size > 0: stream.flush() while True: - got = os_read(fd, rffi.ptradd(target_address, target_pos), size) + got = c_read(fd, rffi.ptradd(target_address, target_pos), size) got = rffi.cast(lltype.Signed, got) if got > 0: target_pos += got diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -4,7 +4,7 @@ OperationError, oefmt, wrap_oserror, wrap_oserror2) from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_longlong -from rpython.rlib.rposix import get_saved_errno +from rpython.rlib.rposix import c_read, get_saved_errno from rpython.rlib.rstring import StringBuilder from rpython.rtyper.lltypesystem import lltype, rffi from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC @@ -116,14 +116,6 @@ return currentsize + SMALLCHUNK -_WIN32 = sys.platform.startswith('win') -UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' - -os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read', - [rffi.INT, rffi.CCHARP, rffi.SIZE_T], - rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) - - class W_FileIO(W_RawIOBase): def __init__(self, space): W_RawIOBase.__init__(self, space) @@ -402,7 +394,7 @@ else: # optimized case: reading more than 64 bytes into a rwbuffer # with a valid raw address - got = os_read(self.fd, target_address, length) + got = c_read(self.fd, target_address, length) keepalive_until_here(rwbuffer) got = rffi.cast(lltype.Signed, got) if got >= 0: From pypy.commits at gmail.com Sat Mar 4 07:37:13 2017 From: pypy.commits at gmail.com (rlamy) Date: Sat, 04 Mar 2017 04:37:13 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge heads Message-ID: <58bab4f9.8f1f190a.e975c.f45b@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90536:ae2aed8de1b2 Date: 2017-03-04 13:36 +0100 http://bitbucket.org/pypy/pypy/changeset/ae2aed8de1b2/ Log: merge heads diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -15,11 +15,10 @@ self.w_posix = self.space.appexec([], """(): import %s as m; return m""" % os.name) - def create_bigfile_w(): + if meth == self.test_readinto_optimized: bigfile = udir.join('bigfile') bigfile.write('a' * 1000, mode='wb') - return self.space.wrap(str(bigfile)) - self.w_create_bigfile = self.space.wrap(interp2app(create_bigfile_w)) + self.w_bigfile = self.space.wrap(self.space.wrap(str(bigfile))) def test_constructor(self): import _io @@ -185,7 +184,7 @@ def test_readinto_optimized(self): import _io a = bytearray(b'x' * 1024) - f = _io.FileIO(self.create_bigfile(), 'r+') + f = _io.FileIO(self.bigfile, 'r+') assert f.readinto(a) == 1000 assert a == b'a' * 1000 + b'x' * 24 From pypy.commits at gmail.com Sat Mar 4 07:37:10 2017 From: pypy.commits at gmail.com (rlamy) Date: Sat, 04 Mar 2017 04:37:10 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Add PyUnicode_AsUTF8AndSize, change PyUnicode_AsUTF8 to use it Message-ID: <58bab4f6.5043190a.15bd4.fcb1@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90535:6ba16736df1d Date: 2017-03-04 13:35 +0100 http://bitbucket.org/pypy/pypy/changeset/6ba16736df1d/ Log: Add PyUnicode_AsUTF8AndSize, change PyUnicode_AsUTF8 to use it diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -235,6 +235,18 @@ """)]) assert module.test_macro_invocations() == u'' + def test_AsUTF8AndSize(self): + module = self.import_extension('foo', [ + ("utf8", "METH_O", + """ + Py_ssize_t size; + char *utf8 = PyUnicode_AsUTF8AndSize(args, &size); + return PyBytes_FromStringAndSize(utf8, size); + """)]) + assert module.utf8('xyz') == b'xyz' + assert module.utf8('café') == 'café'.encode('utf-8') + + class TestUnicode(BaseApiTest): def test_unicodeobject(self, space): encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, )) diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -114,6 +114,9 @@ def set_ascii(py_obj, value): get_state(py_obj).c_ascii = cts.cast('unsigned int', value) +def get_ready(py_obj): + return get_state(py_obj).c_ready + def set_ready(py_obj, value): get_state(py_obj).c_ready = cts.cast('unsigned int', value) @@ -318,8 +321,13 @@ set_wbuffer(ref, rffi.unicode2wcharp(u)) return get_wbuffer(ref) - at cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)") -def PyUnicode_AsUTF8(space, ref): + at cts.decl("char * PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize)") +def PyUnicode_AsUTF8AndSize(space, ref, psize): + if not PyUnicode_Check(space, ref): + PyErr_BadArgument(space) + if not get_ready(ref): + res = _PyUnicode_Ready(space, ref) + if not get_utf8(ref): # Copy unicode buffer w_unicode = from_ref(space, ref) @@ -327,8 +335,15 @@ "strict") s = space.bytes_w(w_encoded) set_utf8(ref, rffi.str2charp(s)) + set_utf8_len(ref, len(s)) + if psize: + psize[0] = get_utf8_len(ref) return get_utf8(ref) + at cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)") +def PyUnicode_AsUTF8(space, ref): + return PyUnicode_AsUTF8AndSize(space, ref, cts.cast('Py_ssize_t *', 0)) + @cpython_api([PyObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1) def PyUnicode_AsWideChar(space, ref, buf, size): """Copy the Unicode object contents into the wchar_t buffer w. At most From pypy.commits at gmail.com Sat Mar 4 11:04:57 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Mar 2017 08:04:57 -0800 (PST) Subject: [pypy-commit] pypy nogil-unsafe-2: (remi previously, arigo) Fix: we must not grab the next object's location from the nursery without having the GIL Message-ID: <58bae5a9.1ce5190a.d3542.06a9@mx.google.com> Author: Armin Rigo Branch: nogil-unsafe-2 Changeset: r90537:8efb35415f7d Date: 2017-03-04 17:04 +0100 http://bitbucket.org/pypy/pypy/changeset/8efb35415f7d/ Log: (remi previously, arigo) Fix: we must not grab the next object's location from the nursery without having the GIL diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -930,15 +930,17 @@ "Calling minor_collection() twice is not " "enough. Too many pinned objects?") self._minor_collection() + continue # # Tried to do something about nursery_free overflowing # nursery_top before this point. Try to reserve totalsize now. # If this succeeds break out of loop. + rgil.leave_master_section() result = self.get_nursery_free() if result + totalsize <= self.get_nursery_top(): self.set_nursery_free(result + totalsize) break - # + rgil.enter_master_section() # if self.debug_tiny_nursery >= 0: # for debugging if (self.get_nursery_top() - self.get_nursery_free() > @@ -946,7 +948,6 @@ self.set_nursery_free(self.get_nursery_top() - self.debug_tiny_nursery) # - rgil.leave_master_section() # rthread.release_NOAUTO(self.wb_slowpath_lock) return result collect_and_reserve._dont_inline_ = True From pypy.commits at gmail.com Sat Mar 4 12:17:56 2017 From: pypy.commits at gmail.com (rlamy) Date: Sat, 04 Mar 2017 09:17:56 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Add PyUnicode_AsUnicodeAndSize() Message-ID: <58baf6c4.d4d9190a.ce329.4ecc@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90538:44c53c062fbe Date: 2017-03-04 17:07 +0100 http://bitbucket.org/pypy/pypy/changeset/44c53c062fbe/ Log: Add PyUnicode_AsUnicodeAndSize() diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -306,8 +306,8 @@ set_ready(py_obj, 1) return 0 - at cpython_api([PyObject], rffi.CWCHARP) -def PyUnicode_AsUnicode(space, ref): + at cts.decl("Py_UNICODE * PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size)") +def PyUnicode_AsUnicodeAndSize(space, ref, psize): """Return a read-only pointer to the Unicode object's internal Py_UNICODE buffer, NULL if unicode is not a Unicode object.""" # Don't use PyUnicode_Check, it will realize the object :-( @@ -319,8 +319,15 @@ w_unicode = from_ref(space, rffi.cast(PyObject, ref)) u = space.unicode_w(w_unicode) set_wbuffer(ref, rffi.unicode2wcharp(u)) + set_wsize(ref, len(u)) + if psize: + psize[0] = get_wsize(ref) return get_wbuffer(ref) + at cts.decl("Py_UNICODE * PyUnicode_AsUnicode(PyObject *unicode)") +def PyUnicode_AsUnicode(space, ref): + return PyUnicode_AsUnicodeAndSize(space, ref, cts.cast('Py_ssize_t *', 0)) + @cts.decl("char * PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize)") def PyUnicode_AsUTF8AndSize(space, ref, psize): if not PyUnicode_Check(space, ref): From pypy.commits at gmail.com Sat Mar 4 13:40:01 2017 From: pypy.commits at gmail.com (fijal) Date: Sat, 04 Mar 2017 10:40:01 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: some progress, still errors in rsplit (we need to think!) Message-ID: <58bb0a01.da12190a.c2456.0adc@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90539:4c1a6dc397c4 Date: 2017-03-04 19:39 +0100 http://bitbucket.org/pypy/pypy/changeset/4c1a6dc397c4/ Log: some progress, still errors in rsplit (we need to think!) diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -37,7 +37,16 @@ if isutf8: return next_codepoint_pos(s, pos) else: - return pos + 1 + return pos + 1 + + at specialize.ll_and_arg(2) +def _decr(s, pos, isutf8): + from rpython.rlib.rutf8 import prev_codepoint_pos + + if isutf8: + return prev_codepoint_pos(s, pos) + else: + return pos - 1 @specialize.ll_and_arg(3) def split(value, by=None, maxsplit=-1, isutf8=0): @@ -132,7 +141,7 @@ while i >= 0: if not _isspace(value, i): break # found - i -= 1 + i = _decr(value, i, isutf8) else: break # end of string, finished @@ -141,18 +150,21 @@ if maxsplit == 0: j = -1 # take all the rest of the string else: - j = i - 1 + j = _decr(value, i, isutf8) while j >= 0 and not _isspace(value, j): - j -= 1 + j = _decr(value, j, isutf8) maxsplit -= 1 # NB. if it's already < 0, it stays < 0 # the word is value[j+1:i+1] - j1 = j + 1 + if j < 0: + j1 = 0 + else: + j1 = _incr(value, j, isutf8) assert j1 >= 0 res.append(value[j1:i+1]) # continue to look from the character before the space before the word - i = j - 1 + i = _decr(value, j, isutf8) res.reverse() return res diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py --- a/rpython/rlib/rutf8.py +++ b/rpython/rlib/rutf8.py @@ -57,6 +57,10 @@ return lgt raise ValueError +# note - table lookups are really slow. Measured on various elements of obama +# chinese wikipedia, they're anywhere between 10% and 30% slower. +# In extreme cases (small, only chinese text), they're 40% slower + def next_codepoint_pos(code, pos): """ Gives the position of the next codepoint after pos, -1 if it's the last one (assumes valid utf8) @@ -64,7 +68,21 @@ chr1 = ord(code[pos]) if chr1 < 0x80: return pos + 1 - return pos + ord(runicode._utf8_code_length[chr1 - 0x80]) + if 0xC2 >= chr1 <= 0xDF: + return pos + 2 + if chr1 >= 0xE0 and chr1 <= 0xEF: + return pos + 3 + return pos + 4 + +def prev_codepoint_pos(code, pos): + """ Gives the position of the previous codepoint + """ + chr1 = ord(code[pos]) + if chr1 < 0x80: + return pos - 1 + while ord(code[pos]) & 0xC0 == 0xC0: + pos -= 1 + return pos def compute_length_utf8(s): pos = 0 diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py --- a/rpython/rlib/test/test_rstring.py +++ b/rpython/rlib/test/test_rstring.py @@ -43,6 +43,12 @@ assert split(u'endcase test', u'test') == [u'endcase ', u''] py.test.raises(ValueError, split, u'abc', u'') +def test_split_utf8(): + assert split('', 'a', isutf8=1) == [''] + assert split('baba', 'a', isutf8=1) == ['b', 'b', ''] + assert split('b b', isutf8=1) == ['b', 'b'] + assert split('b\xe1\x9a\x80b', isutf8=1) == ['b', 'b'] + def test_rsplit(): def check_rsplit(value, sub, *args, **kwargs): result = kwargs['res'] @@ -77,6 +83,12 @@ assert rsplit(u'endcase test', u'test') == [u'endcase ', u''] py.test.raises(ValueError, rsplit, u"abc", u'') +def test_rsplit_utf8(): + assert rsplit('', 'a', isutf8=1) == [''] + assert rsplit('baba', 'a', isutf8=1) == ['b', 'b', ''] + assert rsplit('b b', isutf8=1) == ['b', 'b'] + assert rsplit('b\xe1\x9a\x80b', isutf8=1) == ['b', 'b'] + def test_string_replace(): def check_replace(value, sub, *args, **kwargs): result = kwargs['res'] diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py --- a/rpython/rlib/test/test_rutf8.py +++ b/rpython/rlib/test/test_rutf8.py @@ -25,9 +25,6 @@ else: assert not raised -def error_handler(errors, encoding, msg, char, start, end): - raise UnicodeDecodeError(encoding, char, start, end, msg) - @given(strategies.binary()) def test_str_check_utf8(s): try: @@ -36,11 +33,10 @@ except UnicodeDecodeError as e: valid = False try: - consumed, length = rutf8.str_check_utf8(s, len(s), None, - errorhandler=error_handler, final=True) - except UnicodeDecodeError as a: + consumed, length = rutf8.str_check_utf8(s, len(s), final=True) + except rutf8.Utf8CheckError as a: assert not valid - assert a.start == e.start + assert a.startpos == e.start # assert a.end == e.end, ideally else: assert valid From pypy.commits at gmail.com Sat Mar 4 14:22:22 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Mar 2017 11:22:22 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: Do insert_empty_startblock() earlier to avoid mutating the graph at an Message-ID: <58bb13ee.c8a8190a.aa4fb.1dee@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90540:6f2c12b6a329 Date: 2017-03-04 18:37 +0100 http://bitbucket.org/pypy/pypy/changeset/6f2c12b6a329/ Log: Do insert_empty_startblock() earlier to avoid mutating the graph at an unexpected point (and also, do it only if needed) diff --git a/rpython/memory/gctransform/shadowcolor.py b/rpython/memory/gctransform/shadowcolor.py --- a/rpython/memory/gctransform/shadowcolor.py +++ b/rpython/memory/gctransform/shadowcolor.py @@ -20,6 +20,10 @@ of the 'pending_pred', which is a list of (block, var) tuples. """ entrymap = mkentrymap(graph) + if len(entrymap[graph.startblock]) != 1: + insert_empty_startblock(graph) + entrymap = mkentrymap(graph) + pred = set([v for block, v in pending_pred]) def add(block, v): @@ -233,8 +237,9 @@ if not regalloc: return - insert_empty_startblock(graph) entrymap = mkentrymap(graph) + assert len(entrymap[graph.startblock]) == 1 + inputvars = {} # {inputvar: (its block, its index in inputargs)} for block in graph.iterblocks(): for i, v in enumerate(block.inputargs): From pypy.commits at gmail.com Sat Mar 4 16:20:38 2017 From: pypy.commits at gmail.com (fijal) Date: Sat, 04 Mar 2017 13:20:38 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: fix rsplit Message-ID: <58bb2fa6.8fdb190a.35b46.08ce@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90542:e30fd70a9177 Date: 2017-03-04 21:23 +0100 http://bitbucket.org/pypy/pypy/changeset/e30fd70a9177/ Log: fix rsplit diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -123,6 +123,8 @@ return rutf8.compute_length_utf8(self._utf8) def _val(self, space): + import pdb + pdb.set_trace() return self._utf8.decode('utf8') @staticmethod diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -35,6 +35,8 @@ from rpython.rlib.rutf8 import next_codepoint_pos if isutf8: + if pos == -1: + return 0 return next_codepoint_pos(s, pos) else: return pos + 1 @@ -44,6 +46,8 @@ from rpython.rlib.rutf8 import prev_codepoint_pos if isutf8: + if pos == 0: + return -1 return prev_codepoint_pos(s, pos) else: return pos - 1 @@ -139,7 +143,7 @@ while True: # starting from the end, find the end of the next word while i >= 0: - if not _isspace(value, i): + if not _isspace(value, i, isutf8): break # found i = _decr(value, i, isutf8) else: @@ -151,17 +155,17 @@ j = -1 # take all the rest of the string else: j = _decr(value, i, isutf8) - while j >= 0 and not _isspace(value, j): + while j >= 0 and not _isspace(value, j, isutf8): j = _decr(value, j, isutf8) maxsplit -= 1 # NB. if it's already < 0, it stays < 0 # the word is value[j+1:i+1] + j1 = _incr(value, j, isutf8) + assert j1 >= 0 + i1 = _incr(value, i, isutf8) + res.append(value[j1:i1]) if j < 0: - j1 = 0 - else: - j1 = _incr(value, j, isutf8) - assert j1 >= 0 - res.append(value[j1:i+1]) + break # continue to look from the character before the space before the word i = _decr(value, j, isutf8) diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py --- a/rpython/rlib/rutf8.py +++ b/rpython/rlib/rutf8.py @@ -77,10 +77,11 @@ def prev_codepoint_pos(code, pos): """ Gives the position of the previous codepoint """ + pos -= 1 chr1 = ord(code[pos]) if chr1 < 0x80: - return pos - 1 - while ord(code[pos]) & 0xC0 == 0xC0: + return pos + while ord(code[pos]) & 0xC0 == 0x80: pos -= 1 return pos From pypy.commits at gmail.com Sat Mar 4 16:20:40 2017 From: pypy.commits at gmail.com (fijal) Date: Sat, 04 Mar 2017 13:20:40 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: isupper & islower Message-ID: <58bb2fa8.65012e0a.7540.41c7@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90543:9406445b45f6 Date: 2017-03-04 21:55 +0100 http://bitbucket.org/pypy/pypy/changeset/9406445b45f6/ Log: isupper & islower diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -418,23 +418,30 @@ def descr_islower(self, space): cased = False - val = self._val(space) - for uchar in val: - if (unicodedb.isupper(ord(uchar)) or - unicodedb.istitle(ord(uchar))): + val = self._utf8 + i = 0 + while i < len(val): + uchar = rutf8.codepoint_at_pos(val, i) + if (unicodedb.isupper(uchar) or + unicodedb.istitle(uchar)): return space.w_False - if not cased and unicodedb.islower(ord(uchar)): + if not cased and unicodedb.islower(uchar): cased = True + i = rutf8.next_codepoint_pos(val, i) return space.newbool(cased) def descr_isupper(self, space): cased = False - for uchar in self._val(space): - if (unicodedb.islower(ord(uchar)) or - unicodedb.istitle(ord(uchar))): + i = 0 + val = self._utf8 + while i < len(val): + uchar = rutf8.codepoint_at_pos(val, i) + if (unicodedb.islower(uchar) or + unicodedb.istitle(uchar)): return space.w_False - if not cased and unicodedb.isupper(ord(uchar)): + if not cased and unicodedb.isupper(uchar): cased = True + i = rutf8.next_codepoint_pos(val, i) return space.newbool(cased) def _starts_ends_overflow(self, prefix): @@ -538,7 +545,7 @@ by = self.convert_arg_to_w_unicode(space, w_sep)._utf8 if len(by) == 0: raise oefmt(space.w_ValueError, "empty separator") - res = split(value, by, maxsplit) + res = split(value, by, maxsplit, isutf8=1) return space.newlist_from_unicode(res) @@ -547,13 +554,13 @@ res = [] value = self._utf8 if space.is_none(w_sep): - res = rsplit(value, maxsplit=maxsplit) + res = rsplit(value, maxsplit=maxsplit, isutf8=1) return space.newlist_from_unicode(res) by = self.convert_arg_to_w_unicode(space, w_sep)._utf8 if len(by) == 0: raise oefmt(space.w_ValueError, "empty separator") - res = rsplit(value, by, maxsplit) + res = rsplit(value, by, maxsplit, isutf8=1) return space.newlist_from_unicode(res) From pypy.commits at gmail.com Sat Mar 4 16:20:43 2017 From: pypy.commits at gmail.com (fijal) Date: Sat, 04 Mar 2017 13:20:43 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: ljust/rjust Message-ID: <58bb2fab.13542e0a.69d3d.0e96@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90544:8ef843d54f47 Date: 2017-03-04 22:03 +0100 http://bitbucket.org/pypy/pypy/changeset/8ef843d54f47/ Log: ljust/rjust diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -2,7 +2,7 @@ from rpython.rlib.objectmodel import ( compute_hash, compute_unique_id, import_from_mixin, - enforceargs, newlist_hint, specialize) + enforceargs, newlist_hint, specialize, we_are_translated) from rpython.rlib.buffer import StringBuffer from rpython.rlib.rstring import StringBuilder, split, rsplit, UnicodeBuilder,\ replace @@ -38,6 +38,8 @@ self._utf8 = utf8str self._length = length self._ucs4 = ucs4str + if not we_are_translated() and length != -1: + assert rutf8.compute_length_utf8(utf8str) == length def __repr__(self): """representation for debugging purposes""" @@ -629,6 +631,43 @@ except IndexError: raise oefmt(space.w_IndexError, "string index out of range") + @unwrap_spec(width=int, w_fillchar=WrappedDefault(' ')) + def descr_rjust(self, space, width, w_fillchar): + value = self._utf8 + lgt = self._len() + w_fillchar = self.convert_arg_to_w_unicode(space, w_fillchar) + if w_fillchar._len() != 1: + raise oefmt(space.w_TypeError, + "rjust() argument 2 must be a single character") + d = width - lgt + if d > 0: + if len(w_fillchar._utf8) == 1: + # speedup + value = d * w_fillchar._utf8[0] + value + else: + value = d * w_fillchar._utf8 + value + return W_UnicodeObject(value, width) + + return W_UnicodeObject(value, lgt) + + @unwrap_spec(width=int, w_fillchar=WrappedDefault(' ')) + def descr_ljust(self, space, width, w_fillchar): + value = self._utf8 + w_fillchar = self.convert_arg_to_w_unicode(space, w_fillchar) + if w_fillchar._len() != 1: + raise oefmt(space.w_TypeError, + "ljust() argument 2 must be a single character") + d = width - self._len() + if d > 0: + if len(w_fillchar._utf8) == 1: + # speedup + value = value + d * w_fillchar._utf8[0] + else: + value = value + d * w_fillchar._utf8 + return W_UnicodeObject(value, width) + + return W_UnicodeObject(value, self._len()) + def descr_getnewargs(self, space): return space.newtuple([W_UnicodeObject(self._utf8, self._length)]) From pypy.commits at gmail.com Sat Mar 4 16:48:45 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Mar 2017 13:48:45 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: tweaks for threads Message-ID: <58bb363d.12092e0a.69e5.0b85@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90545:b4fbb27942b9 Date: 2017-03-04 22:47 +0100 http://bitbucket.org/pypy/pypy/changeset/b4fbb27942b9/ Log: tweaks for threads diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -3,6 +3,7 @@ from rpython.rlib.debug import ll_assert from rpython.rlib.nonconst import NonConstant from rpython.rlib import rgc +from rpython.rlib.objectmodel import specialize from rpython.rtyper import rmodel from rpython.rtyper.annlowlevel import llhelper from rpython.rtyper.lltypesystem import lltype, llmemory @@ -57,6 +58,38 @@ return top self.decr_stack = decr_stack + @specialize.call_location() + def walk_stack_root(invoke, arg0, arg1, start, addr, is_minor): + skip = 0 + while addr != start: + addr -= sizeofaddr + #XXX reintroduce support for tagged values? + #if gc.points_to_valid_gc_object(addr): + # callback(gc, addr) + + if skip & 1 == 0: + content = addr.address[0] + n = llmemory.cast_adr_to_int(content) + if n & 1 == 0: + if content: # non-0, non-odd: a regular ptr + invoke(arg0, arg1, addr) + else: + # odd number: a skip bitmask + if n > 0: # initially, an unmarked value + if is_minor: + newcontent = llmemory.cast_int_to_adr(-n) + addr.address[0] = newcontent # mark + skip = n + else: + # a marked value + if is_minor: + return + skip = -n + skip >>= 1 + self.rootstackhook = walk_stack_root + self.invoke_collect_stack_root = specialize.call_location()( + lambda arg0, arg1, addr: arg0(self.gc, addr)) + self.shadow_stack_pool = ShadowStackPool(gcdata) rsd = gctransformer.root_stack_depth if rsd is not None: @@ -75,36 +108,10 @@ BaseRootWalker.setup_root_walker(self) def walk_stack_roots(self, collect_stack_root, is_minor=False): - gc = self.gc gcdata = self.gcdata - start = gcdata.root_stack_base - addr = gcdata.root_stack_top - skip = 0 - while addr != start: - addr -= sizeofaddr - #XXX reintroduce support for tagged values? - #if gc.points_to_valid_gc_object(addr): - # callback(gc, addr) - - if skip & 1 == 0: - content = addr.address[0] - n = llmemory.cast_adr_to_int(content) - if n & 1 == 0: - if content: # non-0, non-odd: a regular ptr - collect_stack_root(gc, addr) - else: - # odd number: a skip bitmask - if n > 0: # initially, an unmarked value - if is_minor: - newcontent = llmemory.cast_int_to_adr(-n) - addr.address[0] = newcontent # mark - skip = n - else: - # a marked value - if is_minor: - return - skip = -n - skip >>= 1 + self.rootstackhook(self.invoke_collect_stack_root, collect_stack_root, + None, gcdata.root_stack_base, gcdata.root_stack_top, + is_minor=is_minor) def need_thread_support(self, gctransformer, getfn): from rpython.rlib import rthread # xxx fish @@ -322,11 +329,9 @@ def customtrace(gc, obj, callback, arg): obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) - addr = obj.top - start = obj.base - while addr != start: - addr -= sizeofaddr - gc._trace_callback(callback, arg, addr) + root_walker.rootstackhook(gc._trace_callback, callback, arg, + obj.base, obj.top, + is_minor=False) # xxx optimize? gc = gctransformer.gcdata.gc assert not hasattr(gc, 'custom_trace_dispatcher') diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -450,7 +450,7 @@ # yield ('typedef struct { void %s; } pypy_ss_t;' % ', '.join(['*s%d' % i for i in range(numcolors)])) - yield 'pypy_ss_t *ss = (pypy_ss_t *)%s;' % gcpol_ss + yield 'pypy_ss_t *ss;' funcgen.gcpol_ss = gcpol_ss def OP_GC_PUSH_ROOTS(self, funcgen, op): @@ -460,7 +460,8 @@ raise Exception("gc_pop_roots should be removed by postprocess_graph") def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op): - return '%s = (void *)(ss+1);' % funcgen.gcpol_ss + return 'ss = (pypy_ss_t *)%s; %s = (void *)(ss+1);' % ( + funcgen.gcpol_ss, funcgen.gcpol_ss) def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op): return '%s = (void *)ss;' % funcgen.gcpol_ss From pypy.commits at gmail.com Sat Mar 4 17:02:39 2017 From: pypy.commits at gmail.com (fijal) Date: Sat, 04 Mar 2017 14:02:39 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: implement isspace as a regex Message-ID: <58bb397f.c59c190a.752b.0a63@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90546:e0e41208baf4 Date: 2017-03-04 22:55 +0100 http://bitbucket.org/pypy/pypy/changeset/e0e41208baf4/ Log: implement isspace as a regex diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -194,7 +194,7 @@ return unicodedb.iscased(ord(ch)) def _islinebreak(self, s, pos): - return rutf8.check_newline_utf8(s, pos) + return rutf8.islinebreak(s, pos) def _upper(self, ch): return unichr(unicodedb.toupper(ord(ch))) @@ -668,6 +668,25 @@ return W_UnicodeObject(value, self._len()) + def _strip_none(self, space, left, right): + "internal function called by str_xstrip methods" + value = self._utf8 + + lpos = 0 + rpos = self._len() + + if left: + while lpos < rpos and self._isspace(value[lpos]): + lpos += 1 + + if right: + while rpos > lpos and self._isspace(value[rpos - 1]): + rpos -= 1 + + assert rpos >= lpos # annotator hint, don't remove + return self._sliced(space, value, lpos, rpos, self) + + def descr_getnewargs(self, space): return space.newtuple([W_UnicodeObject(self._utf8, self._length)]) diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py --- a/rpython/rlib/rutf8.py +++ b/rpython/rlib/rutf8.py @@ -68,7 +68,7 @@ chr1 = ord(code[pos]) if chr1 < 0x80: return pos + 1 - if 0xC2 >= chr1 <= 0xDF: + if 0xC2 <= chr1 <= 0xDF: return pos + 2 if chr1 >= 0xE0 and chr1 <= 0xEF: return pos + 3 @@ -165,7 +165,7 @@ pos += 1 return result.build(), pos, -1 -def check_newline_utf8(s, pos): +def islinebreak(s, pos): chr1 = ord(s[pos]) if 0xa <= chr1 <= 0xd: return True @@ -182,6 +182,41 @@ return chr3 == 0xa8 or chr3 == 0xa9 return False +def isspace(s, pos): + chr1 = ord(s[pos]) + if (chr1 == ord(' ') or chr1 == ord('\n') or chr1 == ord('\t') or + chr1 == ord('\r')): + return True # common + if chr1 == 0x0b or chr1 == 0x0c or (chr1 >= 0x1c and chr1 <= 0x1f): + return True # less common + if chr1 < 0x80: + return False + # obscure cases + chr2 = ord(s[pos + 1]) + if chr1 == 0xc2: + return chr2 == 0x85 or chr2 == 0xa0 + if chr1 == 0xe2: + if chr2 == 0x81 and s[pos + 2] == '\x9f': + return True + if chr2 != 0x80: + return False + chr3 = ord(s[pos + 2]) + if chr3 >= 0x80 and chr3 <= 0x8a: + return True + if chr3 == 0xa9 or chr3 == 0xa8 or chr3 == 0xaf: + return True + return False + if chr1 == 0xe1: + chr3 = ord(s[pos + 2]) + if chr2 == 0x9a and chr3 == 0x80: + return True + if chr2 == 0xa0 and chr3 == 0x8e: + return True + return False + if chr1 == 0xe3 and chr2 == 0x80 and s[pos + 2] == '\x80': + return True + return False + class Utf8CheckError(Exception): def __init__(self, msg, startpos, endpos): self.msg = msg diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py --- a/rpython/rlib/test/test_rutf8.py +++ b/rpython/rlib/test/test_rutf8.py @@ -60,6 +60,13 @@ def test_check_newline_utf8(): for i in xrange(sys.maxunicode): if runicode.unicodedb.islinebreak(i): - assert rutf8.check_newline_utf8(unichr(i).encode('utf8'), 0) + assert rutf8.islinebreak(unichr(i).encode('utf8'), 0) else: - assert not rutf8.check_newline_utf8(unichr(i).encode('utf8'), 0) + assert not rutf8.islinebreak(unichr(i).encode('utf8'), 0) + +def test_isspace_utf8(): + for i in xrange(sys.maxunicode): + if runicode.unicodedb.isspace(i): + assert rutf8.isspace(unichr(i).encode('utf8'), 0) + else: + assert not rutf8.isspace(unichr(i).encode('utf8'), 0) From pypy.commits at gmail.com Sat Mar 4 17:02:41 2017 From: pypy.commits at gmail.com (fijal) Date: Sat, 04 Mar 2017 14:02:41 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: actually use the function I wrote Message-ID: <58bb3981.53532e0a.e0da2.0970@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90547:9f602db4b1ce Date: 2017-03-04 22:58 +0100 http://bitbucket.org/pypy/pypy/changeset/9f602db4b1ce/ Log: actually use the function I wrote diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -20,8 +20,7 @@ def _isspace(s, pos, isutf8=0): if isutf8: from rpython.rlib import rutf8 - char = rutf8.codepoint_at_pos(s, pos) - return unicodedb.isspace(char) + return rutf8.isspace(s, pos) else: char = s[pos] if isinstance(char, str): From pypy.commits at gmail.com Sat Mar 4 17:02:44 2017 From: pypy.commits at gmail.com (fijal) Date: Sat, 04 Mar 2017 14:02:44 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: kill pdb Message-ID: <58bb3984.4d29190a.1ff9b.0f1b@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90548:1239b2e8d564 Date: 2017-03-04 22:59 +0100 http://bitbucket.org/pypy/pypy/changeset/1239b2e8d564/ Log: kill pdb diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -125,8 +125,6 @@ return rutf8.compute_length_utf8(self._utf8) def _val(self, space): - import pdb - pdb.set_trace() return self._utf8.decode('utf8') @staticmethod From pypy.commits at gmail.com Sat Mar 4 22:59:30 2017 From: pypy.commits at gmail.com (fijal) Date: Sat, 04 Mar 2017 19:59:30 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: implement strip - fails on narrow unicode builds Message-ID: <58bb8d22.1099190a.787fd.1fa1@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90549:bd07f3fee9c7 Date: 2017-03-04 23:57 +0100 http://bitbucket.org/pypy/pypy/changeset/bd07f3fee9c7/ Log: implement strip - fails on narrow unicode builds diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -125,6 +125,8 @@ return rutf8.compute_length_utf8(self._utf8) def _val(self, space): + #import pdb + #pdb.set_trace() return self._utf8.decode('utf8') @staticmethod @@ -511,7 +513,8 @@ if keepends: eol = pos lgt += 1 - strs_w.append(W_UnicodeObject(value[sol:eol], lgt)) + # XXX find out why lgt calculation is off + strs_w.append(W_UnicodeObject(value[sol:eol], -1)) return space.newlist(strs_w) @unwrap_spec(width=int) @@ -666,24 +669,58 @@ return W_UnicodeObject(value, self._len()) + def _utf8_sliced(self, start, stop, lgt): + assert start >= 0 + assert stop >= 0 + #if start == 0 and stop == len(s) and space.is_w(space.type(orig_obj), + # space.w_bytes): + # return orig_obj + return W_UnicodeObject(self._utf8[start:stop], lgt) + def _strip_none(self, space, left, right): "internal function called by str_xstrip methods" value = self._utf8 lpos = 0 - rpos = self._len() + rpos = len(value) + lgt = self._len() if left: - while lpos < rpos and self._isspace(value[lpos]): - lpos += 1 + while lpos < rpos and rutf8.isspace(value, lpos): + lpos = rutf8.next_codepoint_pos(value, lpos) + lgt -= 1 if right: - while rpos > lpos and self._isspace(value[rpos - 1]): - rpos -= 1 + while rpos > lpos and rutf8.isspace(value, + rutf8.prev_codepoint_pos(value, rpos)): + rpos = rutf8.prev_codepoint_pos(value, rpos) + lgt -= 1 assert rpos >= lpos # annotator hint, don't remove - return self._sliced(space, value, lpos, rpos, self) + return self._utf8_sliced(lpos, rpos, lgt) + def _strip(self, space, w_chars, left, right, name='strip'): + "internal function called by str_xstrip methods" + value = self._utf8 + chars = self.convert_arg_to_w_unicode(space, w_chars, strict=name)._utf8 + + lpos = 0 + rpos = len(value) + lgt = self._len() + + if left: + while lpos < rpos and rutf8.utf8_in_chars(value, lpos, chars): + lpos = rutf8.next_codepoint_pos(value, lpos) + lgt -= 1 + + if right: + while rpos > lpos and rutf8.utf8_in_chars(value, + rutf8.prev_codepoint_pos(value, rpos), chars): + rpos = rutf8.prev_codepoint_pos(value, rpos) + lgt -= 1 + + assert rpos >= lpos # annotator hint, don't remove + return self._utf8_sliced(lpos, rpos, lgt) def descr_getnewargs(self, space): return space.newtuple([W_UnicodeObject(self._utf8, self._length)]) diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py --- a/rpython/rlib/rutf8.py +++ b/rpython/rlib/rutf8.py @@ -217,6 +217,24 @@ return True return False +def utf8_in_chars(value, pos, chars): + """ equivalent of u'x' in u'xyz', just done in utf8 + """ + lgt = next_codepoint_pos(value, pos) - pos + i = 0 + while i < len(chars): + j = next_codepoint_pos(chars, i) + if j - i != lgt: + i = j + continue + for k in range(lgt): + if value[k + pos] != chars[i + k]: + break + else: + return True + i = j + return False + class Utf8CheckError(Exception): def __init__(self, msg, startpos, endpos): self.msg = msg diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py --- a/rpython/rlib/test/test_rutf8.py +++ b/rpython/rlib/test/test_rutf8.py @@ -70,3 +70,15 @@ assert rutf8.isspace(unichr(i).encode('utf8'), 0) else: assert not rutf8.isspace(unichr(i).encode('utf8'), 0) + + at given(strategies.integers(min_value=0, max_value=sys.maxunicode), + strategies.characters()) +def test_utf8_in_chars(i, uni): + if not uni: + return + if unichr(i) in uni: + response = True + else: + response = False + r = rutf8.utf8_in_chars(unichr(i).encode('utf8'), 0, uni.encode('utf8')) + assert r == response From pypy.commits at gmail.com Sun Mar 5 02:45:01 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Mar 2017 23:45:01 -0800 (PST) Subject: [pypy-commit] pypy default: Fix test (mostly, but I'm not sure I understand why it doesn't seem to Message-ID: <58bbc1fd.09512e0a.56eb0.1989@mx.google.com> Author: Armin Rigo Branch: Changeset: r90550:1a55bff39061 Date: 2017-03-05 08:44 +0100 http://bitbucket.org/pypy/pypy/changeset/1a55bff39061/ Log: Fix test (mostly, but I'm not sure I understand why it doesn't seem to crash in a full pypy) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -773,24 +773,33 @@ from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound,\ IntLowerBound - if mode is None: + length = self.getstrlen1(mode) + if length < 0: # XXX we can do better if we know it's an array return IntLowerBound(0) - else: - return ConstIntBound(self.getstrlen1(mode)) + return ConstIntBound(length) def getstrlen(self, op, string_optimizer, mode): - return ConstInt(self.getstrlen1(mode)) + length = self.getstrlen1(mode) + if length < 0: + return None + return ConstInt(length) def getstrlen1(self, mode): from rpython.jit.metainterp.optimizeopt import vstring - + if mode is vstring.mode_string: s = self._unpack_str(vstring.mode_string) + if s is None: + return -1 + return len(s) + elif mode is vstring.mode_unicode: + s = self._unpack_str(vstring.mode_unicode) + if s is None: + return -1 return len(s) else: - s = self._unpack_str(vstring.mode_unicode) - return len(s) + return -1 def getstrhash(self, op, mode): from rpython.jit.metainterp.optimizeopt import vstring From pypy.commits at gmail.com Sun Mar 5 02:45:59 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Mar 2017 23:45:59 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: hg merge default Message-ID: <58bbc237.8fdb190a.35b46.162b@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90551:d128eb23ffe6 Date: 2017-03-05 07:45 +0000 http://bitbucket.org/pypy/pypy/changeset/d128eb23ffe6/ Log: hg merge default diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -773,24 +773,33 @@ from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound,\ IntLowerBound - if mode is None: + length = self.getstrlen1(mode) + if length < 0: # XXX we can do better if we know it's an array return IntLowerBound(0) - else: - return ConstIntBound(self.getstrlen1(mode)) + return ConstIntBound(length) def getstrlen(self, op, string_optimizer, mode): - return ConstInt(self.getstrlen1(mode)) + length = self.getstrlen1(mode) + if length < 0: + return None + return ConstInt(length) def getstrlen1(self, mode): from rpython.jit.metainterp.optimizeopt import vstring - + if mode is vstring.mode_string: s = self._unpack_str(vstring.mode_string) + if s is None: + return -1 + return len(s) + elif mode is vstring.mode_unicode: + s = self._unpack_str(vstring.mode_unicode) + if s is None: + return -1 return len(s) else: - s = self._unpack_str(vstring.mode_unicode) - return len(s) + return -1 def getstrhash(self, op, mode): from rpython.jit.metainterp.optimizeopt import vstring From pypy.commits at gmail.com Sun Mar 5 02:56:37 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 04 Mar 2017 23:56:37 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: minimal support in llinterp Message-ID: <58bbc4b5.0163190a.399bf.1c0e@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90552:05ad3cfbbde7 Date: 2017-03-05 08:56 +0100 http://bitbucket.org/pypy/pypy/changeset/05ad3cfbbde7/ Log: minimal support in llinterp diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -1086,6 +1086,19 @@ def op_track_alloc_stop(self, addr): checkadr(addr) + def op_gc_enter_roots_frame(self, gcdata, numcolors): + assert not hasattr(self, '_inside_roots_frame') + self._inside_roots_frame = numcolors + + def op_gc_leave_roots_frame(self): + del self._inside_roots_frame + + def op_gc_save_root(self, num, value): + assert 0 <= num < self._inside_roots_frame + + def op_gc_restore_root(self, num, value): + assert 0 <= num < self._inside_roots_frame + # ____________________________________________________________ # Overflow-detecting variants From pypy.commits at gmail.com Sun Mar 5 03:19:58 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 00:19:58 -0800 (PST) Subject: [pypy-commit] pypy default: document branches Message-ID: <58bbca2e.13542e0a.69d3d.1d9c@mx.google.com> Author: Armin Rigo Branch: Changeset: r90553:b97fcf9f65b4 Date: 2017-03-05 09:19 +0100 http://bitbucket.org/pypy/pypy/changeset/b97fcf9f65b4/ Log: document branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -169,3 +169,14 @@ Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews keeping objects alive forever (missing decref), and make sure that bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. + +.. branch: fix-global + +Fix bug (bad reported info) when asked to translate SyntaxWarning to +SyntaxError. + +.. branch: optinfo-into-bridges-3 + +Improve the optimization of branchy Python code by retaining more +information across failing guards. This is done by appending some +carefully encoded extra information into the resume code. From pypy.commits at gmail.com Sun Mar 5 03:51:16 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 00:51:16 -0800 (PST) Subject: [pypy-commit] pypy default: Various new tests. Fix for test_readinto_small_parts. Message-ID: <58bbd184.09532e0a.d4f30.1bde@mx.google.com> Author: Armin Rigo Branch: Changeset: r90554:c415b3b5820c Date: 2017-03-05 09:50 +0100 http://bitbucket.org/pypy/pypy/changeset/c415b3b5820c/ Log: Various new tests. Fix for test_readinto_small_parts. diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -16,6 +16,7 @@ check_readable_w, check_writable_w, check_seekable_w) from pypy.module._io.interp_io import W_BlockingIOError from rpython.rlib import rthread +from rpython.rtyper.lltypesystem import rffi STATE_ZERO, STATE_OK, STATE_DETACHED = range(3) @@ -123,7 +124,8 @@ self.buf[self.start + index] = char def get_raw_address(self): - return nonmoving_raw_ptr_for_resizable_list(self.buf) + ptr = nonmoving_raw_ptr_for_resizable_list(self.buf) + return rffi.ptradd(ptr, self.start) class BufferedMixin: _mixin_ = True diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -12,6 +12,9 @@ tmpfile = udir.join('tmpfile') tmpfile.write("a\nb\nc", mode='wb') cls.w_tmpfile = cls.space.wrap(str(tmpfile)) + bigtmpfile = udir.join('bigtmpfile') + bigtmpfile.write("a\nb\nc" * 20, mode='wb') + cls.w_bigtmpfile = cls.space.wrap(str(bigtmpfile)) def test_simple_read(self): import _io @@ -135,10 +138,14 @@ def test_readinto(self): import _io - a = bytearray('x' * 10) + a1 = bytearray('x') + a = bytearray('x' * 9) raw = _io.FileIO(self.tmpfile) f = _io.BufferedReader(raw) - assert f.readinto(a) == 5 + assert f.readinto(a1) == 1 + assert a1 == 'a' + assert f.readinto(a) == 4 + assert a == '\nb\ncxxxxx' exc = raises(TypeError, f.readinto, u"hello") assert str(exc.value) == "cannot use unicode as modifiable buffer" exc = raises(TypeError, f.readinto, buffer(b"hello")) @@ -148,7 +155,17 @@ exc = raises(TypeError, f.readinto, memoryview(b"hello")) assert str(exc.value) == "must be read-write buffer, not memoryview" f.close() - assert a == 'a\nb\ncxxxxx' + + def test_readinto_big(self): + import _io + a1 = bytearray('x') + a = bytearray('x' * 199) + raw = _io.FileIO(self.bigtmpfile) + f = _io.BufferedReader(raw) + assert f.readinto(a1) == 1 + assert a1 == 'a' + assert f.readinto(a) == 99 + assert a == '\nb\nc' + 'a\nb\nc' * 19 + 'x' * 100 def test_seek(self): import _io @@ -237,7 +254,21 @@ assert rawio.count == 4 class AppTestBufferedReaderWithThreads(AppTestBufferedReader): - spaceconfig = dict(usemodules=['_io', 'thread']) + spaceconfig = dict(usemodules=['_io', 'thread', 'time']) + + def test_readinto_small_parts(self): + import _io, os, thread, time + read_fd, write_fd = os.pipe() + raw = _io.FileIO(read_fd) + f = _io.BufferedReader(raw) + a = bytearray(b'x' * 10) + os.write(write_fd, b"abcde") + def write_more(): + time.sleep(0.5) + os.write(write_fd, b"fghij") + thread.start_new_thread(write_more, ()) + assert f.readinto(a) == 10 + assert a == 'abcdefghij' class AppTestBufferedWriter: From pypy.commits at gmail.com Sun Mar 5 04:01:58 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 01:01:58 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Use rffi.getintfield(). Fixes one translation error Message-ID: <58bbd406.42452e0a.a9d96.1a63@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90555:f72cc14bf6c6 Date: 2017-03-05 09:58 +0100 http://bitbucket.org/pypy/pypy/changeset/f72cc14bf6c6/ Log: Use rffi.getintfield(). Fixes one translation error diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -103,19 +103,19 @@ return py_obj.c_state def get_kind(py_obj): - return get_state(py_obj).c_kind + return rffi.getintfield(get_state(py_obj), 'c_kind') def set_kind(py_obj, value): get_state(py_obj).c_kind = cts.cast('unsigned int', value) def get_ascii(py_obj): - return get_state(py_obj).c_ascii + return rffi.getintfield(get_state(py_obj), 'c_ascii') def set_ascii(py_obj, value): get_state(py_obj).c_ascii = cts.cast('unsigned int', value) def get_ready(py_obj): - return get_state(py_obj).c_ready + return rffi.getintfield(get_state(py_obj), 'c_ready') def set_ready(py_obj, value): get_state(py_obj).c_ready = cts.cast('unsigned int', value) @@ -262,7 +262,7 @@ def _PyUnicode_Ready(space, w_obj): assert isinstance(w_obj, unicodeobject.W_UnicodeObject) py_obj = as_pyobj(space, w_obj) - assert widen(get_kind(py_obj)) == WCHAR_KIND + assert get_kind(py_obj) == WCHAR_KIND maxchar = 0 for c in w_obj._value: if ord(c) > maxchar: From pypy.commits at gmail.com Sun Mar 5 04:21:58 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 01:21:58 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: fix Message-ID: <58bbd8b6.5d542e0a.8ed0b.1b5b@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90556:89ea2cfa36b7 Date: 2017-03-05 10:19 +0100 http://bitbucket.org/pypy/pypy/changeset/89ea2cfa36b7/ Log: fix diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -142,6 +142,9 @@ any_inlining = self.inline and self.inline_helpers_into(graph) self.postprocess_graph(graph, any_inlining) + def postprocess_graph(self, graph, any_inlining): + pass + def compute_borrowed_vars(self, graph): # the input args are borrowed, and stay borrowed for as long as they # are not merged with other values. From pypy.commits at gmail.com Sun Mar 5 04:50:55 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 01:50:55 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: Properly emulate gc_enter_roots_frame() & co. inside the llinterp. Message-ID: <58bbdf7f.02a5190a.502c7.208b@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90557:39add7b84e2a Date: 2017-03-05 10:50 +0100 http://bitbucket.org/pypy/pypy/changeset/39add7b84e2a/ Log: Properly emulate gc_enter_roots_frame() & co. inside the llinterp. diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -1087,17 +1087,57 @@ checkadr(addr) def op_gc_enter_roots_frame(self, gcdata, numcolors): + """Fetch from the gcdata the current root_stack_top; bump it + by 'numcolors'; and assert that the new area is fully + uninitialized so far. + """ assert not hasattr(self, '_inside_roots_frame') - self._inside_roots_frame = numcolors + p = gcdata.inst_root_stack_top.ptr + q = lltype.direct_ptradd(p, numcolors) + self._inside_roots_frame = (p, q, numcolors, gcdata) + gcdata.inst_root_stack_top = llmemory.cast_ptr_to_adr(q) + # + array = p._obj._parentstructure() + index = p._obj._parent_index + for i in range(index, index + numcolors): + assert isinstance(array.getitem(i), lltype._uninitialized) def op_gc_leave_roots_frame(self): + """Cancel gc_enter_roots_frame() by removing the frame from + the root_stack_top. Writes uninitialized entries in its old place. + """ + (p, q, numcolors, gcdata) = self._inside_roots_frame + assert gcdata.inst_root_stack_top.ptr == q + gcdata.inst_root_stack_top = llmemory.cast_ptr_to_adr(p) del self._inside_roots_frame + # + array = p._obj._parentstructure() + index = p._obj._parent_index + for i in range(index, index + numcolors): + array.setitem(i, lltype._uninitialized(llmemory.Address)) def op_gc_save_root(self, num, value): - assert 0 <= num < self._inside_roots_frame + """Save one value (int or ptr) into the frame.""" + (p, q, numcolors, gcdata) = self._inside_roots_frame + assert 0 <= num < numcolors + if isinstance(value, int): + assert value & 1 # must be odd + v = llmemory.cast_int_to_adr(value) + else: + v = llmemory.cast_ptr_to_adr(value) + llmemory.cast_ptr_to_adr(p).address[num] = v - def op_gc_restore_root(self, num, value): - assert 0 <= num < self._inside_roots_frame + def op_gc_restore_root(self, c_num, v_value): + """Restore one value from the frame.""" + num = c_num.value + (p, q, numcolors, gcdata) = self._inside_roots_frame + assert 0 <= num < numcolors + assert isinstance(v_value.concretetype, lltype.Ptr) + assert v_value.concretetype.TO._gckind == 'gc' + newvalue = llmemory.cast_ptr_to_adr(p).address[num] + newvalue = llmemory.cast_adr_to_ptr(newvalue, v_value.concretetype) + self.setvar(v_value, newvalue) + op_gc_restore_root.specialform = True # ____________________________________________________________ # Overflow-detecting variants From pypy.commits at gmail.com Sun Mar 5 04:56:13 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 01:56:13 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: more fixes Message-ID: <58bbe0bd.d3d3190a.366be.e964@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90558:8479fcf141ae Date: 2017-03-05 10:55 +0100 http://bitbucket.org/pypy/pypy/changeset/8479fcf141ae/ Log: more fixes diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -341,6 +341,9 @@ # called first, to initialize self.belongs_to_current_thread. assert not hasattr(self, 'gc_detach_callback_pieces_ptr') + def postprocess_graph(self, gct, graph, any_inlining): + pass + def walk_stack_roots(self, collect_stack_root, is_minor=False): gcdata = self.gcdata gcdata._gc_collect_stack_root = collect_stack_root diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -1139,6 +1139,12 @@ self.setvar(v_value, newvalue) op_gc_restore_root.specialform = True + def op_gc_push_roots(self, *args): + raise NotImplementedError + + def op_gc_pop_roots(self, *args): + raise NotImplementedError + # ____________________________________________________________ # Overflow-detecting variants From pypy.commits at gmail.com Sun Mar 5 05:18:45 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 02:18:45 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: fix Message-ID: <58bbe605.0d50190a.d3d5c.226f@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90559:33d6090edba2 Date: 2017-03-05 11:18 +0100 http://bitbucket.org/pypy/pypy/changeset/33d6090edba2/ Log: fix diff --git a/rpython/memory/gctransform/shadowcolor.py b/rpython/memory/gctransform/shadowcolor.py --- a/rpython/memory/gctransform/shadowcolor.py +++ b/rpython/memory/gctransform/shadowcolor.py @@ -795,6 +795,8 @@ for link in block.exits: track_next = [] for v in track_args: + if not isinstance(v, Variable): + continue i = link.args.index(v) # should really be here w = link.target.inputargs[i] track_next.append(w) From pypy.commits at gmail.com Sun Mar 5 05:37:06 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 02:37:06 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: call postprocess_graph() even if gctransformer.inline is false Message-ID: <58bbea52.690f190a.6f432.3689@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90560:0ff4c0371367 Date: 2017-03-05 11:36 +0100 http://bitbucket.org/pypy/pypy/changeset/0ff4c0371367/ Log: call postprocess_graph() even if gctransformer.inline is false diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -137,7 +137,7 @@ constant_fold_graph(graph) return any_inlining - def inline_helpers(self, graphs): + def inline_helpers_and_postprocess(self, graphs): for graph in graphs: any_inlining = self.inline and self.inline_helpers_into(graph) self.postprocess_graph(graph, any_inlining) diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -346,9 +346,9 @@ assert not self.delayedfunctionptrs self.completed = True - if self.gctransformer is not None and self.gctransformer.inline: + if self.gctransformer is not None: log.database("Inlining GC helpers and postprocessing") - self.gctransformer.inline_helpers(self.all_graphs()) + self.gctransformer.inline_helpers_and_postprocess(self.all_graphs()) if show_progress: dump() log.database("Completed") From pypy.commits at gmail.com Sun Mar 5 05:37:08 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 02:37:08 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: Do not add '-flto' if we're translating with asmgcc Message-ID: <58bbea54.1459190a.c46bd.1efb@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90561:88bd58608e5e Date: 2017-03-05 11:36 +0100 http://bitbucket.org/pypy/pypy/changeset/88bd58608e5e/ Log: Do not add '-flto' if we're translating with asmgcc diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -418,7 +418,7 @@ headers_to_precompile=headers_to_precompile, no_precompile_cfiles = module_files, shared=self.config.translation.shared, - icon=self.config.translation.icon) + config=self.config) if self.has_profopt(): profopt = self.config.translation.profopt diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py --- a/rpython/translator/platform/__init__.py +++ b/rpython/translator/platform/__init__.py @@ -102,7 +102,7 @@ def gen_makefile(self, cfiles, eci, exe_name=None, path=None, shared=False, headers_to_precompile=[], - no_precompile_cfiles = [], icon=None): + no_precompile_cfiles = [], config=None): raise NotImplementedError("Pure abstract baseclass") def __repr__(self): diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -93,7 +93,7 @@ def gen_makefile(self, cfiles, eci, exe_name=None, path=None, shared=False, headers_to_precompile=[], - no_precompile_cfiles = [], icon=None): + no_precompile_cfiles = [], config=None): # ensure frameworks are passed in the Makefile fs = self._frameworks(eci.frameworks) if len(fs) > 0: @@ -103,7 +103,7 @@ shared=shared, headers_to_precompile=headers_to_precompile, no_precompile_cfiles = no_precompile_cfiles, - icon=icon) + config=config) return mk class Darwin_PowerPC(Darwin):#xxx fixme, mwp diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py --- a/rpython/translator/platform/linux.py +++ b/rpython/translator/platform/linux.py @@ -13,7 +13,7 @@ + os.environ.get('LDFLAGS', '').split()) extra_libs = ('-lrt',) cflags = tuple( - ['-O3', '-pthread', '-fomit-frame-pointer', '-flto', + ['-O3', '-pthread', '-fomit-frame-pointer', '-Wall', '-Wno-unused'] + os.environ.get('CFLAGS', '').split()) standalone_only = () diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -102,7 +102,7 @@ def gen_makefile(self, cfiles, eci, exe_name=None, path=None, shared=False, headers_to_precompile=[], - no_precompile_cfiles = [], icon=None): + no_precompile_cfiles = [], config=None): cfiles = self._all_cfiles(cfiles, eci) if path is None: @@ -132,6 +132,10 @@ else: cflags = tuple(self.cflags) + tuple(self.standalone_only) + # xxx check which compilers accept this option or not + if not config or config.translation.gcrootfinder != 'asmgcc': + cflags = ('-flto',) + cflags + m = GnuMakefile(path) m.exe_name = path.join(exe_name.basename) m.eci = eci diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -271,7 +271,7 @@ def gen_makefile(self, cfiles, eci, exe_name=None, path=None, shared=False, headers_to_precompile=[], - no_precompile_cfiles = [], icon=None): + no_precompile_cfiles = [], config=None): cfiles = self._all_cfiles(cfiles, eci) if path is None: @@ -392,6 +392,7 @@ '/Fo$@ /c $< $(INCLUDEDIRS)')) + icon = config.translation.icon if config else None if icon: shutil.copyfile(icon, str(path.join('icon.ico'))) rc_file = path.join('icon.rc') From pypy.commits at gmail.com Sun Mar 5 06:20:09 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 03:20:09 -0800 (PST) Subject: [pypy-commit] pypy default: Basics of sendfile() Message-ID: <58bbf469.97e1190a.47c64.210e@mx.google.com> Author: Armin Rigo Branch: Changeset: r90562:c979853a3b6a Date: 2017-03-05 12:19 +0100 http://bitbucket.org/pypy/pypy/changeset/c979853a3b6a/ Log: Basics of sendfile() diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -254,6 +254,7 @@ PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP') PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER') O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK') + OFF_T = rffi_platform.SimpleType('off_t') OFF_T_SIZE = rffi_platform.SizeOf('off_t') HAVE_UTIMES = rffi_platform.Has('utimes') @@ -2427,3 +2428,21 @@ def set_status_flags(fd, flags): res = c_set_status_flags(fd, flags) handle_posix_error('set_status_flags', res) + +if not _WIN32: + sendfile_eci = ExternalCompilationInfo(includes=["sys/sendfile.h"]) + _OFF_PTR_T = rffi.CArrayPtr(OFF_T) + c_sendfile = rffi.llexternal('sendfile', + [rffi.INT, rffi.INT, _OFF_PTR_T, rffi.SIZE_T], + rffi.SSIZE_T, compilation_info=sendfile_eci) + + def sendfile(out_fd, in_fd, offset, count): + with lltype.scoped_alloc(_OFF_PTR_T.TO, 1) as p_offset: + p_offset[0] = rffi.cast(OFF_T, offset) + res = c_sendfile(out_fd, in_fd, p_offset, count) + return handle_posix_error('sendfile', res) + + def sendfile_no_offset(out_fd, in_fd, count): + """Passes offset==NULL; not support on all OSes""" + res = c_sendfile(out_fd, in_fd, lltype.nullptr(_OFF_PTR_T.TO), count) + return handle_posix_error('sendfile', res) diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -676,3 +676,36 @@ prio = rposix.getpriority(rposix.PRIO_PROCESS, 0) rposix.setpriority(rposix.PRIO_PROCESS, 0, prio) py.test.raises(OSError, rposix.getpriority, rposix.PRIO_PGRP, 123456789) + +if sys.platform != 'win32': + def test_sendfile(): + from rpython.rlib import rsocket + s1, s2 = rsocket.socketpair() + relpath = 'test_sendfile' + filename = str(udir.join(relpath)) + fd = os.open(filename, os.O_RDWR|os.O_CREAT, 0777) + os.write(fd, 'abcdefghij') + res = rposix.sendfile(s1.fd, fd, 3, 5) + assert res == 5 + data = os.read(s2.fd, 10) + assert data == 'defgh' + os.close(fd) + s2.close() + s1.close() + +if sys.platform.startswith('linux'): + def test_sendfile_no_offset(): + from rpython.rlib import rsocket + s1, s2 = rsocket.socketpair() + relpath = 'test_sendfile' + filename = str(udir.join(relpath)) + fd = os.open(filename, os.O_RDWR|os.O_CREAT, 0777) + os.write(fd, 'abcdefghij') + os.lseek(fd, 3, 0) + res = rposix.sendfile_no_offset(s1.fd, fd, 5) + assert res == 5 + data = os.read(s2.fd, 10) + assert data == 'defgh' + os.close(fd) + s2.close() + s1.close() From pypy.commits at gmail.com Sun Mar 5 06:22:55 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 03:22:55 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58bbf50f.4d29190a.1ff9b.2398@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90563:e26d7faea9bc Date: 2017-03-05 12:22 +0100 http://bitbucket.org/pypy/pypy/changeset/e26d7faea9bc/ Log: hg merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -182,3 +182,14 @@ Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews keeping objects alive forever (missing decref), and make sure that bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. + +.. branch: fix-global + +Fix bug (bad reported info) when asked to translate SyntaxWarning to +SyntaxError. + +.. branch: optinfo-into-bridges-3 + +Improve the optimization of branchy Python code by retaining more +information across failing guards. This is done by appending some +carefully encoded extra information into the resume code. diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -15,7 +15,7 @@ W_IOBase, DEFAULT_BUFFER_SIZE, convert_size, trap_eintr, check_readable_w, check_writable_w, check_seekable_w) from rpython.rlib import rthread -from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list +from rpython.rtyper.lltypesystem import rffi STATE_ZERO, STATE_OK, STATE_DETACHED = range(3) @@ -174,7 +174,8 @@ self.buf[index] = char def get_raw_address(self): - return nonmoving_raw_ptr_for_resizable_list(self.buf) + ptr = nonmoving_raw_ptr_for_resizable_list(self.buf) + return rffi.ptradd(ptr, self.start) class BufferedMixin: _mixin_ = True diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -1,10 +1,10 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import ( - OperationError, oefmt, wrap_oserror, wrap_oserror2, exception_from_errno) + OperationError, oefmt, wrap_oserror, wrap_oserror2) from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_longlong -from rpython.rlib.rposix import get_saved_errno +from rpython.rlib.rposix import c_read, get_saved_errno from rpython.rlib.rstring import StringBuilder from rpython.rlib import rposix from rpython.rlib.rposix_stat import STAT_FIELD_TYPES @@ -130,14 +130,6 @@ return currentsize + SMALLCHUNK -_WIN32 = sys.platform.startswith('win') -UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' - -os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read', - [rffi.INT, rffi.CCHARP, rffi.SIZE_T], - rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) - - class W_FileIO(W_RawIOBase): def __init__(self, space): W_RawIOBase.__init__(self, space) @@ -485,7 +477,7 @@ # optimized case: reading more than 64 bytes into a rwbuffer # with a valid raw address # XXX TODO(mjacob): implement PEP 475 here! - got = os_read(self.fd, target_address, length) + got = c_read(self.fd, target_address, length) keepalive_until_here(rwbuffer) got = rffi.cast(lltype.Signed, got) if got >= 0: @@ -494,7 +486,8 @@ err = get_saved_errno() if err == errno.EAGAIN: return space.w_None - raise exception_from_errno(space, space.w_IOError, err) + e = OSError(err, "read failed") + raise wrap_oserror(space, e, exception_name='w_IOError') def readall_w(self, space): self._check_closed(space) diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -12,6 +12,9 @@ tmpfile = udir.join('tmpfile') tmpfile.write("a\nb\nc", mode='wb') cls.w_tmpfile = cls.space.wrap(str(tmpfile)) + bigtmpfile = udir.join('bigtmpfile') + bigtmpfile.write("a\nb\nc" * 20, mode='wb') + cls.w_bigtmpfile = cls.space.wrap(str(bigtmpfile)) def test_simple_read(self): import _io @@ -287,7 +290,21 @@ raises(_io.UnsupportedOperation, bufio.tell) class AppTestBufferedReaderWithThreads(AppTestBufferedReader): - spaceconfig = dict(usemodules=['_io', 'thread']) + spaceconfig = dict(usemodules=['_io', 'thread', 'time']) + + def test_readinto_small_parts(self): + import _io, os, thread, time + read_fd, write_fd = os.pipe() + raw = _io.FileIO(read_fd) + f = _io.BufferedReader(raw) + a = bytearray(b'x' * 10) + os.write(write_fd, b"abcde") + def write_more(): + time.sleep(0.5) + os.write(write_fd, b"fghij") + thread.start_new_thread(write_more, ()) + assert f.readinto(a) == 10 + assert a == 'abcdefghij' class AppTestBufferedWriter: diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -773,24 +773,33 @@ from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound,\ IntLowerBound - if mode is None: + length = self.getstrlen1(mode) + if length < 0: # XXX we can do better if we know it's an array return IntLowerBound(0) - else: - return ConstIntBound(self.getstrlen1(mode)) + return ConstIntBound(length) def getstrlen(self, op, string_optimizer, mode): - return ConstInt(self.getstrlen1(mode)) + length = self.getstrlen1(mode) + if length < 0: + return None + return ConstInt(length) def getstrlen1(self, mode): from rpython.jit.metainterp.optimizeopt import vstring - + if mode is vstring.mode_string: s = self._unpack_str(vstring.mode_string) + if s is None: + return -1 + return len(s) + elif mode is vstring.mode_unicode: + s = self._unpack_str(vstring.mode_unicode) + if s is None: + return -1 return len(s) else: - s = self._unpack_str(vstring.mode_unicode) - return len(s) + return -1 def getstrhash(self, op, mode): from rpython.jit.metainterp.optimizeopt import vstring diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -254,6 +254,7 @@ PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP') PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER') O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK') + OFF_T = rffi_platform.SimpleType('off_t') OFF_T_SIZE = rffi_platform.SizeOf('off_t') HAVE_UTIMES = rffi_platform.Has('utimes') @@ -2427,3 +2428,21 @@ def set_status_flags(fd, flags): res = c_set_status_flags(fd, flags) handle_posix_error('set_status_flags', res) + +if not _WIN32: + sendfile_eci = ExternalCompilationInfo(includes=["sys/sendfile.h"]) + _OFF_PTR_T = rffi.CArrayPtr(OFF_T) + c_sendfile = rffi.llexternal('sendfile', + [rffi.INT, rffi.INT, _OFF_PTR_T, rffi.SIZE_T], + rffi.SSIZE_T, compilation_info=sendfile_eci) + + def sendfile(out_fd, in_fd, offset, count): + with lltype.scoped_alloc(_OFF_PTR_T.TO, 1) as p_offset: + p_offset[0] = rffi.cast(OFF_T, offset) + res = c_sendfile(out_fd, in_fd, p_offset, count) + return handle_posix_error('sendfile', res) + + def sendfile_no_offset(out_fd, in_fd, count): + """Passes offset==NULL; not support on all OSes""" + res = c_sendfile(out_fd, in_fd, lltype.nullptr(_OFF_PTR_T.TO), count) + return handle_posix_error('sendfile', res) diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -676,3 +676,36 @@ prio = rposix.getpriority(rposix.PRIO_PROCESS, 0) rposix.setpriority(rposix.PRIO_PROCESS, 0, prio) py.test.raises(OSError, rposix.getpriority, rposix.PRIO_PGRP, 123456789) + +if sys.platform != 'win32': + def test_sendfile(): + from rpython.rlib import rsocket + s1, s2 = rsocket.socketpair() + relpath = 'test_sendfile' + filename = str(udir.join(relpath)) + fd = os.open(filename, os.O_RDWR|os.O_CREAT, 0777) + os.write(fd, 'abcdefghij') + res = rposix.sendfile(s1.fd, fd, 3, 5) + assert res == 5 + data = os.read(s2.fd, 10) + assert data == 'defgh' + os.close(fd) + s2.close() + s1.close() + +if sys.platform.startswith('linux'): + def test_sendfile_no_offset(): + from rpython.rlib import rsocket + s1, s2 = rsocket.socketpair() + relpath = 'test_sendfile' + filename = str(udir.join(relpath)) + fd = os.open(filename, os.O_RDWR|os.O_CREAT, 0777) + os.write(fd, 'abcdefghij') + os.lseek(fd, 3, 0) + res = rposix.sendfile_no_offset(s1.fd, fd, 5) + assert res == 5 + data = os.read(s2.fd, 10) + assert data == 'defgh' + os.close(fd) + s2.close() + s1.close() From pypy.commits at gmail.com Sun Mar 5 06:54:53 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 03:54:53 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Implement sendfile() in the posix module. (Not implemented so far: the Message-ID: <58bbfc8d.90202e0a.ae30f.2d60@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90564:33eb17e3bdc3 Date: 2017-03-05 12:54 +0100 http://bitbucket.org/pypy/pypy/changeset/33eb17e3bdc3/ Log: Implement sendfile() in the posix module. (Not implemented so far: the extra BSD or OS/X arguments) diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -213,6 +213,9 @@ assert getattr(rposix, _name) is not None, "missing %r" % (_name,) interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name) + if hasattr(rposix, 'sendfile'): + interpleveldefs['sendfile'] = 'interp_posix.sendfile' + for _name in ["O_CLOEXEC"]: if getattr(rposix, _name) is not None: interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name) diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -24,6 +24,7 @@ _WIN32 = sys.platform == 'win32' +_LINUX = sys.platform.startswith('linux') if _WIN32: from rpython.rlib import rwin32 @@ -2346,3 +2347,30 @@ rposix.set_status_flags(fd, flags) except OSError as e: raise wrap_oserror(space, e, eintr_retry=False) + + at unwrap_spec(out=c_int, in_=c_int, count=int) +def sendfile(space, out, in_, w_offset, count): + # NB. argument name difference with CPython: "in" versus "in_". + # According to a comment in posixmodule.c, the authors are aware that + # "in" is a Python keyword and so the argument name is bogus, but don't + # do anything about it anyway. PyPy calls it "in_" instead because + # "in" would require all sorts of special cases. I hope nobody + # relies on the strange name of CPython. + + # XXX only supports the common arguments for now (BSD takes more) + if _LINUX and space.is_none(w_offset): + while True: + try: + res = rposix.sendfile_no_offset(out, in_, count) + break + except OSError as e: + wrap_oserror(space, e, eintr_retry=True) + else: + offset = space.gateway_r_longlong_w(w_offset) + while True: + try: + res = rposix.sendfile(out, in_, offset, count) + break + except OSError as e: + wrap_oserror(space, e, eintr_retry=True) + return space.newint(res) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -14,7 +14,7 @@ USEMODULES = ['binascii', 'posix', 'signal', 'struct', 'time'] # py3k os.open uses subprocess, requiring the following per platform if os.name != 'nt': - USEMODULES += ['fcntl', 'select', '_posixsubprocess'] + USEMODULES += ['fcntl', 'select', '_posixsubprocess', '_socket'] else: USEMODULES += ['_rawffi', 'thread'] @@ -1189,6 +1189,36 @@ raises(OSError, posix.get_blocking, 1234567) raises(OSError, posix.set_blocking, 1234567, True) + if sys.platform != 'win32': + def test_sendfile(self): + import _socket, posix + s1, s2 = _socket.socketpair() + fd = posix.open(self.path, posix.O_RDONLY) + res = posix.sendfile(s1.fileno(), fd, 3, 5) + assert res == 5 + assert posix.lseek(fd, 0, 1) == 0 + data = s2.recv(10) + expected = b'this is a test'[3:8] + assert data == expected + posix.close(fd) + s2.close() + s1.close() + + if sys.platform.startswith('linux'): + def test_sendfile_no_offset(self): + import _socket, posix + s1, s2 = _socket.socketpair() + fd = posix.open(self.path, posix.O_RDONLY) + posix.lseek(fd, 3, 0) + res = posix.sendfile(s1.fileno(), fd, None, 5) + assert res == 5 + assert posix.lseek(fd, 0, 1) == 8 + data = s2.recv(10) + expected = b'this is a test'[3:8] + assert data == expected + posix.close(fd) + s2.close() + s1.close() def test_urandom(self): os = self.posix From pypy.commits at gmail.com Sun Mar 5 06:59:06 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 03:59:06 -0800 (PST) Subject: [pypy-commit] pypy py3.5: only expose sendfile() on linux, instead of warnings/errors during Message-ID: <58bbfd8a.09532e0a.d4f30.2365@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90565:4503c9682c20 Date: 2017-03-05 12:58 +0100 http://bitbucket.org/pypy/pypy/changeset/4503c9682c20/ Log: only expose sendfile() on linux, instead of warnings/errors during OS/X translation diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -213,7 +213,7 @@ assert getattr(rposix, _name) is not None, "missing %r" % (_name,) interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name) - if hasattr(rposix, 'sendfile'): + if sys.platform.startswith('linux'): #hasattr(rposix, 'sendfile'): interpleveldefs['sendfile'] = 'interp_posix.sendfile' for _name in ["O_CLOEXEC"]: diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -24,7 +24,6 @@ _WIN32 = sys.platform == 'win32' -_LINUX = sys.platform.startswith('linux') if _WIN32: from rpython.rlib import rwin32 @@ -2357,8 +2356,9 @@ # "in" would require all sorts of special cases. I hope nobody # relies on the strange name of CPython. - # XXX only supports the common arguments for now (BSD takes more) - if _LINUX and space.is_none(w_offset): + # XXX only supports the common arguments for now (BSD takes more). + # Until that is fixed, we only expose sendfile() on linux. + if space.is_none(w_offset): # linux only while True: try: res = rposix.sendfile_no_offset(out, in_, count) From pypy.commits at gmail.com Sun Mar 5 07:03:21 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 04:03:21 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Found out how to have an argument called "in" without too much troubles Message-ID: <58bbfe89.c8a8190a.aa4fb.381c@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90566:41d8c8e03b7e Date: 2017-03-05 13:02 +0100 http://bitbucket.org/pypy/pypy/changeset/41d8c8e03b7e/ Log: Found out how to have an argument called "in" without too much troubles diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -1,3 +1,4 @@ +import sys from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rposix from rpython.rlib import rdynload diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -2347,14 +2347,11 @@ except OSError as e: raise wrap_oserror(space, e, eintr_retry=False) - at unwrap_spec(out=c_int, in_=c_int, count=int) -def sendfile(space, out, in_, w_offset, count): - # NB. argument name difference with CPython: "in" versus "in_". - # According to a comment in posixmodule.c, the authors are aware that - # "in" is a Python keyword and so the argument name is bogus, but don't - # do anything about it anyway. PyPy calls it "in_" instead because - # "in" would require all sorts of special cases. I hope nobody - # relies on the strange name of CPython. + at unwrap_spec(out=c_int, count=int) +def sendfile(space, out, w_in, w_offset, count): + # why is an argument called "in"??? that doesn't make sense (it is + # a reserved word), but that's what CPython does + in_ = space.c_int_w(w_in) # XXX only supports the common arguments for now (BSD takes more). # Until that is fixed, we only expose sendfile() on linux. From pypy.commits at gmail.com Sun Mar 5 09:00:07 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 06:00:07 -0800 (PST) Subject: [pypy-commit] pypy default: Add -Wno-address on Linux to reduce a bit the warnings flood Message-ID: <58bc19e7.06d2190a.830de.1ac0@mx.google.com> Author: Armin Rigo Branch: Changeset: r90567:b6744bb01d61 Date: 2017-03-05 14:58 +0100 http://bitbucket.org/pypy/pypy/changeset/b6744bb01d61/ Log: Add -Wno-address on Linux to reduce a bit the warnings flood diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py --- a/rpython/translator/platform/linux.py +++ b/rpython/translator/platform/linux.py @@ -14,7 +14,7 @@ extra_libs = ('-lrt',) cflags = tuple( ['-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused'] + '-Wall', '-Wno-unused', '-Wno-address'] + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) From pypy.commits at gmail.com Sun Mar 5 10:17:19 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 07:17:19 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix for the most recent merge of default Message-ID: <58bc2bff.c8012e0a.3cbef.55b6@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90568:8d60fe293e79 Date: 2017-03-05 16:16 +0100 http://bitbucket.org/pypy/pypy/changeset/8d60fe293e79/ Log: fix for the most recent merge of default diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -15,7 +15,6 @@ W_IOBase, DEFAULT_BUFFER_SIZE, convert_size, trap_eintr, check_readable_w, check_writable_w, check_seekable_w) from rpython.rlib import rthread -from rpython.rtyper.lltypesystem import rffi STATE_ZERO, STATE_OK, STATE_DETACHED = range(3) @@ -174,8 +173,7 @@ self.buf[index] = char def get_raw_address(self): - ptr = nonmoving_raw_ptr_for_resizable_list(self.buf) - return rffi.ptradd(ptr, self.start) + return nonmoving_raw_ptr_for_resizable_list(self.buf) class BufferedMixin: _mixin_ = True diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -293,7 +293,7 @@ spaceconfig = dict(usemodules=['_io', 'thread', 'time']) def test_readinto_small_parts(self): - import _io, os, thread, time + import _io, os, _thread, time read_fd, write_fd = os.pipe() raw = _io.FileIO(read_fd) f = _io.BufferedReader(raw) @@ -302,9 +302,9 @@ def write_more(): time.sleep(0.5) os.write(write_fd, b"fghij") - thread.start_new_thread(write_more, ()) + _thread.start_new_thread(write_more, ()) assert f.readinto(a) == 10 - assert a == 'abcdefghij' + assert a == b'abcdefghij' class AppTestBufferedWriter: From pypy.commits at gmail.com Sun Mar 5 10:35:10 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 07:35:10 -0800 (PST) Subject: [pypy-commit] pypy default: Checked in by mistake in the py3.5 branch. Copy it to default Message-ID: <58bc302e.d4d9190a.ce329.75a0@mx.google.com> Author: Armin Rigo Branch: Changeset: r90569:9ca0089bd94f Date: 2017-03-05 16:34 +0100 http://bitbucket.org/pypy/pypy/changeset/9ca0089bd94f/ Log: Checked in by mistake in the py3.5 branch. Copy it to default diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -432,6 +432,39 @@ _encodeUCS4(result, ch) return result.build() +class SurrogateError(Exception): + def __init__(self, char, index): + self.char = char + self.index = index + +def unicode_encode_utf8_forbid_surrogates(s, size): + # Strict surrogate-forbidding utf-8 encoding. Any surrogate character + # raises an interp-level SurrogateError, even on 16-bit hosts. + # --- XXX check in detail what occurs on 16-bit hosts in PyPy 3 --- + assert(size >= 0) + result = StringBuilder(size) + pos = 0 + while pos < size: + ch = ord(s[pos]) + pos += 1 + if ch < 0x80: + # Encode ASCII + result.append(chr(ch)) + elif ch < 0x0800: + # Encode Latin-1 + result.append(chr((0xc0 | (ch >> 6)))) + result.append(chr((0x80 | (ch & 0x3f)))) + elif ch < 0x10000: + if 0xD800 <= ch <= 0xDFFF: + raise SurrogateError(ch, pos) + # Encode UCS2 Unicode ordinals + result.append((chr((0xe0 | (ch >> 12))))) + result.append((chr((0x80 | ((ch >> 6) & 0x3f))))) + result.append((chr((0x80 | (ch & 0x3f))))) + else: + _encodeUCS4(result, ch) + return result.build() + # ____________________________________________________________ # utf-16 From pypy.commits at gmail.com Sun Mar 5 10:44:07 2017 From: pypy.commits at gmail.com (fijal) Date: Sun, 05 Mar 2017 07:44:07 -0800 (PST) Subject: [pypy-commit] pypy unicode-utf8: disable speedups for now Message-ID: <58bc3247.53be190a.f3883.2405@mx.google.com> Author: fijal Branch: unicode-utf8 Changeset: r90570:3e38274ddd35 Date: 2017-03-05 19:42 +0400 http://bitbucket.org/pypy/pypy/changeset/3e38274ddd35/ Log: disable speedups for now diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -295,7 +295,7 @@ i += 1 bits |= ord(ch) if ch == '"': - if bits & 0x80: + if 1 or bits & 0x80: # the 8th bit is set, it's an utf8 strnig content_utf8 = self.getslice(start, i-1) content_unicode = unicodehelper.decode_utf8(self.space, content_utf8) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -20,6 +20,7 @@ from pypy.objspace.std.formatting import mod_format from pypy.objspace.std.stringmethods import StringMethods from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT +from pypy.objspace.std.sliceobject import unwrap_start_stop __all__ = ['W_UnicodeObject', 'wrapunicode', 'plain_str2unicode', 'encode_object', 'decode_object', 'unicode_from_object', @@ -75,6 +76,13 @@ uid = (base << IDTAG_SHIFT) | IDTAG_SPECIAL return space.newint(uid) + def _convert_idx_params_unicode(self, space, w_start, w_end): + """ Specialcase this for unicode - one less element in the tuple + """ + lenself = self._len() + start, end = unwrap_start_stop(space, lenself, w_start, w_end) + return start, end + def str_w(self, space): return space.text_w(space.str(self)) @@ -125,8 +133,8 @@ return rutf8.compute_length_utf8(self._utf8) def _val(self, space): - #import pdb - #pdb.set_trace() + import pdb + pdb.set_trace() return self._utf8.decode('utf8') @staticmethod @@ -446,9 +454,6 @@ i = rutf8.next_codepoint_pos(val, i) return space.newbool(cased) - def _starts_ends_overflow(self, prefix): - return len(prefix) == 0 - def descr_add(self, space, w_other): try: w_other = self.convert_arg_to_w_unicode(space, w_other) @@ -722,6 +727,26 @@ assert rpos >= lpos # annotator hint, don't remove return self._utf8_sliced(lpos, rpos, lgt) + def descr_startswith(self, space, w_prefix, w_start=None, w_end=None): + (start, end) = self._convert_idx_params_unicode(space, w_start, w_end) + if space.isinstance_w(w_prefix, space.w_tuple): + return self._startswith_tuple(space, w_prefix, start, end) + return space.newbool(self._startswith(space, w_prefix, start, end)) + + def _startswith_tuple(self, space, w_prefix, start, end): + for w_prefix in space.fixedview(w_prefix): + if self._startswith(space, w_prefix, start, end): + return space.w_True + return space.w_False + + def _startswith(self, space, w_prefix, start, end): + prefix = self.convert_arg_to_w_unicode(space, w_prefix)._utf8 + if start > self._len(): + return len(prefix) == 0 # bug-to-bug cpython compatibility + xxx + return startswith(self._utf8, prefix, start, end) + + def descr_getnewargs(self, space): return space.newtuple([W_UnicodeObject(self._utf8, self._length)]) From pypy.commits at gmail.com Sun Mar 5 13:12:56 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 10:12:56 -0800 (PST) Subject: [pypy-commit] pypy default: Add an explicit check with a clear error message in case memory Message-ID: <58bc5528.1f002e0a.a2e24.e206@mx.google.com> Author: Armin Rigo Branch: Changeset: r90571:e1aafbb1ad9c Date: 2017-03-05 19:12 +0100 http://bitbucket.org/pypy/pypy/changeset/e1aafbb1ad9c/ Log: Add an explicit check with a clear error message in case memory corruption creates an object "bigger than the nursery" in the nursery diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -2124,7 +2124,8 @@ def _malloc_out_of_nursery(self, totalsize): """Allocate non-movable memory for an object of the given 'totalsize' that lives so far in the nursery.""" - if raw_malloc_usage(totalsize) <= self.small_request_threshold: + if (r_uint(raw_malloc_usage(totalsize)) <= + r_uint(self.small_request_threshold)): # most common path return self.ac.malloc(totalsize) else: @@ -2133,6 +2134,9 @@ _malloc_out_of_nursery._always_inline_ = True def _malloc_out_of_nursery_nonsmall(self, totalsize): + if r_uint(raw_malloc_usage(totalsize)) > r_uint(self.nursery_size): + out_of_memory("memory corruption: bad size for object in the " + "nursery") # 'totalsize' should be aligned. ll_assert(raw_malloc_usage(totalsize) & (WORD-1) == 0, "misaligned totalsize in _malloc_out_of_nursery_nonsmall") From pypy.commits at gmail.com Sun Mar 5 15:34:28 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 12:34:28 -0800 (PST) Subject: [pypy-commit] pypy default: bug fix (memory corruption showing up from a cond_call_value_r) Message-ID: <58bc7654.53be190a.f3883.2ef4@mx.google.com> Author: Armin Rigo Branch: Changeset: r90572:4c7864396507 Date: 2017-03-05 21:33 +0100 http://bitbucket.org/pypy/pypy/changeset/4c7864396507/ Log: bug fix (memory corruption showing up from a cond_call_value_r) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -948,11 +948,11 @@ # If this also contains args[0], this returns the current # location too. arglocs = [self.loc(args[i]) for i in range(2, len(args))] - gcmap = self.get_gcmap() if op.type == 'v': # a plain COND_CALL. Calls the function when args[0] is # true. Often used just after a comparison operation. + gcmap = self.get_gcmap() self.load_condition_into_cc(op.getarg(0)) resloc = None else: @@ -969,6 +969,22 @@ resloc = self.rm.force_result_in_reg(op, args[0], forbidden_vars=args[2:]) + # Get the gcmap here, possibly including the spilled + # location, and always excluding the 'resloc' register. + # Some more details: the only interesting case is the case + # where we're doing the call (if we are not, the gcmap is + # not used); and in this case, the gcmap must include the + # spilled location (it contains a valid GC pointer to fix + # during the call if a GC occurs), and never 'resloc' + # (it will be overwritten with the result of the call, which + # is not computed yet if a GC occurs). + # + # (Note that the spilled value is always NULL at the moment + # if the call really occurs, but it's not worth the effort to + # not list it in the gcmap and get crashes if we tweak + # COND_CALL_VALUE_R in the future) + gcmap = self.get_gcmap([resloc]) + # Test the register for the result. self.assembler.test_location(resloc) self.assembler.guard_success_cc = rx86.Conditions['Z'] From pypy.commits at gmail.com Sun Mar 5 15:35:04 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 12:35:04 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58bc7678.cdd8190a.1686.3aae@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90573:7b9cf75b06d0 Date: 2017-03-05 21:34 +0100 http://bitbucket.org/pypy/pypy/changeset/7b9cf75b06d0/ Log: hg merge default diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -948,11 +948,11 @@ # If this also contains args[0], this returns the current # location too. arglocs = [self.loc(args[i]) for i in range(2, len(args))] - gcmap = self.get_gcmap() if op.type == 'v': # a plain COND_CALL. Calls the function when args[0] is # true. Often used just after a comparison operation. + gcmap = self.get_gcmap() self.load_condition_into_cc(op.getarg(0)) resloc = None else: @@ -969,6 +969,22 @@ resloc = self.rm.force_result_in_reg(op, args[0], forbidden_vars=args[2:]) + # Get the gcmap here, possibly including the spilled + # location, and always excluding the 'resloc' register. + # Some more details: the only interesting case is the case + # where we're doing the call (if we are not, the gcmap is + # not used); and in this case, the gcmap must include the + # spilled location (it contains a valid GC pointer to fix + # during the call if a GC occurs), and never 'resloc' + # (it will be overwritten with the result of the call, which + # is not computed yet if a GC occurs). + # + # (Note that the spilled value is always NULL at the moment + # if the call really occurs, but it's not worth the effort to + # not list it in the gcmap and get crashes if we tweak + # COND_CALL_VALUE_R in the future) + gcmap = self.get_gcmap([resloc]) + # Test the register for the result. self.assembler.test_location(resloc) self.assembler.guard_success_cc = rx86.Conditions['Z'] diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -2124,7 +2124,8 @@ def _malloc_out_of_nursery(self, totalsize): """Allocate non-movable memory for an object of the given 'totalsize' that lives so far in the nursery.""" - if raw_malloc_usage(totalsize) <= self.small_request_threshold: + if (r_uint(raw_malloc_usage(totalsize)) <= + r_uint(self.small_request_threshold)): # most common path return self.ac.malloc(totalsize) else: @@ -2133,6 +2134,9 @@ _malloc_out_of_nursery._always_inline_ = True def _malloc_out_of_nursery_nonsmall(self, totalsize): + if r_uint(raw_malloc_usage(totalsize)) > r_uint(self.nursery_size): + out_of_memory("memory corruption: bad size for object in the " + "nursery") # 'totalsize' should be aligned. ll_assert(raw_malloc_usage(totalsize) & (WORD-1) == 0, "misaligned totalsize in _malloc_out_of_nursery_nonsmall") diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py --- a/rpython/translator/platform/linux.py +++ b/rpython/translator/platform/linux.py @@ -14,7 +14,7 @@ extra_libs = ('-lrt',) cflags = tuple( ['-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused'] + '-Wall', '-Wno-unused', '-Wno-address'] + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) From pypy.commits at gmail.com Sun Mar 5 15:39:46 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 12:39:46 -0800 (PST) Subject: [pypy-commit] pypy default: Untested: same as 4c7864396507 Message-ID: <58bc7792.43502e0a.9f6b9.3379@mx.google.com> Author: Armin Rigo Branch: Changeset: r90574:9c0fa68bb404 Date: 2017-03-05 21:39 +0100 http://bitbucket.org/pypy/pypy/changeset/9c0fa68bb404/ Log: Untested: same as 4c7864396507 diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -666,7 +666,7 @@ jmp_adr = self.mc.get_relative_pos() self.mc.trap() # patched later to a 'bc' - self.load_gcmap(self.mc, r.r2, regalloc.get_gcmap()) + self.load_gcmap(self.mc, r.r2, regalloc.get_gcmap([resloc])) # save away r3, r4, r5, r6, r12 into the jitframe should_be_saved = [ diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -390,7 +390,7 @@ if reg in self._COND_CALL_SAVE_REGS] self._push_core_regs_to_jitframe(self.mc, should_be_saved) - self.push_gcmap(self.mc, regalloc.get_gcmap()) + self.push_gcmap(self.mc, regalloc.get_gcmap([resloc])) # # load the 0-to-4 arguments into these registers, with the address of # the function to call into r11 From pypy.commits at gmail.com Sun Mar 5 15:41:38 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 12:41:38 -0800 (PST) Subject: [pypy-commit] pypy default: Untested: same as 4c7864396507 Message-ID: <58bc7802.421a190a.76988.479b@mx.google.com> Author: Armin Rigo Branch: Changeset: r90575:2784c9a05d04 Date: 2017-03-05 21:41 +0100 http://bitbucket.org/pypy/pypy/changeset/2784c9a05d04/ Log: Untested: same as 4c7864396507 diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -362,9 +362,8 @@ res_loc = arglocs[1] # cond_call_value else: res_loc = None # cond_call - # useless to list res_loc in the gcmap, because if the call is - # done it means res_loc was initially NULL - gcmap = regalloc.get_gcmap([call_loc]) + # see x86.regalloc for why we skip res_loc in the gcmap + gcmap = regalloc.get_gcmap([res_loc]) assert call_loc is r.r4 jmp_adr = self.mc.currpos() From pypy.commits at gmail.com Sun Mar 5 15:50:58 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 05 Mar 2017 12:50:58 -0800 (PST) Subject: [pypy-commit] pypy default: fix comment Message-ID: <58bc7a32.97002e0a.5bf6e.a2e8@mx.google.com> Author: Armin Rigo Branch: Changeset: r90576:b2f0ed2ba5b5 Date: 2017-03-05 21:50 +0100 http://bitbucket.org/pypy/pypy/changeset/b2f0ed2ba5b5/ Log: fix comment diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -998,7 +998,7 @@ size_box = op.getarg(0) assert isinstance(size_box, ConstInt) size = size_box.getint() - # hint: try to move unrelated registers away from eax and edx now + # hint: try to move unrelated registers away from ecx and edx now self.rm.spill_or_move_registers_before_call([ecx, edx]) # the result will be in ecx self.rm.force_allocate_reg(op, selected_reg=ecx) From pypy.commits at gmail.com Sun Mar 5 16:52:29 2017 From: pypy.commits at gmail.com (mjacob) Date: Sun, 05 Mar 2017 13:52:29 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Implement PEP 475 in the optimized path of readinto_w. Message-ID: <58bc889d.421a190a.76988.49a0@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90577:068708147f9e Date: 2017-03-05 21:41 +0100 http://bitbucket.org/pypy/pypy/changeset/068708147f9e/ Log: Implement PEP 475 in the optimized path of readinto_w. diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -468,26 +468,26 @@ except OSError as e: if e.errno == errno.EAGAIN: return space.w_None - wrap_oserror(space, e, - exception_name='w_IOError', + wrap_oserror(space, e, exception_name='w_IOError', eintr_retry=True) rwbuffer.setslice(0, buf) return space.newint(len(buf)) else: # optimized case: reading more than 64 bytes into a rwbuffer # with a valid raw address - # XXX TODO(mjacob): implement PEP 475 here! - got = c_read(self.fd, target_address, length) - keepalive_until_here(rwbuffer) - got = rffi.cast(lltype.Signed, got) - if got >= 0: - return space.newint(got) - else: - err = get_saved_errno() - if err == errno.EAGAIN: - return space.w_None - e = OSError(err, "read failed") - raise wrap_oserror(space, e, exception_name='w_IOError') + while True: + got = c_read(self.fd, target_address, length) + keepalive_until_here(rwbuffer) + got = rffi.cast(lltype.Signed, got) + if got >= 0: + return space.newint(got) + else: + err = get_saved_errno() + if err == errno.EAGAIN: + return space.w_None + e = OSError(err, "read failed") + wrap_oserror(space, e, exception_name='w_IOError', + eintr_retry=True) def readall_w(self, space): self._check_closed(space) From pypy.commits at gmail.com Mon Mar 6 03:52:13 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Mar 2017 00:52:13 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: Fix for test/test_rstacklet.py Message-ID: <58bd233d.46012e0a.682dd.6bbc@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90578:a759310b811d Date: 2017-03-06 09:51 +0100 http://bitbucket.org/pypy/pypy/changeset/a759310b811d/ Log: Fix for test/test_rstacklet.py diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -40,6 +40,36 @@ # shadowcolor.move_pushes_earlier() + at specialize.call_location() +def walk_stack_root(invoke, arg0, arg1, start, addr, is_minor): + skip = 0 + while addr != start: + addr -= sizeofaddr + #XXX reintroduce support for tagged values? + #if gc.points_to_valid_gc_object(addr): + # callback(gc, addr) + + if skip & 1 == 0: + content = addr.address[0] + n = llmemory.cast_adr_to_int(content) + if n & 1 == 0: + if content: # non-0, non-odd: a regular ptr + invoke(arg0, arg1, addr) + else: + # odd number: a skip bitmask + if n > 0: # initially, an unmarked value + if is_minor: + newcontent = llmemory.cast_int_to_adr(-n) + addr.address[0] = newcontent # mark + skip = n + else: + # a marked value + if is_minor: + return + skip = -n + skip >>= 1 + + class ShadowStackRootWalker(BaseRootWalker): def __init__(self, gctransformer): BaseRootWalker.__init__(self, gctransformer) @@ -58,35 +88,6 @@ return top self.decr_stack = decr_stack - @specialize.call_location() - def walk_stack_root(invoke, arg0, arg1, start, addr, is_minor): - skip = 0 - while addr != start: - addr -= sizeofaddr - #XXX reintroduce support for tagged values? - #if gc.points_to_valid_gc_object(addr): - # callback(gc, addr) - - if skip & 1 == 0: - content = addr.address[0] - n = llmemory.cast_adr_to_int(content) - if n & 1 == 0: - if content: # non-0, non-odd: a regular ptr - invoke(arg0, arg1, addr) - else: - # odd number: a skip bitmask - if n > 0: # initially, an unmarked value - if is_minor: - newcontent = llmemory.cast_int_to_adr(-n) - addr.address[0] = newcontent # mark - skip = n - else: - # a marked value - if is_minor: - return - skip = -n - skip >>= 1 - self.rootstackhook = walk_stack_root self.invoke_collect_stack_root = specialize.call_location()( lambda arg0, arg1, addr: arg0(self.gc, addr)) @@ -109,9 +110,9 @@ def walk_stack_roots(self, collect_stack_root, is_minor=False): gcdata = self.gcdata - self.rootstackhook(self.invoke_collect_stack_root, collect_stack_root, - None, gcdata.root_stack_base, gcdata.root_stack_top, - is_minor=is_minor) + walk_stack_root(self.invoke_collect_stack_root, collect_stack_root, + None, gcdata.root_stack_base, gcdata.root_stack_top, + is_minor=is_minor) def need_thread_support(self, gctransformer, getfn): from rpython.rlib import rthread # xxx fish @@ -329,9 +330,8 @@ def customtrace(gc, obj, callback, arg): obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) - root_walker.rootstackhook(gc._trace_callback, callback, arg, - obj.base, obj.top, - is_minor=False) # xxx optimize? + walk_stack_root(gc._trace_callback, callback, arg, obj.base, obj.top, + is_minor=False) # xxx optimize? gc = gctransformer.gcdata.gc assert not hasattr(gc, 'custom_trace_dispatcher') diff --git a/rpython/rlib/_stacklet_shadowstack.py b/rpython/rlib/_stacklet_shadowstack.py --- a/rpython/rlib/_stacklet_shadowstack.py +++ b/rpython/rlib/_stacklet_shadowstack.py @@ -47,14 +47,15 @@ SIZEADDR = llmemory.sizeof(llmemory.Address) def customtrace(gc, obj, callback, arg): + from rpython.memory.gctransform.shadowstack import walk_stack_root + stacklet = llmemory.cast_adr_to_ptr(obj, STACKLET_PTR) sscopy = stacklet.s_sscopy if sscopy: length_bytes = sscopy.signed[0] - while length_bytes > 0: - addr = sscopy + length_bytes - gc._trace_callback(callback, arg, addr) - length_bytes -= SIZEADDR + walk_stack_root(gc._trace_callback, callback, arg, + sscopy + SIZEADDR, sscopy + SIZEADDR + length_bytes, + is_minor=False) lambda_customtrace = lambda: customtrace def sscopy_detach_shadow_stack(): From pypy.commits at gmail.com Mon Mar 6 05:19:00 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Mon, 06 Mar 2017 02:19:00 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Add macros for timing instrumentation Message-ID: <58bd3794.110f2e0a.5bc7f.5a97@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2023:f1f7a2e83f3c Date: 2017-03-06 10:55 +0100 http://bitbucket.org/pypy/stmgc/changeset/f1f7a2e83f3c/ Log: Add macros for timing instrumentation diff --git a/c8/stm/prof.c b/c8/stm/prof.c --- a/c8/stm/prof.c +++ b/c8/stm/prof.c @@ -39,8 +39,11 @@ buf.extra, EXTRA_LEN_MAX); } } else if (payload->type == STM_EVENT_PAYLOAD_DURATION) { - uint32_t duration = payload->data.duration; - buf.extra_length = sprintf(buf.extra, "%u", duration); + struct timespec *duration = payload->data.duration; + buf.extra_length = sprintf(buf.extra, + "s%un%u", + duration->tv_sec, + duration->tv_nsec); } } diff --git a/c8/stm/timing.h b/c8/stm/timing.h new file mode 100644 --- /dev/null +++ b/c8/stm/timing.h @@ -0,0 +1,16 @@ +#include + +/* Use raw monotonic time, i.e., solely based on local hardware (no NTP + adjustments) as in prof.c to obtain values comparable with total program + runtime. */ +#define start_timer() struct timespec start; \ + clock_gettime(CLOCK_MONOTONIC_RAW, &start); + +#define stop_timer() struct timespec stop; \ + clock_gettime(CLOCK_MONOTONIC_RAW, &stop); + +/* Must use start_timer and stop_timer before using this macro. */ +#define get_duration() struct timespec duration = { \ + stop.tv_sec - start.tv_sec, \ + stop.tv_nsec - start.tv_nsec \ + }; diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -201,7 +201,7 @@ /* ==================== PUBLIC API ==================== */ /* Number of segments (i.e. how many transactions can be executed in - parallel, in maximum). If you try to start transactions in more + parallel, at maximum). If you try to start transactions in more threads than the number of segments, it will block, waiting for the next segment to become free. */ @@ -616,9 +616,11 @@ _STM_EVENT_PAYLOAD_N }; + +#include typedef union { stm_loc_marker_t *loc_marker; - uint32_t duration; + struct timespec *duration; } stm_timing_event_payload_data_t; /* Wrapper for payload holding data type and data. */ typedef struct { From pypy.commits at gmail.com Mon Mar 6 05:18:58 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Mon, 06 Mar 2017 02:18:58 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Add names of duration events Message-ID: <58bd3792.8c2b190a.1f257.5497@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2022:ea8cfda82123 Date: 2017-03-03 15:18 +0100 http://bitbucket.org/pypy/stmgc/changeset/ea8cfda82123/ Log: Add names of duration events diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -595,8 +595,13 @@ "gc minor start", \ "gc minor done", \ "gc major start", \ - "gc major done" - /* TODO names for new duration events */ + "gc major done", \ + /* names of duration events */ \ + "duration of write slowpath", \ + "duration of validation", \ + "duration of commit", \ + "duration of minor gc", \ + "duration of major gc" /* The markers pushed in the shadowstack are an odd number followed by a regular object pointer. */ From pypy.commits at gmail.com Mon Mar 6 07:37:40 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Mar 2017 04:37:40 -0800 (PST) Subject: [pypy-commit] pypy py3.5: issue #2489: fix to support half-bogus use case Message-ID: <58bd5814.c35c2e0a.b3faa.57c6@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90579:7e0ff26e1482 Date: 2017-03-06 13:37 +0100 http://bitbucket.org/pypy/pypy/changeset/7e0ff26e1482/ Log: issue #2489: fix to support half-bogus use case diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py --- a/lib-python/3/datetime.py +++ b/lib-python/3/datetime.py @@ -1000,6 +1000,11 @@ else: return (self.__class__, args, state) + # PyPy: added for compatibility with the _datetime module + # issue #2489 + def __new__(cls, *args, **kwds): + return super(tzinfo, cls).__new__(cls) + _tzinfo_class = tzinfo class time: From pypy.commits at gmail.com Mon Mar 6 11:35:03 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Mar 2017 08:35:03 -0800 (PST) Subject: [pypy-commit] pypy default: link to the B3 JIT dropping llvm Message-ID: <58bd8fb7.53be190a.f3883.61b1@mx.google.com> Author: Armin Rigo Branch: Changeset: r90580:774e328def6e Date: 2017-03-06 17:34 +0100 http://bitbucket.org/pypy/pypy/changeset/774e328def6e/ Log: link to the B3 JIT dropping llvm diff --git a/rpython/doc/faq.rst b/rpython/doc/faq.rst --- a/rpython/doc/faq.rst +++ b/rpython/doc/faq.rst @@ -182,7 +182,11 @@ On the other hand, using LLVM as our JIT backend looks interesting as well --- but again we made an attempt, and it failed: LLVM has no way to -patch the generated machine code. +patch the generated machine code, and is not suited at all to tracing +JITs. Even one big method JIT trying to use LLVM `has given up`__ for +similar reasons; read that blog post for more details. + +.. __: https://webkit.org/blog/5852/introducing-the-b3-jit-compiler/ So the position of the core PyPy developers is that if anyone wants to make an N+1'th attempt with LLVM, they are welcome, and they will receive a From pypy.commits at gmail.com Mon Mar 6 14:14:55 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Mar 2017 11:14:55 -0800 (PST) Subject: [pypy-commit] pypy default: more docs Message-ID: <58bdb52f.53532e0a.e0da2.681f@mx.google.com> Author: Armin Rigo Branch: Changeset: r90581:508a0ac8cf55 Date: 2017-03-06 20:14 +0100 http://bitbucket.org/pypy/pypy/changeset/508a0ac8cf55/ Log: more docs diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -255,6 +255,14 @@ getsizeof_missing = """sys.getsizeof() is not implemented on PyPy. +First note that the CPython documentation says that this function may +raise a TypeError, so if you are seeing it, it means that the program +you are using is not correctly handling this case. + +On PyPy, though, it always raises TypeError. Before looking for +alternatives, please take a moment to read the following explanation as +to why it is the case. What you are looking for may not be possible. + A memory profiler using this function is most likely to give results inconsistent with reality on PyPy. It would be possible to have sys.getsizeof() return a number (with enough work), but that may or From pypy.commits at gmail.com Tue Mar 7 01:11:00 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Mar 2017 22:11:00 -0800 (PST) Subject: [pypy-commit] pypy default: typo Message-ID: <58be4ef4.84012e0a.268da.b402@mx.google.com> Author: Armin Rigo Branch: Changeset: r90582:02c99087b39c Date: 2017-03-07 07:10 +0100 http://bitbucket.org/pypy/pypy/changeset/02c99087b39c/ Log: typo diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -11,7 +11,7 @@ * **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flamgraph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). * **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. From pypy.commits at gmail.com Tue Mar 7 01:15:30 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 06 Mar 2017 22:15:30 -0800 (PST) Subject: [pypy-commit] pypy default: add my differing opinion Message-ID: <58be5002.4395190a.a6c86.7a65@mx.google.com> Author: Armin Rigo Branch: Changeset: r90583:40bcf6c2d884 Date: 2017-03-07 07:15 +0100 http://bitbucket.org/pypy/pypy/changeset/40bcf6c2d884/ Log: add my differing opinion diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -7,7 +7,7 @@ PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. -* **Optimize PyPy Memory Usage**: PyPy currently emits a small executable file and a large shared object file. To reduce the base interpreter size we think it would be helpful to have several shared object files that can be dynamically loaded if the module is needed. There are several other potential places where we could improve. +* **Optimize PyPy Memory Usage**: PyPy currently emits a small executable file and a large shared object file. To reduce the base interpreter size it may be helpful to have several shared object files that can be dynamically loaded if the module is needed. (Note: it is unclear if this would change anything at all. Please make a small test first and report.) There are several other potential places where we could improve. * **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. From pypy.commits at gmail.com Tue Mar 7 07:50:27 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 07 Mar 2017 04:50:27 -0800 (PST) Subject: [pypy-commit] pypy py3.5: _Py_RestoreSignals(). Hard to test Message-ID: <58beac93.1a4b2e0a.95e02.9b6f@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90584:25aaba6da424 Date: 2017-03-07 13:49 +0100 http://bitbucket.org/pypy/pypy/changeset/25aaba6da424/ Log: _Py_RestoreSignals(). Hard to test diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -597,6 +597,7 @@ 'PyImport_ImportModuleLevel', 'PyOS_getsig', 'PyOS_setsig', + '_Py_RestoreSignals', 'PyThread_get_thread_ident', 'PyThread_allocate_lock', 'PyThread_free_lock', 'PyThread_acquire_lock', 'PyThread_release_lock', 'PyThread_create_key', 'PyThread_delete_key', 'PyThread_set_key_value', diff --git a/pypy/module/cpyext/include/pylifecycle.h b/pypy/module/cpyext/include/pylifecycle.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/pylifecycle.h @@ -0,0 +1,17 @@ +#ifndef Py_PYLIFECYCLE_H +#define Py_PYLIFECYCLE_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _Py_RestoreSignals(void); +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_PYLIFECYCLE_H */ diff --git a/pypy/module/cpyext/src/pylifecycle.c b/pypy/module/cpyext/src/pylifecycle.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/src/pylifecycle.c @@ -0,0 +1,15 @@ +#include "Python.h" + +void +_Py_RestoreSignals(void) +{ +#ifdef SIGPIPE + PyOS_setsig(SIGPIPE, SIG_DFL); +#endif +#ifdef SIGXFZ + PyOS_setsig(SIGXFZ, SIG_DFL); +#endif +#ifdef SIGXFSZ + PyOS_setsig(SIGXFSZ, SIG_DFL); +#endif +} From pypy.commits at gmail.com Tue Mar 7 08:20:44 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 07 Mar 2017 05:20:44 -0800 (PST) Subject: [pypy-commit] cffi default: typo Message-ID: <58beb3ac.129f190a.de2e2.0003@mx.google.com> Author: Armin Rigo Branch: Changeset: r2896:d9575530d274 Date: 2017-03-07 14:20 +0100 http://bitbucket.org/cffi/cffi/changeset/d9575530d274/ Log: typo diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -452,7 +452,7 @@ ... # the actual callback is this one-liner global function: - @ffi.def_extern + @ffi.def_extern() def my_callback(arg1, arg2, data): return ffi.from_handle(data).callback(arg1, arg2) From pypy.commits at gmail.com Tue Mar 7 13:16:46 2017 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 07 Mar 2017 10:16:46 -0800 (PST) Subject: [pypy-commit] pypy default: rewrite the first gsoc task, do not focus on the interp. size, but generally spot and solve issues where pypy keeps more memory Message-ID: <58bef90e.1b582e0a.3ad41.0dc7@mx.google.com> Author: Richard Plangger Branch: Changeset: r90585:8a2bfc0d6823 Date: 2017-03-07 19:16 +0100 http://bitbucket.org/pypy/pypy/changeset/8a2bfc0d6823/ Log: rewrite the first gsoc task, do not focus on the interp. size, but generally spot and solve issues where pypy keeps more memory diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -7,7 +7,12 @@ PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. -* **Optimize PyPy Memory Usage**: PyPy currently emits a small executable file and a large shared object file. To reduce the base interpreter size it may be helpful to have several shared object files that can be dynamically loaded if the module is needed. (Note: it is unclear if this would change anything at all. Please make a small test first and report.) There are several other potential places where we could improve. +* **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. + Two examples: 1) PyPy seems allocate and keep many strings when importing a big Python modules. + 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. + The general procedure of this project is: Run both CPython and PyPy of the same Python version and + compare the memory usage (using Massif or other tools). + If PyPy consumes a lot more memory then find and resolve the issue. * **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. From pypy.commits at gmail.com Wed Mar 8 03:16:23 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 00:16:23 -0800 (PST) Subject: [pypy-commit] benchmarks default: Add a reasonable way (IMO) to run benchmarks locally and display results. Message-ID: <58bfbdd7.d426190a.f494a.2746@mx.google.com> Author: Armin Rigo Branch: Changeset: r369:9b140bbc577d Date: 2017-03-08 09:16 +0100 http://bitbucket.org/pypy/benchmarks/changeset/9b140bbc577d/ Log: Add a reasonable way (IMO) to run benchmarks locally and display results. diff --git a/display_local.py b/display_local.py new file mode 100755 --- /dev/null +++ b/display_local.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +"""Usage: + + display_local.py first-filename [second-filename] + +Displays the content of the file resulting from 'run_local.py'. With +two arguments, computes statistics and displays the differences. + +(Details: each file must result from a '--full-store' execution of +'runner.py'. The "changed_times" keys are used and the "base_times" +keys are discarded. The option '--base1' and/or '--base2' can be used +to pick the "base_times" instead in the first/second file. These +options are not useful if the files are produced by 'run_local.py' +because it uses nullpython as the base.) +""" + +import sys +import json +from unladen_swallow import perf + + +def load_times(filename, base_times=False): + with open(filename) as f: + d = json.load(f) + + if base_times: + key_times = "base_times" + else: + key_times = "changed_times" + + result = {} + for lst2 in d['results']: + name = str(lst2[0]) + if lst2[1] != 'RawResult': + print ("ERROR: entry for %r is not a RawResult " + "(missing --full-store?)" % (name,)) + continue + result[name] = lst2[2][key_times] + if not result: + print "No valid result in %r." % (filename,) + sys.exit(1) + return result + + +def _report(row, raw1): + if raw1 is None: + row.append('') + row.append('') + row.append('') + row.append('') + row.append('') + elif len(raw1) == 1: + # A single result. Report it. + row.append('') + row.append('') + row.append(str(round(raw1[0], 3))) + row.append('') + row.append('') + elif len(raw1) == 0: + # Should not occur + row.append('???') + row.append('') + row.append('empty') + row.append('') + row.append('???') + else: + # Multiple results. + t_min = min(raw1) + t_avg = perf.avg(raw1) + t_std = perf.SampleStdDev(raw1) + row.append(str(round(t_min, 3))) + row.append('') + row.append(str(round(t_avg, 3))) + row.append('') + row.append(str(round(t_std, 5))) + row.append('') + return raw1 + +def display(times1, times2=None): + if times2 is None: + times2 = {} + all_names = sorted(set(times1) | set(times2)) + table = [['BENCHMARK', ' ', 'min', ' ', 'avg', ' ', 'stddev', ' ', + 'min', ' ', 'avg', ' ', 'stddev', ' ', + 'diff']] + RIGHT_ALIGN = '\x00' + + for name in all_names: + row = [name, ''] + table.append(row) + raw1 = _report(row, times1.get(name)) + raw2 = _report(row, times2.get(name)) + if raw1 and raw2: + row.append(perf.TimeDelta(perf.avg(raw1), perf.avg(raw2))) + + lengths = [] + for row in table: + while len(lengths) < len(row): + lengths.append(0) + for i, cell in enumerate(row): + if len(cell) > lengths[i]: + lengths[i] = len(cell) + for row in table: + s = '' + for cell, l1 in zip(row, lengths): + if cell.startswith(RIGHT_ALIGN): + cell = ' '*(l1 - len(cell) - 1) + cell[1:] + s += cell + ' '*(l1 - len(cell)) + print s + + +def main(argv): + import optparse + parser = optparse.OptionParser( + usage="%prog first-filename [second-filename]", + description=__doc__) + + parser.add_option("--base1", default=False, action="store_true", + help='Pick the "base_times" keys instead of the "changed_times"' + ' ones in the first file') + parser.add_option("--base2", default=False, action="store_true", + help='Pick the "base_times" keys instead of the "changed_times"' + ' ones in the second file') + options, args = parser.parse_args(argv) + + if len(args) == 0: + parser.error("no filenames given; try --help") + elif len(args) > 2: + parser.error("too many filenames") + + times1 = load_times(args[0], base_times=options.base1) + if len(args) > 1: + times2 = load_times(args[1], base_times=options.base2) + else: + times2 = None + display(times1, times2) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/run_local.py b/run_local.py new file mode 100755 --- /dev/null +++ b/run_local.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +"""Usage: + + run_local.py path/to/pypy-c -o output-filename + +This is a wrapper script around runner.py that makes it easier to run +locally all benchmarks on a single given pypy-c. It stores the result +in a JSON file given as 'output-filename'. You can then run +'display_local.py' to display the output or the differences between two +such output files. + +More options can be given on the command line and are passed to runner.py. +Common ones are: + + -b BENCHMARK_LIST + --fast + --args=ARGS arguments to give to pypy-c, must not contain a comma! +""" + +import sys, os +import subprocess + +if len(sys.argv) < 2 or sys.argv[1].startswith('-'): + print __doc__ + sys.exit(2) + +pypy_c = sys.argv[1] + +localdir = os.path.dirname(sys.argv[0]) or '.' + +cmdline = [sys.executable, os.path.join(localdir, 'runner.py'), + '--changed', pypy_c, + '--baseline', os.path.join(localdir, 'nullpython.py'), + '--full-store', + ] + sys.argv[1:] +print +print 'Executing', cmdline +print + +r = subprocess.call(cmdline) +if r: + print >> sys.stderr, '*** exit code %r ***' % (r,) + sys.exit(r) From pypy.commits at gmail.com Wed Mar 8 03:17:46 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 08 Mar 2017 00:17:46 -0800 (PST) Subject: [pypy-commit] pypy default: typo in project ideas Message-ID: <58bfbe2a.1459190a.79f3f.290b@mx.google.com> Author: Richard Plangger Branch: Changeset: r90586:63b4c36d026f Date: 2017-03-08 09:17 +0100 http://bitbucket.org/pypy/pypy/changeset/63b4c36d026f/ Log: typo in project ideas diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -8,7 +8,7 @@ * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems allocate and keep many strings when importing a big Python modules. + Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. The general procedure of this project is: Run both CPython and PyPy of the same Python version and compare the memory usage (using Massif or other tools). From pypy.commits at gmail.com Wed Mar 8 03:29:21 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 00:29:21 -0800 (PST) Subject: [pypy-commit] benchmarks default: Display the geometric averages Message-ID: <58bfc0e1.4d29190a.f51f4.28f9@mx.google.com> Author: Armin Rigo Branch: Changeset: r370:9dfde57979c7 Date: 2017-03-08 09:29 +0100 http://bitbucket.org/pypy/benchmarks/changeset/9dfde57979c7/ Log: Display the geometric averages diff --git a/display_local.py b/display_local.py --- a/display_local.py +++ b/display_local.py @@ -16,6 +16,7 @@ import sys import json +import math from unladen_swallow import perf @@ -76,22 +77,42 @@ row.append('') return raw1 +def geometric_average(lst): + return math.exp(sum(math.log(t) for t in lst) / len(lst)) + def display(times1, times2=None): if times2 is None: times2 = {} all_names = sorted(set(times1) | set(times2)) - table = [['BENCHMARK', ' ', 'min', ' ', 'avg', ' ', 'stddev', ' ', + table = [[], + ['BENCHMARK', ' ', 'min', ' ', 'avg', ' ', 'stddev', ' ', 'min', ' ', 'avg', ' ', 'stddev', ' ', 'diff']] RIGHT_ALIGN = '\x00' + l_avg1 = [] + l_avg2 = [] for name in all_names: row = [name, ''] table.append(row) raw1 = _report(row, times1.get(name)) raw2 = _report(row, times2.get(name)) if raw1 and raw2: - row.append(perf.TimeDelta(perf.avg(raw1), perf.avg(raw2))) + t_avg1 = perf.avg(raw1) + t_avg2 = perf.avg(raw2) + row.append(perf.TimeDelta(t_avg1, t_avg2)) + l_avg1.append(t_avg1) + l_avg2.append(t_avg2) + + table.append([]) + if len(l_avg1) == len(all_names): + g_avg1 = geometric_average(l_avg1) + g_avg2 = geometric_average(l_avg2) + row = ['GEOM AVG', '', + '', '', str(round(g_avg1, 3)), '', '', '', + '', '', str(round(g_avg2, 3)), '', '', '', + perf.TimeDelta(g_avg1, g_avg2)] + table.append(row) lengths = [] for row in table: From pypy.commits at gmail.com Wed Mar 8 04:51:03 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 01:51:03 -0800 (PST) Subject: [pypy-commit] benchmarks default: Baaaaaaah, BM_cpython_doc runs twice with the base python and Message-ID: <58bfd407.1b582e0a.3ad41.2c4a@mx.google.com> Author: Armin Rigo Branch: Changeset: r371:4fc1bb451a1a Date: 2017-03-08 10:50 +0100 http://bitbucket.org/pypy/benchmarks/changeset/4fc1bb451a1a/ Log: Baaaaaaah, BM_cpython_doc runs twice with the base python and zero times with the changed python. This makes *all* reported results of cpython_doc entirely bogus. diff --git a/benchmarks.py b/benchmarks.py --- a/benchmarks.py +++ b/benchmarks.py @@ -206,7 +206,7 @@ os.mkdir(docdir) htmldir = os.path.join(builddir, 'html') os.mkdir(htmldir) - args = base_python + [build, '-b', 'html', '-d', docdir, maindir, htmldir] + args = python + [build, '-b', 'html', '-d', docdir, maindir, htmldir] proc = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = proc.communicate() retcode = proc.poll() From pypy.commits at gmail.com Wed Mar 8 04:51:05 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 01:51:05 -0800 (PST) Subject: [pypy-commit] benchmarks default: Ignore extra options Message-ID: <58bfd409.14542e0a.fc9ed.2ce8@mx.google.com> Author: Armin Rigo Branch: Changeset: r372:aa97f028af85 Date: 2017-03-08 10:50 +0100 http://bitbucket.org/pypy/benchmarks/changeset/aa97f028af85/ Log: Ignore extra options diff --git a/nullpython.py b/nullpython.py --- a/nullpython.py +++ b/nullpython.py @@ -14,6 +14,8 @@ usage="%prog [options]", description="Test the performance of the Go benchmark") util.add_standard_options_to(parser) + parser.add_option('-b') # ignored, for cpython_doc + parser.add_option('-d') # ignored, for cpython_doc options, args = parser.parse_args() main = lambda n: [0.0001 for x in range(options.num_runs)] From pypy.commits at gmail.com Wed Mar 8 05:17:59 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 02:17:59 -0800 (PST) Subject: [pypy-commit] benchmarks default: fix (i.e. hack) nullpython so that all benchmarks can run with it Message-ID: <58bfda57.4121190a.db54f.2cc9@mx.google.com> Author: Armin Rigo Branch: Changeset: r373:2a8ac3f3dea5 Date: 2017-03-08 11:17 +0100 http://bitbucket.org/pypy/benchmarks/changeset/2a8ac3f3dea5/ Log: fix (i.e. hack) nullpython so that all benchmarks can run with it diff --git a/nullpython.py b/nullpython.py --- a/nullpython.py +++ b/nullpython.py @@ -7,16 +7,19 @@ in the difference to a baseline interpreter. ''' from own import util -import optparse +import optparse, sys if __name__ == '__main__': parser = optparse.OptionParser( usage="%prog [options]", - description="Test the performance of the Go benchmark") + description="Dummy") util.add_standard_options_to(parser) - parser.add_option('-b') # ignored, for cpython_doc - parser.add_option('-d') # ignored, for cpython_doc - options, args = parser.parse_args() + options, args = parser.parse_args([]) + + for i in range(1, len(sys.argv)-1): + if sys.argv[i] == '-n': + options.num_runs = int(sys.argv[i+1]) + break main = lambda n: [0.0001 for x in range(options.num_runs)] util.run_benchmark(options, options.num_runs, main) From pypy.commits at gmail.com Wed Mar 8 05:36:29 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 02:36:29 -0800 (PST) Subject: [pypy-commit] benchmarks default: Swap baseline and changed interpreters. Needed for BM_translate() Message-ID: <58bfdead.0a482e0a.b006f.2e26@mx.google.com> Author: Armin Rigo Branch: Changeset: r374:3d7dc7b9b33e Date: 2017-03-08 11:36 +0100 http://bitbucket.org/pypy/benchmarks/changeset/3d7dc7b9b33e/ Log: Swap baseline and changed interpreters. Needed for BM_translate() diff --git a/display_local.py b/display_local.py --- a/display_local.py +++ b/display_local.py @@ -7,11 +7,11 @@ two arguments, computes statistics and displays the differences. (Details: each file must result from a '--full-store' execution of -'runner.py'. The "changed_times" keys are used and the "base_times" -keys are discarded. The option '--base1' and/or '--base2' can be used -to pick the "base_times" instead in the first/second file. These -options are not useful if the files are produced by 'run_local.py' -because it uses nullpython as the base.) +'runner.py'. The "base_times" keys are used and the "changed_times" +keys are discarded. The option '--changed1' and/or '--changed2' can be +used to pick the "changed_times" instead in the first/second file. +These options are not useful if the files are produced by 'run_local.py' +because it uses nullpython as the changed interpreter.) """ import sys @@ -136,12 +136,12 @@ usage="%prog first-filename [second-filename]", description=__doc__) - parser.add_option("--base1", default=False, action="store_true", - help='Pick the "base_times" keys instead of the "changed_times"' - ' ones in the first file') - parser.add_option("--base2", default=False, action="store_true", - help='Pick the "base_times" keys instead of the "changed_times"' - ' ones in the second file') + parser.add_option("--changed1", default=False, action="store_true", + help='Pick the "changed_times" keys instead of the "base_times"' + ' keys in the first file') + parser.add_option("--changed2", default=False, action="store_true", + help='Pick the "changed_times" keys instead of the "base_times"' + ' keys in the second file') options, args = parser.parse_args(argv) if len(args) == 0: @@ -149,9 +149,9 @@ elif len(args) > 2: parser.error("too many filenames") - times1 = load_times(args[0], base_times=options.base1) + times1 = load_times(args[0], base_times=not options.changed1) if len(args) > 1: - times2 = load_times(args[1], base_times=options.base2) + times2 = load_times(args[1], base_times=not options.changed2) else: times2 = None display(times1, times2) diff --git a/run_local.py b/run_local.py --- a/run_local.py +++ b/run_local.py @@ -28,9 +28,12 @@ localdir = os.path.dirname(sys.argv[0]) or '.' +# note: we use nullpython.py as the "changed" interpreter, not the +# "baseline" one, because BM_translate() happens to run only with the +# baseline interpreter and not the changed one. Messy. cmdline = [sys.executable, os.path.join(localdir, 'runner.py'), - '--changed', pypy_c, - '--baseline', os.path.join(localdir, 'nullpython.py'), + '--baseline', pypy_c, + '--changed', os.path.join(localdir, 'nullpython.py'), '--full-store', ] + sys.argv[1:] print From pypy.commits at gmail.com Wed Mar 8 05:38:54 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 02:38:54 -0800 (PST) Subject: [pypy-commit] benchmarks default: blank line Message-ID: <58bfdf3e.4aa7190a.cdc98.2e82@mx.google.com> Author: Armin Rigo Branch: Changeset: r375:b7a6920ddff5 Date: 2017-03-08 11:38 +0100 http://bitbucket.org/pypy/benchmarks/changeset/b7a6920ddff5/ Log: blank line diff --git a/display_local.py b/display_local.py --- a/display_local.py +++ b/display_local.py @@ -113,6 +113,7 @@ '', '', str(round(g_avg2, 3)), '', '', '', perf.TimeDelta(g_avg1, g_avg2)] table.append(row) + table.append([]) lengths = [] for row in table: From pypy.commits at gmail.com Wed Mar 8 08:41:59 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 05:41:59 -0800 (PST) Subject: [pypy-commit] benchmarks default: Display the geom. average of all items dropping the 1st and last, too Message-ID: <58c00a27.18152e0a.65aa6.37ee@mx.google.com> Author: Armin Rigo Branch: Changeset: r376:8602542d6a2c Date: 2017-03-08 14:41 +0100 http://bitbucket.org/pypy/benchmarks/changeset/8602542d6a2c/ Log: Display the geom. average of all items dropping the 1st and last, too diff --git a/display_local.py b/display_local.py --- a/display_local.py +++ b/display_local.py @@ -92,6 +92,7 @@ l_avg1 = [] l_avg2 = [] + l_diff = [] for name in all_names: row = [name, ''] table.append(row) @@ -103,6 +104,7 @@ row.append(perf.TimeDelta(t_avg1, t_avg2)) l_avg1.append(t_avg1) l_avg2.append(t_avg2) + l_diff.append(t_avg1 / t_avg2) table.append([]) if len(l_avg1) == len(all_names): @@ -113,6 +115,18 @@ '', '', str(round(g_avg2, 3)), '', '', '', perf.TimeDelta(g_avg1, g_avg2)] table.append(row) + if len(l_avg1) > 3: + l_avg = zip(l_diff, l_avg1, l_avg2) + l_avg.sort() + del l_avg[0] + del l_avg[-1] + g_avg1 = geometric_average([y for x,y,z in l_avg]) + g_avg2 = geometric_average([z for x,y,z in l_avg]) + row = ['without 1st/last', '', + '', '', str(round(g_avg1, 3)), '', '', '', + '', '', str(round(g_avg2, 3)), '', '', '', + perf.TimeDelta(g_avg1, g_avg2)] + table.append(row) table.append([]) lengths = [] From pypy.commits at gmail.com Wed Mar 8 08:50:30 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 05:50:30 -0800 (PST) Subject: [pypy-commit] pypy shadowstack-perf-2: ready to merge Message-ID: <58c00c26.011c190a.e4c93.3516@mx.google.com> Author: Armin Rigo Branch: shadowstack-perf-2 Changeset: r90587:ba52573c76c3 Date: 2017-03-08 14:43 +0100 http://bitbucket.org/pypy/pypy/changeset/ba52573c76c3/ Log: ready to merge From pypy.commits at gmail.com Wed Mar 8 08:50:32 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 05:50:32 -0800 (PST) Subject: [pypy-commit] pypy default: hg merge shadowstack-perf-2 Message-ID: <58c00c28.010e2e0a.bcaf3.36bc@mx.google.com> Author: Armin Rigo Branch: Changeset: r90588:c05892e069b0 Date: 2017-03-08 14:48 +0100 http://bitbucket.org/pypy/pypy/changeset/c05892e069b0/ Log: hg merge shadowstack-perf-2 Two changes that together bring the performance of shadowstack close to asmgcc---close enough that we can now make shadowstack the default even on Linux. Yay! The changes are: * Compile with "gcc -flto" or "clang -flto". Gives a speed-up, and cannot be used with asmgcc. * Add complicated logic in shadowcolor.py to optimize the placement of the roots in the shadow stack. Long code, but should be ok because it is (hopefully) well-tested and independent. diff too long, truncating to 2000 out of 2343 lines diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -17,10 +17,10 @@ DEFL_GC = "incminimark" # XXX DEFL_ROOTFINDER_WITHJIT = "shadowstack" -if sys.platform.startswith("linux"): - _mach = os.popen('uname -m', 'r').read().strip() - if _mach.startswith('x86') or _mach in ['i386', 'i486', 'i586', 'i686']: - DEFL_ROOTFINDER_WITHJIT = "asmgcc" # only for Linux on x86 / x86-64 +## if sys.platform.startswith("linux"): +## _mach = os.popen('uname -m', 'r').read().strip() +## if _mach.startswith('x86') or _mach in ['i386', 'i486', 'i586', 'i686']: +## DEFL_ROOTFINDER_WITHJIT = "asmgcc" # only for Linux on x86 / x86-64 IS_64_BITS = sys.maxint > 2147483647 diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -96,6 +96,13 @@ from rpython.translator.tool.graphpage import FlowGraphPage FlowGraphPage(t, [self]).display() + def showbg(self, t=None): + import os + self.show(t) + if os.fork() == 0: + self.show(t) + os._exit(0) + view = show @@ -191,6 +198,11 @@ txt = "raise block" else: txt = "codeless block" + if len(self.inputargs) > 0: + if len(self.inputargs) > 1: + txt += '[%s...]' % (self.inputargs[0],) + else: + txt += '[%s]' % (self.inputargs[0],) return txt def __repr__(self): diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -341,6 +341,9 @@ # called first, to initialize self.belongs_to_current_thread. assert not hasattr(self, 'gc_detach_callback_pieces_ptr') + def postprocess_graph(self, gct, graph, any_inlining): + pass + def walk_stack_roots(self, collect_stack_root, is_minor=False): gcdata = self.gcdata gcdata._gc_collect_stack_root = collect_stack_root diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -205,6 +205,9 @@ if minimal_transform: self.need_minimal_transform(graph) if inline: + assert minimal_transform, ( + "%r has both inline=True and minimal_transform=False" + % (graph,)) self.graphs_to_inline[graph] = True return annhelper.graph2const(graph) @@ -410,7 +413,7 @@ self.identityhash_ptr = getfn(GCClass.identityhash.im_func, [s_gc, s_gcref], annmodel.SomeInteger(), - minimal_transform=False, inline=True) + minimal_transform=False) if getattr(GCClass, 'obtain_free_space', False): self.obtainfreespace_ptr = getfn(GCClass.obtain_free_space.im_func, [s_gc, annmodel.SomeInteger()], @@ -419,7 +422,6 @@ if GCClass.moving_gc: self.id_ptr = getfn(GCClass.id.im_func, [s_gc, s_gcref], annmodel.SomeInteger(), - inline = True, minimal_transform = False) else: self.id_ptr = None @@ -600,6 +602,9 @@ "the custom trace hook %r for %r can cause " "the GC to be called!" % (func, TP)) + def postprocess_graph(self, graph, any_inlining): + self.root_walker.postprocess_graph(self, graph, any_inlining) + def consider_constant(self, TYPE, value): self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc) diff --git a/rpython/memory/gctransform/shadowcolor.py b/rpython/memory/gctransform/shadowcolor.py new file mode 100644 --- /dev/null +++ b/rpython/memory/gctransform/shadowcolor.py @@ -0,0 +1,803 @@ +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.flowspace.model import mkentrymap, checkgraph, Block, Link +from rpython.flowspace.model import Variable, Constant, SpaceOperation +from rpython.tool.algo.regalloc import perform_register_allocation +from rpython.tool.algo.unionfind import UnionFind +from rpython.translator.unsimplify import varoftype, insert_empty_block +from rpython.translator.unsimplify import insert_empty_startblock, split_block +from rpython.translator.simplify import join_blocks +from rpython.rlib.rarithmetic import intmask +from collections import defaultdict + + +def is_trivial_rewrite(op): + return (op.opname in ('same_as', 'cast_pointer', 'cast_opaque_ptr') + and isinstance(op.args[0], Variable)) + + +def find_predecessors(graph, pending_pred): + """Return the set of variables whose content can end up inside one + of the 'pending_pred', which is a list of (block, var) tuples. + """ + entrymap = mkentrymap(graph) + if len(entrymap[graph.startblock]) != 1: + insert_empty_startblock(graph) + entrymap = mkentrymap(graph) + + pred = set([v for block, v in pending_pred]) + + def add(block, v): + if isinstance(v, Variable): + if v not in pred: + pending_pred.append((block, v)) + pred.add(v) + + while pending_pred: + block, v = pending_pred.pop() + if v in block.inputargs: + var_index = block.inputargs.index(v) + for link in entrymap[block]: + prevblock = link.prevblock + if prevblock is not None: + add(prevblock, link.args[var_index]) + else: + for op in block.operations: + if op.result is v: + if is_trivial_rewrite(op): + add(block, op.args[0]) + break + return pred + + +def find_successors(graph, pending_succ): + """Return the set of variables where one of the 'pending_succ' can + end up. 'block_succ' is a list of (block, var) tuples. + """ + succ = set([v for block, v in pending_succ]) + + def add(block, v): + if isinstance(v, Variable): + if v not in succ: + pending_succ.append((block, v)) + succ.add(v) + + while pending_succ: + block, v = pending_succ.pop() + for op in block.operations: + if op.args and v is op.args[0] and is_trivial_rewrite(op): + add(block, op.result) + for link in block.exits: + for i, v1 in enumerate(link.args): + if v1 is v: + add(link.target, link.target.inputargs[i]) + return succ + + +def find_interesting_variables(graph): + # Decide which variables are "interesting" or not. Interesting + # variables contain at least the ones that appear in gc_push_roots + # and gc_pop_roots. + pending_pred = [] + pending_succ = [] + interesting_vars = set() + for block in graph.iterblocks(): + for op in block.operations: + if op.opname == 'gc_push_roots': + for v in op.args: + if not isinstance(v, Variable): + continue + interesting_vars.add(v) + pending_pred.append((block, v)) + elif op.opname == 'gc_pop_roots': + for v in op.args: + if not isinstance(v, Variable): + continue + assert v in interesting_vars # must be pushed just above + pending_succ.append((block, v)) + if not interesting_vars: + return None + + # If there is a path from a gc_pop_roots(v) to a subsequent + # gc_push_roots(w) where w contains the same value as v along that + # path, then we consider all intermediate blocks along that path + # which contain a copy of the same value, and add these variables + # as "interesting", too. Formally, a variable in a block is + # "interesting" if it is both a "predecessor" and a "successor", + # where predecessors are variables which (sometimes) end in a + # gc_push_roots, and successors are variables which (sometimes) + # come from a gc_pop_roots. + pred = find_predecessors(graph, pending_pred) + succ = find_successors(graph, pending_succ) + interesting_vars |= (pred & succ) + + return interesting_vars + + +def allocate_registers(graph): + interesting_vars = find_interesting_variables(graph) + if not interesting_vars: + return None + regalloc = perform_register_allocation(graph, interesting_vars.__contains__) + assert regalloc.graph is graph + regalloc.find_num_colors() + return regalloc + + +def _gc_save_root(index, var): + c_index = Constant(index, lltype.Signed) + return SpaceOperation('gc_save_root', [c_index, var], + varoftype(lltype.Void)) + +def _gc_restore_root(index, var): + c_index = Constant(index, lltype.Signed) + return SpaceOperation('gc_restore_root', [c_index, var], + varoftype(lltype.Void)) + +def make_bitmask(filled, graph='?'): + n = filled.count(False) + if n == 0: + return (None, None) + bitmask = 0 + last_index = 0 + for i in range(len(filled)): + if not filled[i]: + bitmask <<= (i - last_index) + last_index = i + bitmask |= 1 + assert bitmask & 1 + if bitmask != intmask(bitmask): + raise GCBitmaskTooLong("the graph %r is too complex: cannot create " + "a bitmask telling than more than 31/63 " + "shadowstack entries are unused" % (graph,)) + # the mask is always a positive value, but it is replaced by a + # negative value during a minor collection root walking. Then, + # if the next minor collection finds an already-negative value, + # we know we can stop. So that's why we don't include here an + # optimization to not re-write a same-valued mask: it is important + # to re-write the value, to turn it from potentially negative back + # to positive, in order to mark this shadow frame as modified. + assert bitmask > 0 + return (last_index, bitmask) + + +def expand_one_push_roots(regalloc, args): + if regalloc is None: + assert len(args) == 0 + else: + filled = [False] * regalloc.numcolors + for v in args: + index = regalloc.getcolor(v) + assert not filled[index] + filled[index] = True + yield _gc_save_root(index, v) + bitmask_index, bitmask = make_bitmask(filled, regalloc.graph) + if bitmask_index is not None: + # xxx we might in some cases avoid this gc_save_root + # entirely, if we know we're after another gc_push/gc_pop + # that wrote exactly the same mask at the same index + bitmask_c = Constant(bitmask, lltype.Signed) + yield _gc_save_root(bitmask_index, bitmask_c) + +def expand_one_pop_roots(regalloc, args): + if regalloc is None: + assert len(args) == 0 + else: + for v in args: + index = regalloc.getcolor(v) + yield _gc_restore_root(index, v) + + +def expand_push_roots(graph, regalloc): + """Expand gc_push_roots into a series of gc_save_root, including + writing a bitmask tag to mark some entries as not-in-use. + (If regalloc is None, it will still remove empty gc_push_roots.) + """ + for block in graph.iterblocks(): + any_change = False + newops = [] + for op in block.operations: + if op.opname == 'gc_push_roots': + args = [v for v in op.args if isinstance(v, Variable)] + newops += expand_one_push_roots(regalloc, args) + any_change = True + else: + newops.append(op) + if any_change: + block.operations = newops + + +def move_pushes_earlier(graph, regalloc): + """gc_push_roots and gc_pop_roots are pushes/pops to the shadowstack, + immediately enclosing the operation that needs them (typically a call). + Here, we try to move individual pushes earlier. + + Should run after expand_push_roots(), but before expand_pop_roots(), + so that it sees individual 'gc_save_root' operations but bulk + 'gc_pop_roots' operations. + """ + # Concrete example (assembler tested on x86-64 gcc 5.3 and clang 3.7): + # + # ----original---- ----move_pushes_earlier---- + # + # while (a > 10) { *foo = b; + # *foo = b; while (a > 10) { + # a = g(a); a = g(a); + # b = *foo; b = *foo; + # // *foo = b; + # } } + # return b; return b; + # + # => the store and the => the store is before, and gcc/clang + # load are in the loop, moves the load after the loop + # even in the assembler (the commented-out '*foo=b' is removed + # here, but gcc/clang would also remove it) + + # Draft of the algorithm: see shadowcolor.txt + + if not regalloc: + return + + entrymap = mkentrymap(graph) + assert len(entrymap[graph.startblock]) == 1 + + inputvars = {} # {inputvar: (its block, its index in inputargs)} + for block in graph.iterblocks(): + for i, v in enumerate(block.inputargs): + inputvars[v] = (block, i) + + Plist = [] + + for index in range(regalloc.numcolors): + U = UnionFind() + + S = set() + for block in graph.iterblocks(): + for op in reversed(block.operations): + if op.opname == 'gc_pop_roots': + break + else: + continue # no gc_pop_roots in this block + for v in op.args: + if isinstance(v, Variable) and regalloc.checkcolor(v, index): + break + else: + continue # no variable goes into index i + + succ = set() + pending_succ = [(block, v)] + while pending_succ: + block1, v1 = pending_succ.pop() + assert regalloc.checkcolor(v1, index) + for op1 in block1.operations: + if is_trivial_rewrite(op1) and op1.args[0] is v1: + if regalloc.checkcolor(op1.result, index): + pending_succ.append((block1, op1.result)) + for link1 in block1.exits: + for i2, v2 in enumerate(link1.args): + if v2 is not v1: + continue + block2 = link1.target + w2 = block2.inputargs[i2] + if w2 in succ or not regalloc.checkcolor(w2, index): + continue + succ.add(w2) + for op2 in block2.operations: + if op2.opname in ('gc_save_root', 'gc_pop_roots'): + break + else: + pending_succ.append((block2, w2)) + U.union_list(list(succ)) + S.update(succ) + + G = defaultdict(set) + for block in graph.iterblocks(): + found = False + for opindex, op in enumerate(block.operations): + if op.opname == 'gc_save_root': + if (isinstance(op.args[1], Constant) and + op.args[1].concretetype == lltype.Signed): + break + elif op.args[0].value == index: + found = True + break + if not found or not isinstance(op.args[1], Variable): + continue # no matching gc_save_root in this block + + key = (block, op) + pred = set() + pending_pred = [(block, op.args[1], opindex)] + while pending_pred: + block1, v1, opindex1 = pending_pred.pop() + assert regalloc.getcolor(v1) == index + for i in range(opindex1-1, -1, -1): + op1 = block1.operations[i] + if op1.opname == 'gc_pop_roots': + break # stop + if op1.result is v1: + if not is_trivial_rewrite(op1): + break # stop + if not regalloc.checkcolor(op1.args[0], index): + break # stop + v1 = op1.args[0] + else: + varindex = block1.inputargs.index(v1) + if v1 in pred: + continue # already done + pred.add(v1) + for link1 in entrymap[block1]: + prevblock1 = link1.prevblock + if prevblock1 is not None: + w1 = link1.args[varindex] + if isinstance(w1, Variable) and w1 not in pred: + if regalloc.checkcolor(w1, index): + pending_pred.append((prevblock1, w1, + len(prevblock1.operations))) + U.union_list(list(pred)) + for v1 in pred: + G[v1].add(key) + + M = S.intersection(G) + + parts_target = {} + for v in M: + vp = U.find_rep(v) + if vp not in parts_target: + new_part = (index, set(), set()) + # (index, + # subset P of variables, + # set of (block, gc_save_root)) + Plist.append(new_part) + parts_target[vp] = new_part + part = parts_target[vp] + part[1].add(v) + part[2].update(G[v]) + + # Sort P so that it prefers places that would avoid multiple + # gcsaveroots (smaller 'heuristic' result, so first in sorted + # order); but also prefers smaller overall pieces, because it + # might be possible to remove several small-scale pieces instead + # of one big-scale one. + def heuristic((index, P, gcsaveroots)): + return float(len(P)) / len(gcsaveroots) + Plist.sort(key=heuristic) + + variables_along_changes = {} + live_at_start_of_block = set() # set of (block, index) + insert_gc_push_root = defaultdict(list) + + for index, P, gcsaveroots in Plist: + # if this Plist entry is not valid any more because of changes + # done by the previous entries, drop it + if any((inputvars[v][0], index) in live_at_start_of_block for v in P): + continue + if any(op not in block.operations for block, op in gcsaveroots): + continue + for v in P: + assert regalloc.getcolor(v) == index + assert v not in variables_along_changes + + success_count = 0 + mark = [] + + for v in P: + block, varindex = inputvars[v] + for link in entrymap[block]: + w = link.args[varindex] + if link.prevblock is not None: + prevoperations = link.prevblock.operations + else: + prevoperations = [] + for op in reversed(prevoperations): + if op.opname == 'gc_pop_roots': + # it is possible to have gc_pop_roots() without + # w in the args, if w is the result of the call + # that comes just before. + if (isinstance(w, Variable) and + w in op.args and + regalloc.checkcolor(w, index)): + success_count += 1 + else: + mark.append((index, link, varindex)) + break + if op.result is w: + if is_trivial_rewrite(op) and ( + regalloc.checkcolor(op.args[0], index)): + w = op.args[0] + else: + mark.append((index, link, varindex)) + break + else: + if not isinstance(w, Variable) or w not in P: + mark.append((index, link, varindex)) + + if success_count > 0: + for block, op in gcsaveroots: + newops = list(block.operations) + newops.remove(op) + block.operations = newops + for index, link, varindex in mark: + insert_gc_push_root[link].append((index, link.args[varindex])) + for v in P: + block, varindex = inputvars[v] + variables_along_changes[v] = block, index + live_at_start_of_block.add((block, index)) + + for link in insert_gc_push_root: + newops = [_gc_save_root(index, v) + for index, v in sorted(insert_gc_push_root[link])] + insert_empty_block(link, newops=newops) + + +def expand_pop_roots(graph, regalloc): + """gc_pop_roots => series of gc_restore_root; this is done after + move_pushes_earlier() because that one doesn't work correctly if + a completely-empty gc_pop_roots is removed. + + Also notice in-block code sequences like gc_pop_roots(v) followed + by a gc_save_root(v), and drop the gc_save_root. + """ + drop = {} + for block in graph.iterblocks(): + any_change = False + newops = [] + for op in block.operations: + if op.opname == 'gc_pop_roots': + args = [v for v in op.args if isinstance(v, Variable)] + expanded = list(expand_one_pop_roots(regalloc, args)) + drop = {} + for op1 in expanded: + if isinstance(op1.args[1], Variable): + drop[op1.args[1]] = op1.args[0].value + newops += expanded + any_change = True + elif (op.opname == 'gc_save_root' and + drop.get(op.args[1]) == op.args[0].value): + any_change = True # kill the operation + else: + newops.append(op) + if any_change: + block.operations = newops + + +def add_enter_leave_roots_frame(graph, regalloc, c_gcdata): + # put 'gc_enter_roots_frame' as late as possible, but before the + # first 'gc_save_root' is reached. + # + # put the 'gc_leave_roots_frame' operations as early as possible, + # that is, just after the last 'gc_restore_root' reached. This is + # done by putting it along a link, such that the previous block + # contains a 'gc_restore_root' and from the next block it is not + # possible to reach any extra 'gc_restore_root'; then, as doing + # this is not as precise as we'd like, we first break every block + # just after their last 'gc_restore_root'. + if regalloc is None: + return + + # break blocks after their last 'gc_restore_root', unless they + # are already at the last position + for block in graph.iterblocks(): + ops = block.operations + for i in range(len(ops)-1, -1, -1): + if ops[i].opname == 'gc_restore_root': + if i < len(ops) - 1: + split_block(block, i + 1) + break + # done + + insert_empty_startblock(graph) + entrymap = mkentrymap(graph) + + # helpers + + def is_interesting_op(op): + if op.opname == 'gc_restore_root': + return True + if op.opname == 'gc_save_root': + # ignore saves that say "everything is free" + return not (isinstance(op.args[1], Constant) and + isinstance(op.args[1].value, int) and + op.args[1].value == bitmask_all_free) + return False + bitmask_all_free = (1 << regalloc.numcolors) - 1 + + def insert_along_link(link, opname, args, cache): + b2 = link.target + if b2 not in cache: + newblock = Block([v.copy() for v in b2.inputargs]) + newblock.operations.append( + SpaceOperation(opname, args, varoftype(lltype.Void))) + newblock.closeblock(Link(list(newblock.inputargs), b2)) + cache[b2] = newblock + link.target = cache[b2] + + # make a list of blocks with gc_save_root/gc_restore_root in them + interesting_blocks = [] + for block in graph.iterblocks(): + for op in block.operations: + if is_interesting_op(op): + assert block is not graph.startblock + assert block is not graph.returnblock + interesting_blocks.append(block) + break # interrupt this block, go to the next one + + # compute the blocks such that 'gc_save_root/gc_restore_root' + # exist anywhere before the start of this block + before_blocks = set() + pending = list(interesting_blocks) + seen = set(pending) + while pending: + block = pending.pop() + for link in block.exits: + before_blocks.add(link.target) + if link.target not in seen: + seen.add(link.target) + pending.append(link.target) + assert graph.startblock not in before_blocks + + # compute the blocks such that 'gc_save_root/gc_restore_root' + # exist anywhere after the start of this block + after_blocks = set(interesting_blocks) + pending = list(interesting_blocks) + while pending: + block = pending.pop() + for link in entrymap[block]: + if link.prevblock is not None: + if link.prevblock not in after_blocks: + after_blocks.add(link.prevblock) + pending.append(link.prevblock) + assert graph.returnblock not in after_blocks + + # this is the set of blocks such that, at the start of the block, + # we're "in frame", i.e. there are 'gc_save_root/gc_restore_root' + # both before and after the start of the block. + inside_blocks = before_blocks & after_blocks + inside_or_interesting_blocks = set(interesting_blocks) | inside_blocks + + # if a block contains gc_save_root/gc_restore_root but is not + # an "inside_block", then add gc_enter_roots_frame where needed + c_num = Constant(regalloc.numcolors, lltype.Signed) + for block in interesting_blocks: + if block not in inside_blocks: + i = 0 + while not is_interesting_op(block.operations[i]): + i += 1 + block.operations.insert(i, + SpaceOperation('gc_enter_roots_frame', [c_gcdata, c_num], + varoftype(lltype.Void))) + + # If a link goes from a "non-inside, non-interesting block" + # straight to an "inside_block", insert a gc_enter_roots_frame + # along the link. Similarly, if a block is a "inside-or- + # interesting_block" and exits with a link going to a + # "non-inside_block", then insert a gc_leave_roots_frame along the + # link. + cache1 = {} + cache2 = {} + for block in list(graph.iterblocks()): + if block not in inside_or_interesting_blocks: + for link in block.exits: + if link.target in inside_blocks: + insert_along_link(link, 'gc_enter_roots_frame', + [c_gcdata, c_num], cache1) + else: + for link in block.exits: + if link.target not in inside_blocks: + insert_along_link(link, 'gc_leave_roots_frame', + [], cache2) + + # check all blocks not in "inside_block": they might contain a + # gc_save_root() that writes the bitmask meaning "everything is + # free". Look only before gc_enter_roots_frame, if there is one + # in that block. Remove these out-of-frame gc_save_root(). + for block in graph.iterblocks(): + if block not in inside_blocks: + newops = [] + for i, op in enumerate(block.operations): + if op.opname == 'gc_enter_roots_frame': + newops.extend(block.operations[i:]) + break + if op.opname == 'gc_save_root' and not is_interesting_op(op): + pass # don't add in newops + else: + newops.append(op) + if len(newops) < len(block.operations): + block.operations = newops + + join_blocks(graph) # for the extra new blocks made in this function + + +class GCBitmaskTooLong(Exception): + pass + +class PostProcessCheckError(Exception): + pass + +def postprocess_double_check(graph): + # Debugging only: double-check that the placement is correct. + # Assumes that every gc_restore_root() indicates that the variable + # must be saved at the given position in the shadowstack frame (in + # practice it may have moved because of the GC, but in theory it + # is still the "same" object). So we build the set of all known + # valid-in-all-paths saved locations, and check that. + + saved = {} # {var-from-inputargs: location} where location is: + # : we haven't seen this variable so far + # set-of-indexes: says where the variable is always + # saved at the start of this block + # empty-set: same as above, so: "saved nowhere" + + in_frame = {} # {block: bool}, tells if, at the start of this block, + # we're in status "frame entered" or not + + in_frame[graph.startblock] = False + pending = set([graph.startblock]) + while pending: + block = pending.pop() + locsaved = {} + currently_in_frame = in_frame[block] + if currently_in_frame: + for v in block.inputargs: + locsaved[v] = saved[v] + for op in block.operations: + if op.opname == 'gc_restore_root': + if not currently_in_frame: + raise PostProcessCheckError(graph, block, op, 'no frame!') + if isinstance(op.args[1], Constant): + continue + num = op.args[0].value + if num not in locsaved[op.args[1]]: + raise PostProcessCheckError(graph, block, op, num, locsaved) + elif op.opname == 'gc_save_root': + if not currently_in_frame: + raise PostProcessCheckError(graph, block, op, 'no frame!') + num = op.args[0].value + # first, cancel any other variable that would be saved in 'num' + for v in locsaved: + locsaved[v] = locsaved[v].difference([num]) + # + v = op.args[1] + if isinstance(v, Variable): + locsaved[v] = locsaved[v].union([num]) + else: + if v.concretetype != lltype.Signed: + locsaved[v] = locsaved.get(v, frozenset()).union([num]) + continue + bitmask = v.value + if bitmask != 1: + # cancel any variable that would be saved in any + # position shown by the bitmask, not just 'num' + assert bitmask & 1 + assert 1 < bitmask < (2< at the start of this block, this variable's + value is already saved in the frame index i (provided "success" below) + + split M_i into a partition of independent parts; add (i, P), (i, P'), ... + to the global list + + +for every (i, P), ideally in some suitable order: + + for every variable in P, for every link entering this block: + + if prevblock's corresponding variable is from the last gc_pop_roots + of that block, at index i: + + *success = True* + + elif prevblock's corresponding variable is not in P: + + mark the link + + if success: + + insert a new gc_save_root() along all links marked above; + remove the original gc_save_root + + for any P' after P that has any variables in common, kill that P' diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -3,6 +3,7 @@ from rpython.rlib.debug import ll_assert from rpython.rlib.nonconst import NonConstant from rpython.rlib import rgc +from rpython.rlib.objectmodel import specialize from rpython.rtyper import rmodel from rpython.rtyper.annlowlevel import llhelper from rpython.rtyper.lltypesystem import lltype, llmemory @@ -10,6 +11,7 @@ from rpython.memory.gctransform.framework import ( BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr) from rpython.rtyper.rbuiltin import gen_cast +from rpython.memory.gctransform.log import log class ShadowStackFrameworkGCTransformer(BaseFrameworkGCTransformer): @@ -29,30 +31,43 @@ def push_roots(self, hop, keep_current_args=False): livevars = self.get_livevars_for_roots(hop, keep_current_args) self.num_pushs += len(livevars) - if not livevars: - return [] - c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) - base_addr = hop.genop("direct_call", [self.incr_stack_ptr, c_len ], - resulttype=llmemory.Address) - for k,var in enumerate(livevars): - c_k = rmodel.inputconst(lltype.Signed, k * sizeofaddr) - v_adr = gen_cast(hop.llops, llmemory.Address, var) - hop.genop("raw_store", [base_addr, c_k, v_adr]) + hop.genop("gc_push_roots", livevars) return livevars def pop_roots(self, hop, livevars): - if not livevars: - return - c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) - base_addr = hop.genop("direct_call", [self.decr_stack_ptr, c_len ], - resulttype=llmemory.Address) - if self.gcdata.gc.moving_gc: - # for moving collectors, reload the roots into the local variables - for k,var in enumerate(livevars): - c_k = rmodel.inputconst(lltype.Signed, k * sizeofaddr) - v_newaddr = hop.genop("raw_load", [base_addr, c_k], - resulttype=llmemory.Address) - hop.genop("gc_reload_possibly_moved", [v_newaddr, var]) + hop.genop("gc_pop_roots", livevars) + # NB. we emit it even if len(livevars) == 0; this is needed for + # shadowcolor.move_pushes_earlier() + + + at specialize.call_location() +def walk_stack_root(invoke, arg0, arg1, start, addr, is_minor): + skip = 0 + while addr != start: + addr -= sizeofaddr + #XXX reintroduce support for tagged values? + #if gc.points_to_valid_gc_object(addr): + # callback(gc, addr) + + if skip & 1 == 0: + content = addr.address[0] + n = llmemory.cast_adr_to_int(content) + if n & 1 == 0: + if content: # non-0, non-odd: a regular ptr + invoke(arg0, arg1, addr) + else: + # odd number: a skip bitmask + if n > 0: # initially, an unmarked value + if is_minor: + newcontent = llmemory.cast_int_to_adr(-n) + addr.address[0] = newcontent # mark + skip = n + else: + # a marked value + if is_minor: + return + skip = -n + skip >>= 1 class ShadowStackRootWalker(BaseRootWalker): @@ -73,14 +88,8 @@ return top self.decr_stack = decr_stack - def walk_stack_root(callback, start, end): - gc = self.gc - addr = end - while addr != start: - addr -= sizeofaddr - if gc.points_to_valid_gc_object(addr): - callback(gc, addr) - self.rootstackhook = walk_stack_root + self.invoke_collect_stack_root = specialize.call_location()( + lambda arg0, arg1, addr: arg0(self.gc, addr)) self.shadow_stack_pool = ShadowStackPool(gcdata) rsd = gctransformer.root_stack_depth @@ -101,8 +110,9 @@ def walk_stack_roots(self, collect_stack_root, is_minor=False): gcdata = self.gcdata - self.rootstackhook(collect_stack_root, - gcdata.root_stack_base, gcdata.root_stack_top) + walk_stack_root(self.invoke_collect_stack_root, collect_stack_root, + None, gcdata.root_stack_base, gcdata.root_stack_top, + is_minor=is_minor) def need_thread_support(self, gctransformer, getfn): from rpython.rlib import rthread # xxx fish @@ -208,7 +218,7 @@ self.thread_setup = thread_setup self.thread_run_ptr = getfn(thread_run, [], annmodel.s_None, - inline=True, minimal_transform=False) + minimal_transform=False) self.thread_die_ptr = getfn(thread_die, [], annmodel.s_None, minimal_transform=False) # no thread_before_fork_ptr here @@ -222,6 +232,16 @@ from rpython.rlib import _stacklet_shadowstack _stacklet_shadowstack.complete_destrptr(gctransformer) + def postprocess_graph(self, gct, graph, any_inlining): + from rpython.memory.gctransform import shadowcolor + if any_inlining: + shadowcolor.postprocess_inlining(graph) + use_push_pop = shadowcolor.postprocess_graph(graph, gct.c_const_gcdata) + if use_push_pop and graph in gct.graphs_to_inline: + log.WARNING("%r is marked for later inlining, " + "but is using push/pop roots. Disabled" % (graph,)) + del gct.graphs_to_inline[graph] + # ____________________________________________________________ class ShadowStackPool(object): @@ -310,11 +330,8 @@ def customtrace(gc, obj, callback, arg): obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) - addr = obj.top - start = obj.base - while addr != start: - addr -= sizeofaddr - gc._trace_callback(callback, arg, addr) + walk_stack_root(gc._trace_callback, callback, arg, obj.base, obj.top, + is_minor=False) # xxx optimize? gc = gctransformer.gcdata.gc assert not hasattr(gc, 'custom_trace_dispatcher') diff --git a/rpython/memory/gctransform/test/test_shadowcolor.py b/rpython/memory/gctransform/test/test_shadowcolor.py new file mode 100644 --- /dev/null +++ b/rpython/memory/gctransform/test/test_shadowcolor.py @@ -0,0 +1,791 @@ +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.test.test_llinterp import gengraph +from rpython.conftest import option +from rpython.memory.gctransform.shadowcolor import * +from rpython.flowspace import model as graphmodel +from rpython.translator.simplify import join_blocks, cleanup_graph +from hypothesis import given, strategies + + +def make_graph(f, argtypes): + t, rtyper, graph = gengraph(f, argtypes, viewbefore=False) + if getattr(option, 'view', False): + graph.show() + return graph + +def nameof(v): + return v._name.rstrip('_') + +def summary(interesting_vars): + result = {} + for v in interesting_vars: + name = nameof(v) + result[name] = result.get(name, 0) + 1 + return result + +def summary_regalloc(regalloc): + result = [] + for block in regalloc.graph.iterblocks(): + print block.inputargs + for op in block.operations: + print '\t', op + blockvars = block.inputargs + [op.result for op in block.operations] + for v in blockvars: + if regalloc.consider_var(v): + result.append((nameof(v), regalloc.getcolor(v))) + print '\t\t%s: %s' % (v, regalloc.getcolor(v)) + result.sort() + return result + + +def test_find_predecessors_1(): + def f(a, b): + c = a + b + return c + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'c': 1, 'v': 1} + +def test_find_predecessors_2(): + def f(a, b): + c = a + b + while a > 0: + a -= 2 + return c + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'c': 3, 'v': 1} + +def test_find_predecessors_3(): + def f(a, b): + while b > 100: + b -= 2 + if b > 10: + c = a + b # 'c' created in this block + else: + c = a - b # 'c' created in this block + return c # 'v' is the return var + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'c': 2, 'v': 1} + +def test_find_predecessors_4(): + def f(a, b): # 'a' in the input block + while b > 100: # 'a' in the loop header block + b -= 2 # 'a' in the loop body block + if b > 10: # 'a' in the condition block + while b > 5: # nothing + b -= 2 # nothing + c = a + b # 'c' created in this block + else: + c = a + return c # 'v' is the return var + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'a': 4, 'c': 1, 'v': 1} + +def test_find_predecessors_trivial_rewrite(): + def f(a, b): # 'b' in empty startblock + while a > 100: # 'b' + a -= 2 # 'b' + c = llop.same_as(lltype.Signed, b) # 'c', 'b' + while b > 10: # 'c' + b -= 2 # 'c' + d = llop.same_as(lltype.Signed, c) # 'd', 'c' + return d # 'v' is the return var + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'b': 4, 'c': 4, 'd': 1, 'v': 1} + +def test_find_successors_1(): + def f(a, b): + return a + b + graph = make_graph(f, [int, int]) + succ = find_successors(graph, [(graph.startblock, graph.getargs()[0])]) + assert summary(succ) == {'a': 1} + +def test_find_successors_2(): + def f(a, b): + if b > 10: + return a + b + else: + return a - b + graph = make_graph(f, [int, int]) + succ = find_successors(graph, [(graph.startblock, graph.getargs()[0])]) + assert summary(succ) == {'a': 3} + +def test_find_successors_3(): + def f(a, b): + if b > 10: # 'a' condition block + a = a + b # 'a' input + while b > 100: + b -= 2 + while b > 5: # 'a' in loop header + b -= 2 # 'a' in loop body + return a * b # 'a' in product + graph = make_graph(f, [int, int]) + succ = find_successors(graph, [(graph.startblock, graph.getargs()[0])]) + assert summary(succ) == {'a': 5} + +def test_find_successors_trivial_rewrite(): + def f(a, b): # 'b' in empty startblock + while a > 100: # 'b' + a -= 2 # 'b' + c = llop.same_as(lltype.Signed, b) # 'c', 'b' + while b > 10: # 'c', 'b' + b -= 2 # 'c', 'b' + d = llop.same_as(lltype.Signed, c) # 'd', 'c' + return d # 'v' is the return var + graph = make_graph(f, [int, int]) + pred = find_successors(graph, [(graph.startblock, graph.getargs()[1])]) + assert summary(pred) == {'b': 6, 'c': 4, 'd': 1, 'v': 1} + + +def test_interesting_vars_0(): + def f(a, b): + pass + graph = make_graph(f, [llmemory.GCREF, int]) + assert not find_interesting_variables(graph) + +def test_interesting_vars_1(): + def f(a, b): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + graph = make_graph(f, [llmemory.GCREF, int]) + assert summary(find_interesting_variables(graph)) == {'a': 1} + +def test_interesting_vars_2(): + def f(a, b, c): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + while b > 0: + b -= 5 + llop.gc_push_roots(lltype.Void, c) + llop.gc_pop_roots(lltype.Void, c) + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + assert summary(find_interesting_variables(graph)) == {'a': 1, 'c': 1} + +def test_interesting_vars_3(): + def f(a, b): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + while b > 0: # 'a' remains interesting across the blocks of this loop + b -= 5 + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + graph = make_graph(f, [llmemory.GCREF, int]) + assert summary(find_interesting_variables(graph)) == {'a': 4} + +def test_allocate_registers_1(): + def f(a, b): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + while b > 0: # 'a' remains interesting across the blocks of this loop + b -= 5 + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + graph = make_graph(f, [llmemory.GCREF, int]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 0)] * 4 + +def test_allocate_registers_2(): + def f(a, b, c): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + while b > 0: + b -= 5 + llop.gc_push_roots(lltype.Void, c) + llop.gc_pop_roots(lltype.Void, c) + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 0), ('c', 0)] + +def test_allocate_registers_3(): + def f(a, b, c): + llop.gc_push_roots(lltype.Void, c, a) + llop.gc_pop_roots(lltype.Void, c, a) + while b > 0: + b -= 5 + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 1)] * 4 + [('c', 0)] + +def test_allocate_registers_4(): + def g(a, x): + return x # (or something different) + def f(a, b, c): + llop.gc_push_roots(lltype.Void, a, c) # 'a', 'c' + llop.gc_pop_roots(lltype.Void, a, c) + while b > 0: # 'a' only; 'c' not in push_roots + b -= 5 + llop.gc_push_roots(lltype.Void, a)# 'a' + d = g(a, c) + llop.gc_pop_roots(lltype.Void, a) + c = d + return c + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 1)] * 3 + [('c', 0)] + +def test_allocate_registers_5(): + def g(a, x): + return x # (or something different) + def f(a, b, c): + while b > 0: # 'a', 'c' + b -= 5 + llop.gc_push_roots(lltype.Void, a, c) # 'a', 'c' + g(a, c) + llop.gc_pop_roots(lltype.Void, a, c) + while b < 10: + b += 2 + return c + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 1)] * 2 + [('c', 0)] * 2 + + at given(strategies.lists(strategies.booleans())) +def test_make_bitmask(boollist): + index, bitmask = make_bitmask(boollist) + if index is None: + assert bitmask is None + else: + assert 0 <= index < len(boollist) + assert boollist[index] == False + assert bitmask >= 1 + while bitmask: + if bitmask & 1: + assert index >= 0 + assert boollist[index] == False + boollist[index] = True + bitmask >>= 1 + index -= 1 + assert boollist == [True] * len(boollist) + + +class FakeRegAlloc: + graph = '?' + + def __init__(self, expected_op, **colors): + self.expected_op = expected_op + self.numcolors = len(colors) + self.getcolor = colors.__getitem__ + + def check(self, got): + got = list(got) + result = [] + for spaceop in got: + assert spaceop.opname == self.expected_op + result.append((spaceop.args[0].value, spaceop.args[1])) + return result + +def test_expand_one_push_roots(): + regalloc = FakeRegAlloc('gc_save_root', a=0, b=1, c=2) + assert regalloc.check(expand_one_push_roots(regalloc, ['a', 'b', 'c'])) == [ + (0, 'a'), (1, 'b'), (2, 'c')] + assert regalloc.check(expand_one_push_roots(regalloc, ['a', 'c'])) == [ + (0, 'a'), (2, 'c'), (1, Constant(0x1, lltype.Signed))] + assert regalloc.check(expand_one_push_roots(regalloc, ['b'])) == [ + (1, 'b'), (2, Constant(0x5, lltype.Signed))] + assert regalloc.check(expand_one_push_roots(regalloc, ['a'])) == [ + (0, 'a'), (2, Constant(0x3, lltype.Signed))] + assert regalloc.check(expand_one_push_roots(regalloc, [])) == [ + (2, Constant(0x7, lltype.Signed))] + + assert list(expand_one_push_roots(None, [])) == [] + +def test_expand_one_pop_roots(): + regalloc = FakeRegAlloc('gc_restore_root', a=0, b=1, c=2) + assert regalloc.check(expand_one_pop_roots(regalloc, ['a', 'b', 'c'])) == [ + (0, 'a'), (1, 'b'), (2, 'c')] + assert regalloc.check(expand_one_pop_roots(regalloc, ['a', 'c'])) == [ + (0, 'a'), (2, 'c')] + assert regalloc.check(expand_one_pop_roots(regalloc, ['b'])) == [ + (1, 'b')] + assert regalloc.check(expand_one_pop_roots(regalloc, ['a'])) == [ + (0, 'a')] + assert regalloc.check(expand_one_pop_roots(regalloc, [])) == [] + + assert list(expand_one_pop_roots(None, [])) == [] + +def test_move_pushes_earlier_1(): + def g(a): + return a - 1 + def f(a, b): + a *= 2 + while a > 10: + llop.gc_push_roots(lltype.Void, b) + a = g(a) + llop.gc_pop_roots(lltype.Void, b) + return b + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert graphmodel.summary(graph) == { + 'int_mul': 1, + 'gc_enter_roots_frame': 1, + 'gc_save_root': 1, + 'gc_restore_root': 1, + 'int_gt': 1, + 'direct_call': 1, + 'gc_leave_roots_frame': 1, + } + assert len(graph.startblock.operations) == 3 + assert graph.startblock.operations[0].opname == 'int_mul' + assert graph.startblock.operations[1].opname == 'gc_enter_roots_frame' + assert graph.startblock.operations[2].opname == 'gc_save_root' + assert graph.startblock.operations[2].args[0].value == 0 + postprocess_double_check(graph) + +def test_move_pushes_earlier_2(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + while a > 10: + a -= 2 + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + return b + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'int_gt': 1, + 'int_sub': 1, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_remove_intrablock_push_roots(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + return b + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'direct_call': 2, + } + +PSTRUCT = lltype.Ptr(lltype.GcStruct('S')) + +def test_move_pushes_earlier_rename_1(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + c = lltype.cast_opaque_ptr(PSTRUCT, b) + while a > 10: + a -= 2 + llop.gc_push_roots(lltype.Void, c) + g(a) + llop.gc_pop_roots(lltype.Void, c) + return c + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'cast_opaque_ptr': 1, + 'int_gt': 1, + 'int_sub': 1, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_move_pushes_earlier_rename_2(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + while a > 10: + a -= 2 + c = lltype.cast_opaque_ptr(PSTRUCT, b) + llop.gc_push_roots(lltype.Void, c) + g(a) + llop.gc_pop_roots(lltype.Void, c) + return c + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'cast_opaque_ptr': 1, + 'int_gt': 1, + 'int_sub': 1, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_move_pushes_earlier_rename_3(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + while a > 10: + a -= 2 + c = lltype.cast_opaque_ptr(PSTRUCT, b) + while a > 10: + a -= 2 + llop.gc_push_roots(lltype.Void, c) + g(a) + llop.gc_pop_roots(lltype.Void, c) + return c + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'cast_opaque_ptr': 1, + 'int_gt': 2, + 'int_sub': 2, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_move_pushes_earlier_rename_4(): + def g(a): + return a - 2 + def f(a, b): + while a > 10: + b1 = lltype.cast_opaque_ptr(PSTRUCT, b) + while a > 100: + a -= 3 + b2 = lltype.cast_opaque_ptr(llmemory.GCREF, b1) + llop.gc_push_roots(lltype.Void, b2) + a = g(a) + llop.gc_pop_roots(lltype.Void, b2) + b3 = lltype.cast_opaque_ptr(PSTRUCT, b2) + while a > 100: + a -= 4 + b4 = lltype.cast_opaque_ptr(llmemory.GCREF, b3) + llop.gc_push_roots(lltype.Void, b4) + a = g(a) + llop.gc_pop_roots(lltype.Void, b4) + b5 = lltype.cast_opaque_ptr(PSTRUCT, b4) + while a > 100: + a -= 5 + b = lltype.cast_opaque_ptr(llmemory.GCREF, b5) + return b + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'cast_opaque_ptr': 6, + 'int_gt': 4, + 'int_sub': 3, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_add_leave_roots_frame_1(): + def g(b): + pass + def f(a, b): + if a & 1: + llop.gc_push_roots(lltype.Void, b) + g(b) + llop.gc_pop_roots(lltype.Void, b) + a += 5 + else: + llop.gc_push_roots(lltype.Void, b) + g(b) + llop.gc_pop_roots(lltype.Void, b) + a += 6 + #...b forgotten here, even though it is pushed/popped above + while a > 100: + a -= 3 + return a + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert len(graph.startblock.exits) == 2 + for link in graph.startblock.exits: + assert [op.opname for op in link.target.operations] == [ + 'gc_enter_roots_frame', + 'gc_save_root', + 'direct_call', + 'gc_restore_root', + 'gc_leave_roots_frame', + 'int_add'] + postprocess_double_check(graph) + +def test_add_leave_roots_frame_2(): + def g(b): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(b) + llop.gc_pop_roots(lltype.Void, b) + #...b forgotten here; the next push/pop is empty + llop.gc_push_roots(lltype.Void) + g(b) + llop.gc_pop_roots(lltype.Void) + while a > 100: + a -= 3 + return a + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert [op.opname for op in graph.startblock.operations] == [ + 'gc_enter_roots_frame', + 'gc_save_root', + 'direct_call', + 'gc_restore_root', + 'gc_leave_roots_frame', + 'direct_call'] + postprocess_double_check(graph) + +def test_bug_1(): + class W: + pass + def foo(a): + if a < 10: + return W() + else: + return None + def compare(w_a, w_b): + return W() + def fetch_compare(w_a, w_b): + return W() + def is_true(a, w_b): + return not a + def call_next(w_a): + return W() + + def f(a, w_tup): + llop.gc_push_roots(lltype.Void, w_tup) + w_key = foo(a) + llop.gc_pop_roots(lltype.Void, w_tup) + + llop.gc_push_roots(lltype.Void, w_key) + w_iter = foo(a) + llop.gc_pop_roots(lltype.Void, w_key) + + has_key = w_key is not None + hasit = False + w_maxit = None + w_max_val = None + + while True: + llop.gc_push_roots(lltype.Void, w_iter, w_key, w_maxit, w_max_val) + w_item = call_next(w_iter) + llop.gc_pop_roots(lltype.Void, w_iter, w_key, w_maxit, w_max_val) + + if has_key: + llop.gc_push_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item) + w_compare_with = fetch_compare(w_key, w_item) + llop.gc_pop_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item) + else: + w_compare_with = w_item + + if hasit: + llop.gc_push_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item, w_compare_with) + w_bool = compare(w_compare_with, w_max_val) + llop.gc_pop_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item, w_compare_with) + + llop.gc_push_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item, w_compare_with) + condition = is_true(a, w_bool) + llop.gc_pop_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item, w_compare_with) + else: + condition = True + + if condition: + hasit = True + w_maxit = w_item + w_max_val = w_compare_with + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_bug_2(): + def f(w_tup): + while True: + llop.gc_push_roots(lltype.Void, w_tup) + llop.gc_pop_roots(lltype.Void, w_tup) + + graph = make_graph(f, [llmemory.GCREF]) + assert not graph.startblock.operations + # this test is about what occurs if the startblock of the graph + # is also reached from another block. None of the 'simplify' + # functions actually remove that, but the JIT transformation can... + graph.startblock = graph.startblock.exits[0].target + + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_add_enter_roots_frame_remove_empty(): + class W: + pass + def g(): + return W() + def h(x): + pass + def k(): + pass + def f(): + llop.gc_push_roots(lltype.Void) + x = g() + llop.gc_pop_roots(lltype.Void) + llop.gc_push_roots(lltype.Void, x) + h(x) + llop.gc_pop_roots(lltype.Void, x) + llop.gc_push_roots(lltype.Void) + h(x) + llop.gc_pop_roots(lltype.Void) + llop.gc_push_roots(lltype.Void) + k() + llop.gc_pop_roots(lltype.Void) + + graph = make_graph(f, []) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert [op.opname for op in graph.startblock.operations] == [ + "direct_call", + "gc_enter_roots_frame", + "gc_save_root", + "direct_call", + "gc_restore_root", + "gc_leave_roots_frame", + "direct_call", + "direct_call", + ] + postprocess_double_check(graph) + +def test_add_enter_roots_frame_avoided(): + def g(x): + return x + def f(x, n): + if n > 100: + llop.gc_push_roots(lltype.Void, x) + g(x) + llop.gc_pop_roots(lltype.Void, x) + return x + + graph = make_graph(f, [llmemory.GCREF, int]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert [op.opname for op in graph.startblock.operations] == [ + 'int_gt', 'same_as'] + [fastpath, slowpath] = graph.startblock.exits + assert fastpath.target is graph.returnblock + block2 = slowpath.target + assert [op.opname for op in block2.operations] == [ + 'gc_enter_roots_frame', + 'gc_save_root', + 'direct_call', + 'gc_restore_root', + 'gc_leave_roots_frame'] + postprocess_double_check(graph) + +def test_fix_graph_after_inlining(): + # the graph of f looks like it inlined another graph, which itself + # would be "if x > 100: foobar()". The foobar() function is supposed + # to be the big slow-path. + def foobar(): + print 42 + def f(x): + llop.gc_push_roots(lltype.Void, x) + if x > 100: # slow-path + foobar() + llop.gc_pop_roots(lltype.Void, x) + return x + graph = make_graph(f, [int]) + postprocess_inlining(graph) + cleanup_graph(graph) + assert [op.opname for op in graph.startblock.operations] == [ + 'int_gt', 'same_as'] + [fastpath, slowpath] = graph.startblock.exits + assert fastpath.target is graph.returnblock + block2 = slowpath.target + [v] = block2.inputargs + assert block2.operations[0].opname == 'gc_push_roots' + assert block2.operations[0].args == [v] + assert block2.operations[1].opname == 'direct_call' # -> foobar + assert block2.operations[2].opname == 'gc_pop_roots' + assert block2.operations[2].args == [v] + assert len(block2.exits) == 1 + assert block2.exits[0].target is graph.returnblock diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -97,6 +97,7 @@ self.inline = inline if translator and inline: self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping() + self.raise_analyzer = RaiseAnalyzer(translator) self.graphs_to_inline = {} self.graph_dependencies = {} self.ll_finalizers_ptrs = [] @@ -113,28 +114,36 @@ self.seen_graphs.add(graph) self.minimal_transform.add(graph) - def inline_helpers(self, graphs): + def inline_helpers_into(self, graph): from rpython.translator.backendopt.inline import iter_callsites - raise_analyzer = RaiseAnalyzer(self.translator) + to_enum = [] + for called, block, i in iter_callsites(graph, None): + if called in self.graphs_to_inline: + to_enum.append(called) + any_inlining = False + for inline_graph in to_enum: + try: + inline.inline_function(self.translator, inline_graph, graph, + self.lltype_to_classdef, + self.raise_analyzer, + cleanup=False) + any_inlining = True + except inline.CannotInline as e: + print 'CANNOT INLINE:', e + print '\t%s into %s' % (inline_graph, graph) + raise # for now, make it a fatal error + cleanup_graph(graph) + if any_inlining: + constant_fold_graph(graph) + return any_inlining + + def inline_helpers_and_postprocess(self, graphs): for graph in graphs: - to_enum = [] - for called, block, i in iter_callsites(graph, None): - if called in self.graphs_to_inline: - to_enum.append(called) - must_constfold = False - for inline_graph in to_enum: - try: - inline.inline_function(self.translator, inline_graph, graph, - self.lltype_to_classdef, - raise_analyzer, - cleanup=False) - must_constfold = True - except inline.CannotInline as e: - print 'CANNOT INLINE:', e - print '\t%s into %s' % (inline_graph, graph) - cleanup_graph(graph) - if must_constfold: - constant_fold_graph(graph) + any_inlining = self.inline and self.inline_helpers_into(graph) + self.postprocess_graph(graph, any_inlining) + + def postprocess_graph(self, graph, any_inlining): + pass def compute_borrowed_vars(self, graph): # the input args are borrowed, and stay borrowed for as long as they diff --git a/rpython/rlib/_stacklet_shadowstack.py b/rpython/rlib/_stacklet_shadowstack.py --- a/rpython/rlib/_stacklet_shadowstack.py +++ b/rpython/rlib/_stacklet_shadowstack.py @@ -47,14 +47,15 @@ SIZEADDR = llmemory.sizeof(llmemory.Address) def customtrace(gc, obj, callback, arg): + from rpython.memory.gctransform.shadowstack import walk_stack_root + stacklet = llmemory.cast_adr_to_ptr(obj, STACKLET_PTR) sscopy = stacklet.s_sscopy if sscopy: length_bytes = sscopy.signed[0] - while length_bytes > 0: - addr = sscopy + length_bytes - gc._trace_callback(callback, arg, addr) - length_bytes -= SIZEADDR + walk_stack_root(gc._trace_callback, callback, arg, + sscopy + SIZEADDR, sscopy + SIZEADDR + length_bytes, + is_minor=False) lambda_customtrace = lambda: customtrace def sscopy_detach_shadow_stack(): diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py From pypy.commits at gmail.com Wed Mar 8 08:54:20 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 05:54:20 -0800 (PST) Subject: [pypy-commit] pypy default: document branch Message-ID: <58c00d0c.1a0e2e0a.ccb2e.3601@mx.google.com> Author: Armin Rigo Branch: Changeset: r90589:9e10b6b82ebf Date: 2017-03-08 14:53 +0100 http://bitbucket.org/pypy/pypy/changeset/9e10b6b82ebf/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -180,3 +180,10 @@ Improve the optimization of branchy Python code by retaining more information across failing guards. This is done by appending some carefully encoded extra information into the resume code. + +.. branch: shadowstack-perf-2 + +Two changes that together bring the performance of shadowstack close to +asmgcc---close enough that we can now make shadowstack the default even +on Linux. This should remove a whole class of rare bugs introduced by +asmgcc. From pypy.commits at gmail.com Wed Mar 8 09:07:53 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 06:07:53 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Import 'faulthandler' after we initialize 'sys.stderr', e.g. in case Message-ID: <58c01039.d426190a.f494a.356b@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90590:7b4c85cdb7e4 Date: 2017-03-08 15:07 +0100 http://bitbucket.org/pypy/pypy/changeset/7b4c85cdb7e4/ Log: Import 'faulthandler' after we initialize 'sys.stderr', e.g. in case '-v' is given diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -534,14 +534,6 @@ sys._xoptions = dict(x.split('=', 1) if '=' in x else (x, True) for x in options['_xoptions']) - if 'faulthandler' in sys.builtin_module_names: - if 'faulthandler' in sys._xoptions or os.getenv('PYTHONFAULTHANDLER'): - import faulthandler - try: - faulthandler.enable(2) # manually set to stderr - except ValueError: - pass # ignore "2 is not a valid file descriptor" - ## if not we_are_translated(): ## for key in sorted(options): ## print '%40s: %s' % (key, options[key]) @@ -578,6 +570,14 @@ io_encoding = os.getenv("PYTHONIOENCODING") if readenv else None initstdio(io_encoding, unbuffered) + if 'faulthandler' in sys.builtin_module_names: + if 'faulthandler' in sys._xoptions or os.getenv('PYTHONFAULTHANDLER'): + import faulthandler + try: + faulthandler.enable(2) # manually set to stderr + except ValueError: + pass # ignore "2 is not a valid file descriptor" + if we_are_translated(): import __pypy__ __pypy__.save_module_content_for_future_reload(sys) From pypy.commits at gmail.com Wed Mar 8 09:25:06 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 06:25:06 -0800 (PST) Subject: [pypy-commit] pypy py3.5: The -I flag should also prevent '' (or the directory containing the file Message-ID: <58c01442.97002e0a.ea706.38fa@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90591:51760c64a213 Date: 2017-03-08 15:24 +0100 http://bitbucket.org/pypy/pypy/changeset/51760c64a213/ Log: The -I flag should also prevent '' (or the directory containing the file name on the command line) from being inserted as sys.path[0]. diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -559,6 +559,7 @@ ignore_environment, quiet, verbose, + isolated, **ignored): # with PyPy in top of CPython we can only have around 100 # but we need more in the translated PyPy for the compiler package @@ -648,12 +649,14 @@ display_exception(e) success = False else: - sys.path.insert(0, '') + if not isolated: + sys.path.insert(0, '') success = run_toplevel(exec_, bytes, mainmodule.__dict__) elif run_module != 0: # handle the "-m" command # '' on sys.path is required also here - sys.path.insert(0, '') + if not isolated: + sys.path.insert(0, '') import runpy success = run_toplevel(runpy._run_module_as_main, run_module) elif run_stdin: @@ -664,7 +667,8 @@ # "site.py" file in the script's directory. Only run this if we're # executing the interactive prompt, if we're running a script we # put it's directory on sys.path - sys.path.insert(0, '') + if not isolated: + sys.path.insert(0, '') if interactive or sys.stdin.isatty(): # If stdin is a tty or if "-i" is specified, we print a @@ -715,7 +719,8 @@ filename = sys.argv[0] mainmodule.__file__ = filename mainmodule.__cached__ = None - sys.path.insert(0, sys.pypy_resolvedirof(filename)) + if not isolated: + sys.path.insert(0, sys.pypy_resolvedirof(filename)) # assume it's a pyc file only if its name says so. # CPython goes to great lengths to detect other cases # of pyc file format, but I think it's ok not to care. From pypy.commits at gmail.com Wed Mar 8 09:28:05 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 06:28:05 -0800 (PST) Subject: [pypy-commit] pypy py3.5: "python3.5 -V" writes the version to stdout Message-ID: <58c014f5.56162e0a.4ff28.37a0@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90592:da847c465160 Date: 2017-03-08 15:27 +0100 http://bitbucket.org/pypy/pypy/changeset/da847c465160/ Log: "python3.5 -V" writes the version to stdout diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -212,7 +212,7 @@ def print_version(*args): initstdio() - print ("Python", sys.version, file=sys.stderr) + print("Python", sys.version) raise SystemExit From pypy.commits at gmail.com Wed Mar 8 09:39:58 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 06:39:58 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix an issue in ctypes Message-ID: <58c017be.4f5c190a.b2790.36c7@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90593:4ceaad7f1e7d Date: 2017-03-08 15:39 +0100 http://bitbucket.org/pypy/pypy/changeset/4ceaad7f1e7d/ Log: fix an issue in ctypes diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -94,14 +94,21 @@ # array accepts very strange parameters as part of structure # or function argument... from ctypes import c_char, c_wchar - if issubclass(self._type_, (c_char, c_wchar)): - # XXX: this should maybe be stricer for py3 (c_char disallowing str?) - if isinstance(value, (bytes, str)): + if issubclass(self._type_, c_char): + if isinstance(value, bytes): if len(value) > self._length_: raise ValueError("Invalid length") value = self(*value) elif not isinstance(value, self): - raise TypeError("expected string, %s found" + raise TypeError("expected bytes, %s found" + % (value.__class__.__name__,)) + elif issubclass(self._type_, c_wchar): + if isinstance(value, str): + if len(value) > self._length_: + raise ValueError("Invalid length") + value = self(*value) + elif not isinstance(value, self): + raise TypeError("expected unicode string, %s found" % (value.__class__.__name__,)) else: if isinstance(value, tuple): From pypy.commits at gmail.com Wed Mar 8 09:56:52 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 06:56:52 -0800 (PST) Subject: [pypy-commit] pypy default: test and fix: rposix.sendfile() didn't capture errno Message-ID: <58c01bb4.065c2e0a.f1ceb.3aac@mx.google.com> Author: Armin Rigo Branch: Changeset: r90594:a88dd1282cf9 Date: 2017-03-08 15:56 +0100 http://bitbucket.org/pypy/pypy/changeset/a88dd1282cf9/ Log: test and fix: rposix.sendfile() didn't capture errno diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -2434,7 +2434,8 @@ _OFF_PTR_T = rffi.CArrayPtr(OFF_T) c_sendfile = rffi.llexternal('sendfile', [rffi.INT, rffi.INT, _OFF_PTR_T, rffi.SIZE_T], - rffi.SSIZE_T, compilation_info=sendfile_eci) + rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=sendfile_eci) def sendfile(out_fd, in_fd, offset, count): with lltype.scoped_alloc(_OFF_PTR_T.TO, 1) as p_offset: diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -693,6 +693,20 @@ s2.close() s1.close() + def test_sendfile_invalid_offset(): + from rpython.rlib import rsocket + s1, s2 = rsocket.socketpair() + relpath = 'test_sendfile_invalid_offset' + filename = str(udir.join(relpath)) + fd = os.open(filename, os.O_RDWR|os.O_CREAT, 0777) + os.write(fd, 'abcdefghij') + with py.test.raises(OSError) as excinfo: + rposix.sendfile(s1.fd, fd, -1, 5) + assert excinfo.value.errno == errno.EINVAL + os.close(fd) + s2.close() + s1.close() + if sys.platform.startswith('linux'): def test_sendfile_no_offset(): from rpython.rlib import rsocket From pypy.commits at gmail.com Wed Mar 8 09:57:13 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 06:57:13 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58c01bc9.065c2e0a.f1ceb.3aae@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90595:8ed27d2fcf73 Date: 2017-03-08 15:56 +0100 http://bitbucket.org/pypy/pypy/changeset/8ed27d2fcf73/ Log: hg merge default diff too long, truncating to 2000 out of 2501 lines diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -7,11 +7,16 @@ PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. -* **Optimize PyPy Memory Usage**: PyPy currently emits a small executable file and a large shared object file. To reduce the base interpreter size we think it would be helpful to have several shared object files that can be dynamically loaded if the module is needed. There are several other potential places where we could improve. +* **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. + Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. + 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. + The general procedure of this project is: Run both CPython and PyPy of the same Python version and + compare the memory usage (using Massif or other tools). + If PyPy consumes a lot more memory then find and resolve the issue. * **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flamgraph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). * **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -193,3 +193,10 @@ Improve the optimization of branchy Python code by retaining more information across failing guards. This is done by appending some carefully encoded extra information into the resume code. + +.. branch: shadowstack-perf-2 + +Two changes that together bring the performance of shadowstack close to +asmgcc---close enough that we can now make shadowstack the default even +on Linux. This should remove a whole class of rare bugs introduced by +asmgcc. diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -277,6 +277,14 @@ getsizeof_missing = """sys.getsizeof() is not implemented on PyPy. +First note that the CPython documentation says that this function may +raise a TypeError, so if you are seeing it, it means that the program +you are using is not correctly handling this case. + +On PyPy, though, it always raises TypeError. Before looking for +alternatives, please take a moment to read the following explanation as +to why it is the case. What you are looking for may not be possible. + A memory profiler using this function is most likely to give results inconsistent with reality on PyPy. It would be possible to have sys.getsizeof() return a number (with enough work), but that may or diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -17,10 +17,10 @@ DEFL_GC = "incminimark" # XXX DEFL_ROOTFINDER_WITHJIT = "shadowstack" -if sys.platform.startswith("linux"): - _mach = os.popen('uname -m', 'r').read().strip() - if _mach.startswith('x86') or _mach in ['i386', 'i486', 'i586', 'i686']: - DEFL_ROOTFINDER_WITHJIT = "asmgcc" # only for Linux on x86 / x86-64 +## if sys.platform.startswith("linux"): +## _mach = os.popen('uname -m', 'r').read().strip() +## if _mach.startswith('x86') or _mach in ['i386', 'i486', 'i586', 'i686']: +## DEFL_ROOTFINDER_WITHJIT = "asmgcc" # only for Linux on x86 / x86-64 IS_64_BITS = sys.maxint > 2147483647 diff --git a/rpython/doc/faq.rst b/rpython/doc/faq.rst --- a/rpython/doc/faq.rst +++ b/rpython/doc/faq.rst @@ -182,7 +182,11 @@ On the other hand, using LLVM as our JIT backend looks interesting as well --- but again we made an attempt, and it failed: LLVM has no way to -patch the generated machine code. +patch the generated machine code, and is not suited at all to tracing +JITs. Even one big method JIT trying to use LLVM `has given up`__ for +similar reasons; read that blog post for more details. + +.. __: https://webkit.org/blog/5852/introducing-the-b3-jit-compiler/ So the position of the core PyPy developers is that if anyone wants to make an N+1'th attempt with LLVM, they are welcome, and they will receive a diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -96,6 +96,13 @@ from rpython.translator.tool.graphpage import FlowGraphPage FlowGraphPage(t, [self]).display() + def showbg(self, t=None): + import os + self.show(t) + if os.fork() == 0: + self.show(t) + os._exit(0) + view = show @@ -191,6 +198,11 @@ txt = "raise block" else: txt = "codeless block" + if len(self.inputargs) > 0: + if len(self.inputargs) > 1: + txt += '[%s...]' % (self.inputargs[0],) + else: + txt += '[%s]' % (self.inputargs[0],) return txt def __repr__(self): diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -362,9 +362,8 @@ res_loc = arglocs[1] # cond_call_value else: res_loc = None # cond_call - # useless to list res_loc in the gcmap, because if the call is - # done it means res_loc was initially NULL - gcmap = regalloc.get_gcmap([call_loc]) + # see x86.regalloc for why we skip res_loc in the gcmap + gcmap = regalloc.get_gcmap([res_loc]) assert call_loc is r.r4 jmp_adr = self.mc.currpos() diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -666,7 +666,7 @@ jmp_adr = self.mc.get_relative_pos() self.mc.trap() # patched later to a 'bc' - self.load_gcmap(self.mc, r.r2, regalloc.get_gcmap()) + self.load_gcmap(self.mc, r.r2, regalloc.get_gcmap([resloc])) # save away r3, r4, r5, r6, r12 into the jitframe should_be_saved = [ diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -998,7 +998,7 @@ size_box = op.getarg(0) assert isinstance(size_box, ConstInt) size = size_box.getint() - # hint: try to move unrelated registers away from eax and edx now + # hint: try to move unrelated registers away from ecx and edx now self.rm.spill_or_move_registers_before_call([ecx, edx]) # the result will be in ecx self.rm.force_allocate_reg(op, selected_reg=ecx) diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -390,7 +390,7 @@ if reg in self._COND_CALL_SAVE_REGS] self._push_core_regs_to_jitframe(self.mc, should_be_saved) - self.push_gcmap(self.mc, regalloc.get_gcmap()) + self.push_gcmap(self.mc, regalloc.get_gcmap([resloc])) # # load the 0-to-4 arguments into these registers, with the address of # the function to call into r11 diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -341,6 +341,9 @@ # called first, to initialize self.belongs_to_current_thread. assert not hasattr(self, 'gc_detach_callback_pieces_ptr') + def postprocess_graph(self, gct, graph, any_inlining): + pass + def walk_stack_roots(self, collect_stack_root, is_minor=False): gcdata = self.gcdata gcdata._gc_collect_stack_root = collect_stack_root diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -205,6 +205,9 @@ if minimal_transform: self.need_minimal_transform(graph) if inline: + assert minimal_transform, ( + "%r has both inline=True and minimal_transform=False" + % (graph,)) self.graphs_to_inline[graph] = True return annhelper.graph2const(graph) @@ -410,7 +413,7 @@ self.identityhash_ptr = getfn(GCClass.identityhash.im_func, [s_gc, s_gcref], annmodel.SomeInteger(), - minimal_transform=False, inline=True) + minimal_transform=False) if getattr(GCClass, 'obtain_free_space', False): self.obtainfreespace_ptr = getfn(GCClass.obtain_free_space.im_func, [s_gc, annmodel.SomeInteger()], @@ -419,7 +422,6 @@ if GCClass.moving_gc: self.id_ptr = getfn(GCClass.id.im_func, [s_gc, s_gcref], annmodel.SomeInteger(), - inline = True, minimal_transform = False) else: self.id_ptr = None @@ -600,6 +602,9 @@ "the custom trace hook %r for %r can cause " "the GC to be called!" % (func, TP)) + def postprocess_graph(self, graph, any_inlining): + self.root_walker.postprocess_graph(self, graph, any_inlining) + def consider_constant(self, TYPE, value): self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc) diff --git a/rpython/memory/gctransform/shadowcolor.py b/rpython/memory/gctransform/shadowcolor.py new file mode 100644 --- /dev/null +++ b/rpython/memory/gctransform/shadowcolor.py @@ -0,0 +1,803 @@ +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.flowspace.model import mkentrymap, checkgraph, Block, Link +from rpython.flowspace.model import Variable, Constant, SpaceOperation +from rpython.tool.algo.regalloc import perform_register_allocation +from rpython.tool.algo.unionfind import UnionFind +from rpython.translator.unsimplify import varoftype, insert_empty_block +from rpython.translator.unsimplify import insert_empty_startblock, split_block +from rpython.translator.simplify import join_blocks +from rpython.rlib.rarithmetic import intmask +from collections import defaultdict + + +def is_trivial_rewrite(op): + return (op.opname in ('same_as', 'cast_pointer', 'cast_opaque_ptr') + and isinstance(op.args[0], Variable)) + + +def find_predecessors(graph, pending_pred): + """Return the set of variables whose content can end up inside one + of the 'pending_pred', which is a list of (block, var) tuples. + """ + entrymap = mkentrymap(graph) + if len(entrymap[graph.startblock]) != 1: + insert_empty_startblock(graph) + entrymap = mkentrymap(graph) + + pred = set([v for block, v in pending_pred]) + + def add(block, v): + if isinstance(v, Variable): + if v not in pred: + pending_pred.append((block, v)) + pred.add(v) + + while pending_pred: + block, v = pending_pred.pop() + if v in block.inputargs: + var_index = block.inputargs.index(v) + for link in entrymap[block]: + prevblock = link.prevblock + if prevblock is not None: + add(prevblock, link.args[var_index]) + else: + for op in block.operations: + if op.result is v: + if is_trivial_rewrite(op): + add(block, op.args[0]) + break + return pred + + +def find_successors(graph, pending_succ): + """Return the set of variables where one of the 'pending_succ' can + end up. 'block_succ' is a list of (block, var) tuples. + """ + succ = set([v for block, v in pending_succ]) + + def add(block, v): + if isinstance(v, Variable): + if v not in succ: + pending_succ.append((block, v)) + succ.add(v) + + while pending_succ: + block, v = pending_succ.pop() + for op in block.operations: + if op.args and v is op.args[0] and is_trivial_rewrite(op): + add(block, op.result) + for link in block.exits: + for i, v1 in enumerate(link.args): + if v1 is v: + add(link.target, link.target.inputargs[i]) + return succ + + +def find_interesting_variables(graph): + # Decide which variables are "interesting" or not. Interesting + # variables contain at least the ones that appear in gc_push_roots + # and gc_pop_roots. + pending_pred = [] + pending_succ = [] + interesting_vars = set() + for block in graph.iterblocks(): + for op in block.operations: + if op.opname == 'gc_push_roots': + for v in op.args: + if not isinstance(v, Variable): + continue + interesting_vars.add(v) + pending_pred.append((block, v)) + elif op.opname == 'gc_pop_roots': + for v in op.args: + if not isinstance(v, Variable): + continue + assert v in interesting_vars # must be pushed just above + pending_succ.append((block, v)) + if not interesting_vars: + return None + + # If there is a path from a gc_pop_roots(v) to a subsequent + # gc_push_roots(w) where w contains the same value as v along that + # path, then we consider all intermediate blocks along that path + # which contain a copy of the same value, and add these variables + # as "interesting", too. Formally, a variable in a block is + # "interesting" if it is both a "predecessor" and a "successor", + # where predecessors are variables which (sometimes) end in a + # gc_push_roots, and successors are variables which (sometimes) + # come from a gc_pop_roots. + pred = find_predecessors(graph, pending_pred) + succ = find_successors(graph, pending_succ) + interesting_vars |= (pred & succ) + + return interesting_vars + + +def allocate_registers(graph): + interesting_vars = find_interesting_variables(graph) + if not interesting_vars: + return None + regalloc = perform_register_allocation(graph, interesting_vars.__contains__) + assert regalloc.graph is graph + regalloc.find_num_colors() + return regalloc + + +def _gc_save_root(index, var): + c_index = Constant(index, lltype.Signed) + return SpaceOperation('gc_save_root', [c_index, var], + varoftype(lltype.Void)) + +def _gc_restore_root(index, var): + c_index = Constant(index, lltype.Signed) + return SpaceOperation('gc_restore_root', [c_index, var], + varoftype(lltype.Void)) + +def make_bitmask(filled, graph='?'): + n = filled.count(False) + if n == 0: + return (None, None) + bitmask = 0 + last_index = 0 + for i in range(len(filled)): + if not filled[i]: + bitmask <<= (i - last_index) + last_index = i + bitmask |= 1 + assert bitmask & 1 + if bitmask != intmask(bitmask): + raise GCBitmaskTooLong("the graph %r is too complex: cannot create " + "a bitmask telling than more than 31/63 " + "shadowstack entries are unused" % (graph,)) + # the mask is always a positive value, but it is replaced by a + # negative value during a minor collection root walking. Then, + # if the next minor collection finds an already-negative value, + # we know we can stop. So that's why we don't include here an + # optimization to not re-write a same-valued mask: it is important + # to re-write the value, to turn it from potentially negative back + # to positive, in order to mark this shadow frame as modified. + assert bitmask > 0 + return (last_index, bitmask) + + +def expand_one_push_roots(regalloc, args): + if regalloc is None: + assert len(args) == 0 + else: + filled = [False] * regalloc.numcolors + for v in args: + index = regalloc.getcolor(v) + assert not filled[index] + filled[index] = True + yield _gc_save_root(index, v) + bitmask_index, bitmask = make_bitmask(filled, regalloc.graph) + if bitmask_index is not None: + # xxx we might in some cases avoid this gc_save_root + # entirely, if we know we're after another gc_push/gc_pop + # that wrote exactly the same mask at the same index + bitmask_c = Constant(bitmask, lltype.Signed) + yield _gc_save_root(bitmask_index, bitmask_c) + +def expand_one_pop_roots(regalloc, args): + if regalloc is None: + assert len(args) == 0 + else: + for v in args: + index = regalloc.getcolor(v) + yield _gc_restore_root(index, v) + + +def expand_push_roots(graph, regalloc): + """Expand gc_push_roots into a series of gc_save_root, including + writing a bitmask tag to mark some entries as not-in-use. + (If regalloc is None, it will still remove empty gc_push_roots.) + """ + for block in graph.iterblocks(): + any_change = False + newops = [] + for op in block.operations: + if op.opname == 'gc_push_roots': + args = [v for v in op.args if isinstance(v, Variable)] + newops += expand_one_push_roots(regalloc, args) + any_change = True + else: + newops.append(op) + if any_change: + block.operations = newops + + +def move_pushes_earlier(graph, regalloc): + """gc_push_roots and gc_pop_roots are pushes/pops to the shadowstack, + immediately enclosing the operation that needs them (typically a call). + Here, we try to move individual pushes earlier. + + Should run after expand_push_roots(), but before expand_pop_roots(), + so that it sees individual 'gc_save_root' operations but bulk + 'gc_pop_roots' operations. + """ + # Concrete example (assembler tested on x86-64 gcc 5.3 and clang 3.7): + # + # ----original---- ----move_pushes_earlier---- + # + # while (a > 10) { *foo = b; + # *foo = b; while (a > 10) { + # a = g(a); a = g(a); + # b = *foo; b = *foo; + # // *foo = b; + # } } + # return b; return b; + # + # => the store and the => the store is before, and gcc/clang + # load are in the loop, moves the load after the loop + # even in the assembler (the commented-out '*foo=b' is removed + # here, but gcc/clang would also remove it) + + # Draft of the algorithm: see shadowcolor.txt + + if not regalloc: + return + + entrymap = mkentrymap(graph) + assert len(entrymap[graph.startblock]) == 1 + + inputvars = {} # {inputvar: (its block, its index in inputargs)} + for block in graph.iterblocks(): + for i, v in enumerate(block.inputargs): + inputvars[v] = (block, i) + + Plist = [] + + for index in range(regalloc.numcolors): + U = UnionFind() + + S = set() + for block in graph.iterblocks(): + for op in reversed(block.operations): + if op.opname == 'gc_pop_roots': + break + else: + continue # no gc_pop_roots in this block + for v in op.args: + if isinstance(v, Variable) and regalloc.checkcolor(v, index): + break + else: + continue # no variable goes into index i + + succ = set() + pending_succ = [(block, v)] + while pending_succ: + block1, v1 = pending_succ.pop() + assert regalloc.checkcolor(v1, index) + for op1 in block1.operations: + if is_trivial_rewrite(op1) and op1.args[0] is v1: + if regalloc.checkcolor(op1.result, index): + pending_succ.append((block1, op1.result)) + for link1 in block1.exits: + for i2, v2 in enumerate(link1.args): + if v2 is not v1: + continue + block2 = link1.target + w2 = block2.inputargs[i2] + if w2 in succ or not regalloc.checkcolor(w2, index): + continue + succ.add(w2) + for op2 in block2.operations: + if op2.opname in ('gc_save_root', 'gc_pop_roots'): + break + else: + pending_succ.append((block2, w2)) + U.union_list(list(succ)) + S.update(succ) + + G = defaultdict(set) + for block in graph.iterblocks(): + found = False + for opindex, op in enumerate(block.operations): + if op.opname == 'gc_save_root': + if (isinstance(op.args[1], Constant) and + op.args[1].concretetype == lltype.Signed): + break + elif op.args[0].value == index: + found = True + break + if not found or not isinstance(op.args[1], Variable): + continue # no matching gc_save_root in this block + + key = (block, op) + pred = set() + pending_pred = [(block, op.args[1], opindex)] + while pending_pred: + block1, v1, opindex1 = pending_pred.pop() + assert regalloc.getcolor(v1) == index + for i in range(opindex1-1, -1, -1): + op1 = block1.operations[i] + if op1.opname == 'gc_pop_roots': + break # stop + if op1.result is v1: + if not is_trivial_rewrite(op1): + break # stop + if not regalloc.checkcolor(op1.args[0], index): + break # stop + v1 = op1.args[0] + else: + varindex = block1.inputargs.index(v1) + if v1 in pred: + continue # already done + pred.add(v1) + for link1 in entrymap[block1]: + prevblock1 = link1.prevblock + if prevblock1 is not None: + w1 = link1.args[varindex] + if isinstance(w1, Variable) and w1 not in pred: + if regalloc.checkcolor(w1, index): + pending_pred.append((prevblock1, w1, + len(prevblock1.operations))) + U.union_list(list(pred)) + for v1 in pred: + G[v1].add(key) + + M = S.intersection(G) + + parts_target = {} + for v in M: + vp = U.find_rep(v) + if vp not in parts_target: + new_part = (index, set(), set()) + # (index, + # subset P of variables, + # set of (block, gc_save_root)) + Plist.append(new_part) + parts_target[vp] = new_part + part = parts_target[vp] + part[1].add(v) + part[2].update(G[v]) + + # Sort P so that it prefers places that would avoid multiple + # gcsaveroots (smaller 'heuristic' result, so first in sorted + # order); but also prefers smaller overall pieces, because it + # might be possible to remove several small-scale pieces instead + # of one big-scale one. + def heuristic((index, P, gcsaveroots)): + return float(len(P)) / len(gcsaveroots) + Plist.sort(key=heuristic) + + variables_along_changes = {} + live_at_start_of_block = set() # set of (block, index) + insert_gc_push_root = defaultdict(list) + + for index, P, gcsaveroots in Plist: + # if this Plist entry is not valid any more because of changes + # done by the previous entries, drop it + if any((inputvars[v][0], index) in live_at_start_of_block for v in P): + continue + if any(op not in block.operations for block, op in gcsaveroots): + continue + for v in P: + assert regalloc.getcolor(v) == index + assert v not in variables_along_changes + + success_count = 0 + mark = [] + + for v in P: + block, varindex = inputvars[v] + for link in entrymap[block]: + w = link.args[varindex] + if link.prevblock is not None: + prevoperations = link.prevblock.operations + else: + prevoperations = [] + for op in reversed(prevoperations): + if op.opname == 'gc_pop_roots': + # it is possible to have gc_pop_roots() without + # w in the args, if w is the result of the call + # that comes just before. + if (isinstance(w, Variable) and + w in op.args and + regalloc.checkcolor(w, index)): + success_count += 1 + else: + mark.append((index, link, varindex)) + break + if op.result is w: + if is_trivial_rewrite(op) and ( + regalloc.checkcolor(op.args[0], index)): + w = op.args[0] + else: + mark.append((index, link, varindex)) + break + else: + if not isinstance(w, Variable) or w not in P: + mark.append((index, link, varindex)) + + if success_count > 0: + for block, op in gcsaveroots: + newops = list(block.operations) + newops.remove(op) + block.operations = newops + for index, link, varindex in mark: + insert_gc_push_root[link].append((index, link.args[varindex])) + for v in P: + block, varindex = inputvars[v] + variables_along_changes[v] = block, index + live_at_start_of_block.add((block, index)) + + for link in insert_gc_push_root: + newops = [_gc_save_root(index, v) + for index, v in sorted(insert_gc_push_root[link])] + insert_empty_block(link, newops=newops) + + +def expand_pop_roots(graph, regalloc): + """gc_pop_roots => series of gc_restore_root; this is done after + move_pushes_earlier() because that one doesn't work correctly if + a completely-empty gc_pop_roots is removed. + + Also notice in-block code sequences like gc_pop_roots(v) followed + by a gc_save_root(v), and drop the gc_save_root. + """ + drop = {} + for block in graph.iterblocks(): + any_change = False + newops = [] + for op in block.operations: + if op.opname == 'gc_pop_roots': + args = [v for v in op.args if isinstance(v, Variable)] + expanded = list(expand_one_pop_roots(regalloc, args)) + drop = {} + for op1 in expanded: + if isinstance(op1.args[1], Variable): + drop[op1.args[1]] = op1.args[0].value + newops += expanded + any_change = True + elif (op.opname == 'gc_save_root' and + drop.get(op.args[1]) == op.args[0].value): + any_change = True # kill the operation + else: + newops.append(op) + if any_change: + block.operations = newops + + +def add_enter_leave_roots_frame(graph, regalloc, c_gcdata): + # put 'gc_enter_roots_frame' as late as possible, but before the + # first 'gc_save_root' is reached. + # + # put the 'gc_leave_roots_frame' operations as early as possible, + # that is, just after the last 'gc_restore_root' reached. This is + # done by putting it along a link, such that the previous block + # contains a 'gc_restore_root' and from the next block it is not + # possible to reach any extra 'gc_restore_root'; then, as doing + # this is not as precise as we'd like, we first break every block + # just after their last 'gc_restore_root'. + if regalloc is None: + return + + # break blocks after their last 'gc_restore_root', unless they + # are already at the last position + for block in graph.iterblocks(): + ops = block.operations + for i in range(len(ops)-1, -1, -1): + if ops[i].opname == 'gc_restore_root': + if i < len(ops) - 1: + split_block(block, i + 1) + break + # done + + insert_empty_startblock(graph) + entrymap = mkentrymap(graph) + + # helpers + + def is_interesting_op(op): + if op.opname == 'gc_restore_root': + return True + if op.opname == 'gc_save_root': + # ignore saves that say "everything is free" + return not (isinstance(op.args[1], Constant) and + isinstance(op.args[1].value, int) and + op.args[1].value == bitmask_all_free) + return False + bitmask_all_free = (1 << regalloc.numcolors) - 1 + + def insert_along_link(link, opname, args, cache): + b2 = link.target + if b2 not in cache: + newblock = Block([v.copy() for v in b2.inputargs]) + newblock.operations.append( + SpaceOperation(opname, args, varoftype(lltype.Void))) + newblock.closeblock(Link(list(newblock.inputargs), b2)) + cache[b2] = newblock + link.target = cache[b2] + + # make a list of blocks with gc_save_root/gc_restore_root in them + interesting_blocks = [] + for block in graph.iterblocks(): + for op in block.operations: + if is_interesting_op(op): + assert block is not graph.startblock + assert block is not graph.returnblock + interesting_blocks.append(block) + break # interrupt this block, go to the next one + + # compute the blocks such that 'gc_save_root/gc_restore_root' + # exist anywhere before the start of this block + before_blocks = set() + pending = list(interesting_blocks) + seen = set(pending) + while pending: + block = pending.pop() + for link in block.exits: + before_blocks.add(link.target) + if link.target not in seen: + seen.add(link.target) + pending.append(link.target) + assert graph.startblock not in before_blocks + + # compute the blocks such that 'gc_save_root/gc_restore_root' + # exist anywhere after the start of this block + after_blocks = set(interesting_blocks) + pending = list(interesting_blocks) + while pending: + block = pending.pop() + for link in entrymap[block]: + if link.prevblock is not None: + if link.prevblock not in after_blocks: + after_blocks.add(link.prevblock) + pending.append(link.prevblock) + assert graph.returnblock not in after_blocks + + # this is the set of blocks such that, at the start of the block, + # we're "in frame", i.e. there are 'gc_save_root/gc_restore_root' + # both before and after the start of the block. + inside_blocks = before_blocks & after_blocks + inside_or_interesting_blocks = set(interesting_blocks) | inside_blocks + + # if a block contains gc_save_root/gc_restore_root but is not + # an "inside_block", then add gc_enter_roots_frame where needed + c_num = Constant(regalloc.numcolors, lltype.Signed) + for block in interesting_blocks: + if block not in inside_blocks: + i = 0 + while not is_interesting_op(block.operations[i]): + i += 1 + block.operations.insert(i, + SpaceOperation('gc_enter_roots_frame', [c_gcdata, c_num], + varoftype(lltype.Void))) + + # If a link goes from a "non-inside, non-interesting block" + # straight to an "inside_block", insert a gc_enter_roots_frame + # along the link. Similarly, if a block is a "inside-or- + # interesting_block" and exits with a link going to a + # "non-inside_block", then insert a gc_leave_roots_frame along the + # link. + cache1 = {} + cache2 = {} + for block in list(graph.iterblocks()): + if block not in inside_or_interesting_blocks: + for link in block.exits: + if link.target in inside_blocks: + insert_along_link(link, 'gc_enter_roots_frame', + [c_gcdata, c_num], cache1) + else: + for link in block.exits: + if link.target not in inside_blocks: + insert_along_link(link, 'gc_leave_roots_frame', + [], cache2) + + # check all blocks not in "inside_block": they might contain a + # gc_save_root() that writes the bitmask meaning "everything is + # free". Look only before gc_enter_roots_frame, if there is one + # in that block. Remove these out-of-frame gc_save_root(). + for block in graph.iterblocks(): + if block not in inside_blocks: + newops = [] + for i, op in enumerate(block.operations): + if op.opname == 'gc_enter_roots_frame': + newops.extend(block.operations[i:]) + break + if op.opname == 'gc_save_root' and not is_interesting_op(op): + pass # don't add in newops + else: + newops.append(op) + if len(newops) < len(block.operations): + block.operations = newops + + join_blocks(graph) # for the extra new blocks made in this function + + +class GCBitmaskTooLong(Exception): + pass + +class PostProcessCheckError(Exception): + pass + +def postprocess_double_check(graph): + # Debugging only: double-check that the placement is correct. + # Assumes that every gc_restore_root() indicates that the variable + # must be saved at the given position in the shadowstack frame (in + # practice it may have moved because of the GC, but in theory it + # is still the "same" object). So we build the set of all known + # valid-in-all-paths saved locations, and check that. + + saved = {} # {var-from-inputargs: location} where location is: + # : we haven't seen this variable so far + # set-of-indexes: says where the variable is always + # saved at the start of this block + # empty-set: same as above, so: "saved nowhere" + + in_frame = {} # {block: bool}, tells if, at the start of this block, + # we're in status "frame entered" or not + + in_frame[graph.startblock] = False + pending = set([graph.startblock]) + while pending: + block = pending.pop() + locsaved = {} + currently_in_frame = in_frame[block] + if currently_in_frame: + for v in block.inputargs: + locsaved[v] = saved[v] + for op in block.operations: + if op.opname == 'gc_restore_root': + if not currently_in_frame: + raise PostProcessCheckError(graph, block, op, 'no frame!') + if isinstance(op.args[1], Constant): + continue + num = op.args[0].value + if num not in locsaved[op.args[1]]: + raise PostProcessCheckError(graph, block, op, num, locsaved) + elif op.opname == 'gc_save_root': + if not currently_in_frame: + raise PostProcessCheckError(graph, block, op, 'no frame!') + num = op.args[0].value + # first, cancel any other variable that would be saved in 'num' + for v in locsaved: + locsaved[v] = locsaved[v].difference([num]) + # + v = op.args[1] + if isinstance(v, Variable): + locsaved[v] = locsaved[v].union([num]) + else: + if v.concretetype != lltype.Signed: + locsaved[v] = locsaved.get(v, frozenset()).union([num]) + continue + bitmask = v.value + if bitmask != 1: + # cancel any variable that would be saved in any + # position shown by the bitmask, not just 'num' + assert bitmask & 1 + assert 1 < bitmask < (2< at the start of this block, this variable's + value is already saved in the frame index i (provided "success" below) + + split M_i into a partition of independent parts; add (i, P), (i, P'), ... + to the global list + + +for every (i, P), ideally in some suitable order: + + for every variable in P, for every link entering this block: + + if prevblock's corresponding variable is from the last gc_pop_roots + of that block, at index i: + + *success = True* + + elif prevblock's corresponding variable is not in P: + + mark the link + + if success: + + insert a new gc_save_root() along all links marked above; + remove the original gc_save_root + + for any P' after P that has any variables in common, kill that P' diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -3,6 +3,7 @@ from rpython.rlib.debug import ll_assert from rpython.rlib.nonconst import NonConstant from rpython.rlib import rgc +from rpython.rlib.objectmodel import specialize from rpython.rtyper import rmodel from rpython.rtyper.annlowlevel import llhelper from rpython.rtyper.lltypesystem import lltype, llmemory @@ -10,6 +11,7 @@ from rpython.memory.gctransform.framework import ( BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr) from rpython.rtyper.rbuiltin import gen_cast +from rpython.memory.gctransform.log import log class ShadowStackFrameworkGCTransformer(BaseFrameworkGCTransformer): @@ -29,30 +31,43 @@ def push_roots(self, hop, keep_current_args=False): livevars = self.get_livevars_for_roots(hop, keep_current_args) self.num_pushs += len(livevars) - if not livevars: - return [] - c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) - base_addr = hop.genop("direct_call", [self.incr_stack_ptr, c_len ], - resulttype=llmemory.Address) - for k,var in enumerate(livevars): - c_k = rmodel.inputconst(lltype.Signed, k * sizeofaddr) - v_adr = gen_cast(hop.llops, llmemory.Address, var) - hop.genop("raw_store", [base_addr, c_k, v_adr]) + hop.genop("gc_push_roots", livevars) return livevars def pop_roots(self, hop, livevars): - if not livevars: - return - c_len = rmodel.inputconst(lltype.Signed, len(livevars) ) - base_addr = hop.genop("direct_call", [self.decr_stack_ptr, c_len ], - resulttype=llmemory.Address) - if self.gcdata.gc.moving_gc: - # for moving collectors, reload the roots into the local variables - for k,var in enumerate(livevars): - c_k = rmodel.inputconst(lltype.Signed, k * sizeofaddr) - v_newaddr = hop.genop("raw_load", [base_addr, c_k], - resulttype=llmemory.Address) - hop.genop("gc_reload_possibly_moved", [v_newaddr, var]) + hop.genop("gc_pop_roots", livevars) + # NB. we emit it even if len(livevars) == 0; this is needed for + # shadowcolor.move_pushes_earlier() + + + at specialize.call_location() +def walk_stack_root(invoke, arg0, arg1, start, addr, is_minor): + skip = 0 + while addr != start: + addr -= sizeofaddr + #XXX reintroduce support for tagged values? + #if gc.points_to_valid_gc_object(addr): + # callback(gc, addr) + + if skip & 1 == 0: + content = addr.address[0] + n = llmemory.cast_adr_to_int(content) + if n & 1 == 0: + if content: # non-0, non-odd: a regular ptr + invoke(arg0, arg1, addr) + else: + # odd number: a skip bitmask + if n > 0: # initially, an unmarked value + if is_minor: + newcontent = llmemory.cast_int_to_adr(-n) + addr.address[0] = newcontent # mark + skip = n + else: + # a marked value + if is_minor: + return + skip = -n + skip >>= 1 class ShadowStackRootWalker(BaseRootWalker): @@ -73,14 +88,8 @@ return top self.decr_stack = decr_stack - def walk_stack_root(callback, start, end): - gc = self.gc - addr = end - while addr != start: - addr -= sizeofaddr - if gc.points_to_valid_gc_object(addr): - callback(gc, addr) - self.rootstackhook = walk_stack_root + self.invoke_collect_stack_root = specialize.call_location()( + lambda arg0, arg1, addr: arg0(self.gc, addr)) self.shadow_stack_pool = ShadowStackPool(gcdata) rsd = gctransformer.root_stack_depth @@ -101,8 +110,9 @@ def walk_stack_roots(self, collect_stack_root, is_minor=False): gcdata = self.gcdata - self.rootstackhook(collect_stack_root, - gcdata.root_stack_base, gcdata.root_stack_top) + walk_stack_root(self.invoke_collect_stack_root, collect_stack_root, + None, gcdata.root_stack_base, gcdata.root_stack_top, + is_minor=is_minor) def need_thread_support(self, gctransformer, getfn): from rpython.rlib import rthread # xxx fish @@ -208,7 +218,7 @@ self.thread_setup = thread_setup self.thread_run_ptr = getfn(thread_run, [], annmodel.s_None, - inline=True, minimal_transform=False) + minimal_transform=False) self.thread_die_ptr = getfn(thread_die, [], annmodel.s_None, minimal_transform=False) # no thread_before_fork_ptr here @@ -222,6 +232,16 @@ from rpython.rlib import _stacklet_shadowstack _stacklet_shadowstack.complete_destrptr(gctransformer) + def postprocess_graph(self, gct, graph, any_inlining): + from rpython.memory.gctransform import shadowcolor + if any_inlining: + shadowcolor.postprocess_inlining(graph) + use_push_pop = shadowcolor.postprocess_graph(graph, gct.c_const_gcdata) + if use_push_pop and graph in gct.graphs_to_inline: + log.WARNING("%r is marked for later inlining, " + "but is using push/pop roots. Disabled" % (graph,)) + del gct.graphs_to_inline[graph] + # ____________________________________________________________ class ShadowStackPool(object): @@ -310,11 +330,8 @@ def customtrace(gc, obj, callback, arg): obj = llmemory.cast_adr_to_ptr(obj, SHADOWSTACKREFPTR) - addr = obj.top - start = obj.base - while addr != start: - addr -= sizeofaddr - gc._trace_callback(callback, arg, addr) + walk_stack_root(gc._trace_callback, callback, arg, obj.base, obj.top, + is_minor=False) # xxx optimize? gc = gctransformer.gcdata.gc assert not hasattr(gc, 'custom_trace_dispatcher') diff --git a/rpython/memory/gctransform/test/test_shadowcolor.py b/rpython/memory/gctransform/test/test_shadowcolor.py new file mode 100644 --- /dev/null +++ b/rpython/memory/gctransform/test/test_shadowcolor.py @@ -0,0 +1,791 @@ +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.test.test_llinterp import gengraph +from rpython.conftest import option +from rpython.memory.gctransform.shadowcolor import * +from rpython.flowspace import model as graphmodel +from rpython.translator.simplify import join_blocks, cleanup_graph +from hypothesis import given, strategies + + +def make_graph(f, argtypes): + t, rtyper, graph = gengraph(f, argtypes, viewbefore=False) + if getattr(option, 'view', False): + graph.show() + return graph + +def nameof(v): + return v._name.rstrip('_') + +def summary(interesting_vars): + result = {} + for v in interesting_vars: + name = nameof(v) + result[name] = result.get(name, 0) + 1 + return result + +def summary_regalloc(regalloc): + result = [] + for block in regalloc.graph.iterblocks(): + print block.inputargs + for op in block.operations: + print '\t', op + blockvars = block.inputargs + [op.result for op in block.operations] + for v in blockvars: + if regalloc.consider_var(v): + result.append((nameof(v), regalloc.getcolor(v))) + print '\t\t%s: %s' % (v, regalloc.getcolor(v)) + result.sort() + return result + + +def test_find_predecessors_1(): + def f(a, b): + c = a + b + return c + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'c': 1, 'v': 1} + +def test_find_predecessors_2(): + def f(a, b): + c = a + b + while a > 0: + a -= 2 + return c + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'c': 3, 'v': 1} + +def test_find_predecessors_3(): + def f(a, b): + while b > 100: + b -= 2 + if b > 10: + c = a + b # 'c' created in this block + else: + c = a - b # 'c' created in this block + return c # 'v' is the return var + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'c': 2, 'v': 1} + +def test_find_predecessors_4(): + def f(a, b): # 'a' in the input block + while b > 100: # 'a' in the loop header block + b -= 2 # 'a' in the loop body block + if b > 10: # 'a' in the condition block + while b > 5: # nothing + b -= 2 # nothing + c = a + b # 'c' created in this block + else: + c = a + return c # 'v' is the return var + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'a': 4, 'c': 1, 'v': 1} + +def test_find_predecessors_trivial_rewrite(): + def f(a, b): # 'b' in empty startblock + while a > 100: # 'b' + a -= 2 # 'b' + c = llop.same_as(lltype.Signed, b) # 'c', 'b' + while b > 10: # 'c' + b -= 2 # 'c' + d = llop.same_as(lltype.Signed, c) # 'd', 'c' + return d # 'v' is the return var + graph = make_graph(f, [int, int]) + pred = find_predecessors(graph, [(graph.returnblock, graph.getreturnvar())]) + assert summary(pred) == {'b': 4, 'c': 4, 'd': 1, 'v': 1} + +def test_find_successors_1(): + def f(a, b): + return a + b + graph = make_graph(f, [int, int]) + succ = find_successors(graph, [(graph.startblock, graph.getargs()[0])]) + assert summary(succ) == {'a': 1} + +def test_find_successors_2(): + def f(a, b): + if b > 10: + return a + b + else: + return a - b + graph = make_graph(f, [int, int]) + succ = find_successors(graph, [(graph.startblock, graph.getargs()[0])]) + assert summary(succ) == {'a': 3} + +def test_find_successors_3(): + def f(a, b): + if b > 10: # 'a' condition block + a = a + b # 'a' input + while b > 100: + b -= 2 + while b > 5: # 'a' in loop header + b -= 2 # 'a' in loop body + return a * b # 'a' in product + graph = make_graph(f, [int, int]) + succ = find_successors(graph, [(graph.startblock, graph.getargs()[0])]) + assert summary(succ) == {'a': 5} + +def test_find_successors_trivial_rewrite(): + def f(a, b): # 'b' in empty startblock + while a > 100: # 'b' + a -= 2 # 'b' + c = llop.same_as(lltype.Signed, b) # 'c', 'b' + while b > 10: # 'c', 'b' + b -= 2 # 'c', 'b' + d = llop.same_as(lltype.Signed, c) # 'd', 'c' + return d # 'v' is the return var + graph = make_graph(f, [int, int]) + pred = find_successors(graph, [(graph.startblock, graph.getargs()[1])]) + assert summary(pred) == {'b': 6, 'c': 4, 'd': 1, 'v': 1} + + +def test_interesting_vars_0(): + def f(a, b): + pass + graph = make_graph(f, [llmemory.GCREF, int]) + assert not find_interesting_variables(graph) + +def test_interesting_vars_1(): + def f(a, b): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + graph = make_graph(f, [llmemory.GCREF, int]) + assert summary(find_interesting_variables(graph)) == {'a': 1} + +def test_interesting_vars_2(): + def f(a, b, c): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + while b > 0: + b -= 5 + llop.gc_push_roots(lltype.Void, c) + llop.gc_pop_roots(lltype.Void, c) + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + assert summary(find_interesting_variables(graph)) == {'a': 1, 'c': 1} + +def test_interesting_vars_3(): + def f(a, b): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + while b > 0: # 'a' remains interesting across the blocks of this loop + b -= 5 + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + graph = make_graph(f, [llmemory.GCREF, int]) + assert summary(find_interesting_variables(graph)) == {'a': 4} + +def test_allocate_registers_1(): + def f(a, b): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + while b > 0: # 'a' remains interesting across the blocks of this loop + b -= 5 + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + graph = make_graph(f, [llmemory.GCREF, int]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 0)] * 4 + +def test_allocate_registers_2(): + def f(a, b, c): + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + while b > 0: + b -= 5 + llop.gc_push_roots(lltype.Void, c) + llop.gc_pop_roots(lltype.Void, c) + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 0), ('c', 0)] + +def test_allocate_registers_3(): + def f(a, b, c): + llop.gc_push_roots(lltype.Void, c, a) + llop.gc_pop_roots(lltype.Void, c, a) + while b > 0: + b -= 5 + llop.gc_push_roots(lltype.Void, a) + llop.gc_pop_roots(lltype.Void, a) + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 1)] * 4 + [('c', 0)] + +def test_allocate_registers_4(): + def g(a, x): + return x # (or something different) + def f(a, b, c): + llop.gc_push_roots(lltype.Void, a, c) # 'a', 'c' + llop.gc_pop_roots(lltype.Void, a, c) + while b > 0: # 'a' only; 'c' not in push_roots + b -= 5 + llop.gc_push_roots(lltype.Void, a)# 'a' + d = g(a, c) + llop.gc_pop_roots(lltype.Void, a) + c = d + return c + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 1)] * 3 + [('c', 0)] + +def test_allocate_registers_5(): + def g(a, x): + return x # (or something different) + def f(a, b, c): + while b > 0: # 'a', 'c' + b -= 5 + llop.gc_push_roots(lltype.Void, a, c) # 'a', 'c' + g(a, c) + llop.gc_pop_roots(lltype.Void, a, c) + while b < 10: + b += 2 + return c + graph = make_graph(f, [llmemory.GCREF, int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + assert summary_regalloc(regalloc) == [('a', 1)] * 2 + [('c', 0)] * 2 + + at given(strategies.lists(strategies.booleans())) +def test_make_bitmask(boollist): + index, bitmask = make_bitmask(boollist) + if index is None: + assert bitmask is None + else: + assert 0 <= index < len(boollist) + assert boollist[index] == False + assert bitmask >= 1 + while bitmask: + if bitmask & 1: + assert index >= 0 + assert boollist[index] == False + boollist[index] = True + bitmask >>= 1 + index -= 1 + assert boollist == [True] * len(boollist) + + +class FakeRegAlloc: + graph = '?' + + def __init__(self, expected_op, **colors): + self.expected_op = expected_op + self.numcolors = len(colors) + self.getcolor = colors.__getitem__ + + def check(self, got): + got = list(got) + result = [] + for spaceop in got: + assert spaceop.opname == self.expected_op + result.append((spaceop.args[0].value, spaceop.args[1])) + return result + +def test_expand_one_push_roots(): + regalloc = FakeRegAlloc('gc_save_root', a=0, b=1, c=2) + assert regalloc.check(expand_one_push_roots(regalloc, ['a', 'b', 'c'])) == [ + (0, 'a'), (1, 'b'), (2, 'c')] + assert regalloc.check(expand_one_push_roots(regalloc, ['a', 'c'])) == [ + (0, 'a'), (2, 'c'), (1, Constant(0x1, lltype.Signed))] + assert regalloc.check(expand_one_push_roots(regalloc, ['b'])) == [ + (1, 'b'), (2, Constant(0x5, lltype.Signed))] + assert regalloc.check(expand_one_push_roots(regalloc, ['a'])) == [ + (0, 'a'), (2, Constant(0x3, lltype.Signed))] + assert regalloc.check(expand_one_push_roots(regalloc, [])) == [ + (2, Constant(0x7, lltype.Signed))] + + assert list(expand_one_push_roots(None, [])) == [] + +def test_expand_one_pop_roots(): + regalloc = FakeRegAlloc('gc_restore_root', a=0, b=1, c=2) + assert regalloc.check(expand_one_pop_roots(regalloc, ['a', 'b', 'c'])) == [ + (0, 'a'), (1, 'b'), (2, 'c')] + assert regalloc.check(expand_one_pop_roots(regalloc, ['a', 'c'])) == [ + (0, 'a'), (2, 'c')] + assert regalloc.check(expand_one_pop_roots(regalloc, ['b'])) == [ + (1, 'b')] + assert regalloc.check(expand_one_pop_roots(regalloc, ['a'])) == [ + (0, 'a')] + assert regalloc.check(expand_one_pop_roots(regalloc, [])) == [] + + assert list(expand_one_pop_roots(None, [])) == [] + +def test_move_pushes_earlier_1(): + def g(a): + return a - 1 + def f(a, b): + a *= 2 + while a > 10: + llop.gc_push_roots(lltype.Void, b) + a = g(a) + llop.gc_pop_roots(lltype.Void, b) + return b + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert graphmodel.summary(graph) == { + 'int_mul': 1, + 'gc_enter_roots_frame': 1, + 'gc_save_root': 1, + 'gc_restore_root': 1, + 'int_gt': 1, + 'direct_call': 1, + 'gc_leave_roots_frame': 1, + } + assert len(graph.startblock.operations) == 3 + assert graph.startblock.operations[0].opname == 'int_mul' + assert graph.startblock.operations[1].opname == 'gc_enter_roots_frame' + assert graph.startblock.operations[2].opname == 'gc_save_root' + assert graph.startblock.operations[2].args[0].value == 0 + postprocess_double_check(graph) + +def test_move_pushes_earlier_2(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + while a > 10: + a -= 2 + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + return b + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'int_gt': 1, + 'int_sub': 1, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_remove_intrablock_push_roots(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + return b + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'direct_call': 2, + } + +PSTRUCT = lltype.Ptr(lltype.GcStruct('S')) + +def test_move_pushes_earlier_rename_1(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + c = lltype.cast_opaque_ptr(PSTRUCT, b) + while a > 10: + a -= 2 + llop.gc_push_roots(lltype.Void, c) + g(a) + llop.gc_pop_roots(lltype.Void, c) + return c + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'cast_opaque_ptr': 1, + 'int_gt': 1, + 'int_sub': 1, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_move_pushes_earlier_rename_2(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + while a > 10: + a -= 2 + c = lltype.cast_opaque_ptr(PSTRUCT, b) + llop.gc_push_roots(lltype.Void, c) + g(a) + llop.gc_pop_roots(lltype.Void, c) + return c + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'cast_opaque_ptr': 1, + 'int_gt': 1, + 'int_sub': 1, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_move_pushes_earlier_rename_3(): + def g(a): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(a) + llop.gc_pop_roots(lltype.Void, b) + while a > 10: + a -= 2 + c = lltype.cast_opaque_ptr(PSTRUCT, b) + while a > 10: + a -= 2 + llop.gc_push_roots(lltype.Void, c) + g(a) + llop.gc_pop_roots(lltype.Void, c) + return c + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'cast_opaque_ptr': 1, + 'int_gt': 2, + 'int_sub': 2, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_move_pushes_earlier_rename_4(): + def g(a): + return a - 2 + def f(a, b): + while a > 10: + b1 = lltype.cast_opaque_ptr(PSTRUCT, b) + while a > 100: + a -= 3 + b2 = lltype.cast_opaque_ptr(llmemory.GCREF, b1) + llop.gc_push_roots(lltype.Void, b2) + a = g(a) + llop.gc_pop_roots(lltype.Void, b2) + b3 = lltype.cast_opaque_ptr(PSTRUCT, b2) + while a > 100: + a -= 4 + b4 = lltype.cast_opaque_ptr(llmemory.GCREF, b3) + llop.gc_push_roots(lltype.Void, b4) + a = g(a) + llop.gc_pop_roots(lltype.Void, b4) + b5 = lltype.cast_opaque_ptr(PSTRUCT, b4) + while a > 100: + a -= 5 + b = lltype.cast_opaque_ptr(llmemory.GCREF, b5) + return b + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + assert graphmodel.summary(graph) == { + 'gc_save_root': 1, + 'gc_restore_root': 2, + 'cast_opaque_ptr': 6, + 'int_gt': 4, + 'int_sub': 3, + 'direct_call': 2, + } + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_add_leave_roots_frame_1(): + def g(b): + pass + def f(a, b): + if a & 1: + llop.gc_push_roots(lltype.Void, b) + g(b) + llop.gc_pop_roots(lltype.Void, b) + a += 5 + else: + llop.gc_push_roots(lltype.Void, b) + g(b) + llop.gc_pop_roots(lltype.Void, b) + a += 6 + #...b forgotten here, even though it is pushed/popped above + while a > 100: + a -= 3 + return a + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert len(graph.startblock.exits) == 2 + for link in graph.startblock.exits: + assert [op.opname for op in link.target.operations] == [ + 'gc_enter_roots_frame', + 'gc_save_root', + 'direct_call', + 'gc_restore_root', + 'gc_leave_roots_frame', + 'int_add'] + postprocess_double_check(graph) + +def test_add_leave_roots_frame_2(): + def g(b): + pass + def f(a, b): + llop.gc_push_roots(lltype.Void, b) + g(b) + llop.gc_pop_roots(lltype.Void, b) + #...b forgotten here; the next push/pop is empty + llop.gc_push_roots(lltype.Void) + g(b) + llop.gc_pop_roots(lltype.Void) + while a > 100: + a -= 3 + return a + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert [op.opname for op in graph.startblock.operations] == [ + 'gc_enter_roots_frame', + 'gc_save_root', + 'direct_call', + 'gc_restore_root', + 'gc_leave_roots_frame', + 'direct_call'] + postprocess_double_check(graph) + +def test_bug_1(): + class W: + pass + def foo(a): + if a < 10: + return W() + else: + return None + def compare(w_a, w_b): + return W() + def fetch_compare(w_a, w_b): + return W() + def is_true(a, w_b): + return not a + def call_next(w_a): + return W() + + def f(a, w_tup): + llop.gc_push_roots(lltype.Void, w_tup) + w_key = foo(a) + llop.gc_pop_roots(lltype.Void, w_tup) + + llop.gc_push_roots(lltype.Void, w_key) + w_iter = foo(a) + llop.gc_pop_roots(lltype.Void, w_key) + + has_key = w_key is not None + hasit = False + w_maxit = None + w_max_val = None + + while True: + llop.gc_push_roots(lltype.Void, w_iter, w_key, w_maxit, w_max_val) + w_item = call_next(w_iter) + llop.gc_pop_roots(lltype.Void, w_iter, w_key, w_maxit, w_max_val) + + if has_key: + llop.gc_push_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item) + w_compare_with = fetch_compare(w_key, w_item) + llop.gc_pop_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item) + else: + w_compare_with = w_item + + if hasit: + llop.gc_push_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item, w_compare_with) + w_bool = compare(w_compare_with, w_max_val) + llop.gc_pop_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item, w_compare_with) + + llop.gc_push_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item, w_compare_with) + condition = is_true(a, w_bool) + llop.gc_pop_roots(lltype.Void, w_iter, w_key, + w_maxit, w_max_val, w_item, w_compare_with) + else: + condition = True + + if condition: + hasit = True + w_maxit = w_item + w_max_val = w_compare_with + + graph = make_graph(f, [int, llmemory.GCREF]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_bug_2(): + def f(w_tup): + while True: + llop.gc_push_roots(lltype.Void, w_tup) + llop.gc_pop_roots(lltype.Void, w_tup) + + graph = make_graph(f, [llmemory.GCREF]) + assert not graph.startblock.operations + # this test is about what occurs if the startblock of the graph + # is also reached from another block. None of the 'simplify' + # functions actually remove that, but the JIT transformation can... + graph.startblock = graph.startblock.exits[0].target + + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + postprocess_double_check(graph) + +def test_add_enter_roots_frame_remove_empty(): + class W: + pass + def g(): + return W() + def h(x): + pass + def k(): + pass + def f(): + llop.gc_push_roots(lltype.Void) + x = g() + llop.gc_pop_roots(lltype.Void) + llop.gc_push_roots(lltype.Void, x) + h(x) + llop.gc_pop_roots(lltype.Void, x) + llop.gc_push_roots(lltype.Void) + h(x) + llop.gc_pop_roots(lltype.Void) + llop.gc_push_roots(lltype.Void) + k() + llop.gc_pop_roots(lltype.Void) + + graph = make_graph(f, []) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert [op.opname for op in graph.startblock.operations] == [ + "direct_call", + "gc_enter_roots_frame", + "gc_save_root", + "direct_call", + "gc_restore_root", + "gc_leave_roots_frame", + "direct_call", + "direct_call", + ] + postprocess_double_check(graph) + +def test_add_enter_roots_frame_avoided(): + def g(x): + return x + def f(x, n): + if n > 100: + llop.gc_push_roots(lltype.Void, x) + g(x) + llop.gc_pop_roots(lltype.Void, x) + return x + + graph = make_graph(f, [llmemory.GCREF, int]) + regalloc = allocate_registers(graph) + expand_push_roots(graph, regalloc) + move_pushes_earlier(graph, regalloc) + expand_pop_roots(graph, regalloc) + add_enter_leave_roots_frame(graph, regalloc, Constant('fake gcdata')) + assert [op.opname for op in graph.startblock.operations] == [ + 'int_gt', 'same_as'] + [fastpath, slowpath] = graph.startblock.exits + assert fastpath.target is graph.returnblock + block2 = slowpath.target + assert [op.opname for op in block2.operations] == [ + 'gc_enter_roots_frame', + 'gc_save_root', + 'direct_call', + 'gc_restore_root', + 'gc_leave_roots_frame'] + postprocess_double_check(graph) From pypy.commits at gmail.com Wed Mar 8 10:07:43 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 07:07:43 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix for test_subprocess assuming stat.S_ISDIIR() exists---even though it Message-ID: <58c01e3f.8d63190a.ea185.3a0f@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90596:b8f9d7ba084a Date: 2017-03-08 16:07 +0100 http://bitbucket.org/pypy/pypy/changeset/b8f9d7ba084a/ Log: fix for test_subprocess assuming stat.S_ISDIIR() exists---even though it only exists in the _stat optional built-in module on CPython diff --git a/lib_pypy/_stat.py b/lib_pypy/_stat.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_stat.py @@ -0,0 +1,6 @@ +# Assume not Solaris + +S_IFDOOR = 0 + +def S_ISDOOR(mode): + return False From pypy.commits at gmail.com Wed Mar 8 10:29:53 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 07:29:53 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Fix test_venv.test_executable Message-ID: <58c02371.53532e0a.87661.38a8@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90597:f539adf3c8fd Date: 2017-03-08 16:29 +0100 http://bitbucket.org/pypy/pypy/changeset/f539adf3c8fd/ Log: Fix test_venv.test_executable diff --git a/lib-python/3/venv/__init__.py b/lib-python/3/venv/__init__.py --- a/lib-python/3/venv/__init__.py +++ b/lib-python/3/venv/__init__.py @@ -217,6 +217,17 @@ copier(context.env_exe, path, relative_symlinks_ok=True) if not os.path.islink(path): os.chmod(path, 0o755) + # PyPy extension: also copy the main library, not just the + # small executable + for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + dest_library = os.path.join(binpath, libname) + src_library = os.path.join(os.path.dirname(context.executable), + libname) + if (not os.path.exists(dest_library) and + os.path.exists(src_library)): + copier(src_library, dest_library) + if not os.path.islink(dest_library): + os.chmod(dest_library, 0o755) else: subdir = 'DLLs' include = self.include_binary From pypy.commits at gmail.com Wed Mar 8 11:11:50 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 08:11:50 -0800 (PST) Subject: [pypy-commit] pypy py3.5: find and follow 'pyvenv.cfg' files, hopefully like CPython does Message-ID: <58c02d46.d3d3190a.8bf2d.3cc8@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90598:d129aab5df7d Date: 2017-03-08 17:11 +0100 http://bitbucket.org/pypy/pypy/changeset/d129aab5df7d/ Log: find and follow 'pyvenv.cfg' files, hopefully like CPython does diff --git a/lib-python/3/venv/__init__.py b/lib-python/3/venv/__init__.py --- a/lib-python/3/venv/__init__.py +++ b/lib-python/3/venv/__init__.py @@ -119,6 +119,14 @@ executable = os.environ['__PYVENV_LAUNCHER__'] else: executable = sys.executable + # + # PyPy extension: resolve 'executable' if it is a symlink + try: + executable = os.path.join(os.path.dirname(executable), + os.readlink(executable)) + except OSError: + pass + # dirname, exename = os.path.split(os.path.abspath(executable)) context.executable = executable context.python_dir = dirname @@ -217,6 +225,7 @@ copier(context.env_exe, path, relative_symlinks_ok=True) if not os.path.islink(path): os.chmod(path, 0o755) + # # PyPy extension: also copy the main library, not just the # small executable for libname in ['libpypy-c.so', 'libpypy-c.dylib']: @@ -228,6 +237,7 @@ copier(src_library, dest_library) if not os.path.islink(dest_library): os.chmod(dest_library, 0o755) + # else: subdir = 'DLLs' include = self.include_binary diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -73,18 +73,57 @@ return dirname +def find_pyvenv_cfg(dirname): + try: + fd = os.open(os.path.join(dirname, 'pyvenv.cfg'), os.O_RDONLY) + try: + content = os.read(fd, 16384) + finally: + os.close(fd) + except OSError: + return '' + # painfully parse the file for a line 'home = PATH' + for line in content.splitlines(): + line += '\n' + i = 0 + while line[i] == ' ': + i += 1 + if (line[i] == 'h' and + line[i+1] == 'o' and + line[i+2] == 'm' and + line[i+3] == 'e'): + i += 4 + while line[i] == ' ': + i += 1 + if line[i] == '=': + i += 1 + return line[i:].strip() + return '' + + def find_stdlib(state, executable): """ Find and compute the stdlib path, starting from the directory where ``executable`` is and going one level up until we find it. Return a tuple (path, prefix), where ``prefix`` is the root directory which contains the stdlib. If it cannot be found, return (None, None). + + On PyPy3, it will also look for 'pyvenv.cfg' either in the same or + in the parent directory of 'executable', and search from the 'home' + entry instead of from the path to 'executable'. """ search = 'pypy-c' if executable == '' else executable + search_pyvenv_cfg = 2 while True: dirname = resolvedirof(search) if dirname == search: return None, None # not found :-( + if search_pyvenv_cfg > 0: + search_pyvenv_cfg -= 1 + home = find_pyvenv_cfg(dirname) + if home: + dirname = home + search_pyvenv_cfg = 0 newpath = compute_stdlib_path_maybe(state, dirname) if newpath is not None: return newpath, dirname diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py --- a/pypy/module/sys/test/test_initpath.py +++ b/pypy/module/sys/test/test_initpath.py @@ -2,7 +2,8 @@ import os.path from pypy.module.sys.initpath import (compute_stdlib_path, find_executable, find_stdlib, resolvedirof, - pypy_init_home, pypy_init_free) + pypy_init_home, pypy_init_free, + find_pyvenv_cfg) from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION from rpython.rtyper.lltypesystem import rffi @@ -114,3 +115,26 @@ myfile2 = bar.join('myfile') myfile2.mksymlinkto(myfile) assert resolvedirof(str(myfile2)) == foo + +def test_find_pyvenv_cfg(tmpdir): + subdir = tmpdir.join('find_cfg').ensure(dir=True) + assert find_pyvenv_cfg(str(subdir)) == '' + subdir.join('pyvenv.cfg').write('foobar') + assert find_pyvenv_cfg(str(subdir)) == '' + subdir.join('pyvenv.cfg').write('foobar\nhome=xyz') + assert find_pyvenv_cfg(str(subdir)) == 'xyz' + subdir.join('pyvenv.cfg').write('foohome=xyz') + assert find_pyvenv_cfg(str(subdir)) == '' + subdir.join('pyvenv.cfg').write('home = xyx \nbar = baz\n') + assert find_pyvenv_cfg(str(subdir)) == 'xyx' + +def test_find_stdlib_follow_pyvenv_cfg(tmpdir): + mydir = tmpdir.join('follow_pyvenv_cfg').ensure(dir=True) + otherdir = tmpdir.join('otherdir').ensure(dir=True) + bin_dir = mydir.join('bin').ensure(dir=True) + pypy = bin_dir.join('pypy').ensure(file=True) + build_hierarchy(otherdir) + for homedir in [otherdir, otherdir.join('bin')]: + mydir.join('pyvenv.cfg').write('home = %s\n' % (homedir,)) + _, prefix = find_stdlib(None, str(pypy)) + assert prefix == otherdir From pypy.commits at gmail.com Wed Mar 8 11:58:19 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 08:58:19 -0800 (PST) Subject: [pypy-commit] pypy py3.5: translation fix Message-ID: <58c0382b.15502e0a.cc5af.3f55@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90599:25fccae415a0 Date: 2017-03-08 17:57 +0100 http://bitbucket.org/pypy/pypy/changeset/25fccae415a0/ Log: translation fix diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -8,6 +8,7 @@ import sys from rpython.rlib import rpath, rdynload +from rpython.rlib.rstring import assert_str0 from rpython.rlib.objectmodel import we_are_translated from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -75,7 +76,7 @@ def find_pyvenv_cfg(dirname): try: - fd = os.open(os.path.join(dirname, 'pyvenv.cfg'), os.O_RDONLY) + fd = os.open(os.path.join(dirname, 'pyvenv.cfg'), os.O_RDONLY, 0) try: content = os.read(fd, 16384) finally: @@ -84,7 +85,7 @@ return '' # painfully parse the file for a line 'home = PATH' for line in content.splitlines(): - line += '\n' + line += '\x00' i = 0 while line[i] == ' ': i += 1 @@ -96,8 +97,11 @@ while line[i] == ' ': i += 1 if line[i] == '=': - i += 1 - return line[i:].strip() + line = line[i+1:] + n = line.find('\x00') + assert n >= 0 + line = line[:n] + return assert_str0(line.strip()) return '' From pypy.commits at gmail.com Wed Mar 8 12:47:46 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 08 Mar 2017 09:47:46 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Try harder to resolve symlinks Message-ID: <58c043c2.129f190a.de2e2.4019@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90600:fc90e57573c9 Date: 2017-03-08 18:47 +0100 http://bitbucket.org/pypy/pypy/changeset/fc90e57573c9/ Log: Try harder to resolve symlinks diff --git a/lib-python/3/test/test_venv.py b/lib-python/3/test/test_venv.py --- a/lib-python/3/test/test_venv.py +++ b/lib-python/3/test/test_venv.py @@ -46,6 +46,22 @@ def failsOnWindows(f): return f +def _my_executable(): + # PyPy: resolve the executable if it is a symlink + if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ: + executable = os.environ['__PYVENV_LAUNCHER__'] + else: + executable = sys.executable + try: + for i in range(10): + executable = os.path.abspath(executable) + executable = os.path.join(os.path.dirname(executable), + os.readlink(executable)) + except OSError: + pass + return executable + + class BaseTest(unittest.TestCase): """Base class for venv tests.""" @@ -59,10 +75,7 @@ self.bindir = 'bin' self.lib = ('lib', 'python%s' % sys.version[:3]) self.include = 'include' - if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ: - executable = os.environ['__PYVENV_LAUNCHER__'] - else: - executable = sys.executable + executable = _my_executable() self.exe = os.path.split(executable)[-1] def tearDown(self): @@ -107,11 +120,7 @@ else: self.assertFalse(os.path.exists(p)) data = self.get_text_file_contents('pyvenv.cfg') - if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__' - in os.environ): - executable = os.environ['__PYVENV_LAUNCHER__'] - else: - executable = sys.executable + executable = _my_executable() path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) fn = self.get_env_file(self.bindir, self.exe) diff --git a/lib-python/3/venv/__init__.py b/lib-python/3/venv/__init__.py --- a/lib-python/3/venv/__init__.py +++ b/lib-python/3/venv/__init__.py @@ -122,8 +122,10 @@ # # PyPy extension: resolve 'executable' if it is a symlink try: - executable = os.path.join(os.path.dirname(executable), - os.readlink(executable)) + for i in range(10): + executable = os.path.abspath(executable) + executable = os.path.join(os.path.dirname(executable), + os.readlink(executable)) except OSError: pass # From pypy.commits at gmail.com Wed Mar 8 13:43:14 2017 From: pypy.commits at gmail.com (mattip) Date: Wed, 08 Mar 2017 10:43:14 -0800 (PST) Subject: [pypy-commit] pypy default: hard code npy_intp, py_uintp to make linux32 happy Message-ID: <58c050c2.504f190a.1d2a8.4645@mx.google.com> Author: Matti Picus Branch: Changeset: r90601:e58d3c02d05c Date: 2017-03-08 20:41 +0200 http://bitbucket.org/pypy/pypy/changeset/e58d3c02d05c/ Log: hard code npy_intp, py_uintp to make linux32 happy diff --git a/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h b/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h --- a/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h @@ -1,8 +1,8 @@ #ifndef _NPY_COMMON_H_ #define _NPY_COMMON_H_ -typedef Py_intptr_t npy_intp; -typedef Py_uintptr_t npy_uintp; +typedef long npy_intp; +typedef unsigned long npy_uintp; typedef PY_LONG_LONG npy_longlong; typedef unsigned PY_LONG_LONG npy_ulonglong; typedef unsigned char npy_bool; From pypy.commits at gmail.com Thu Mar 9 03:17:31 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 00:17:31 -0800 (PST) Subject: [pypy-commit] pypy default: proper fix Message-ID: <58c10f9b.c819190a.7baba.5cef@mx.google.com> Author: Armin Rigo Branch: Changeset: r90602:98bd8eef9a25 Date: 2017-03-09 09:16 +0100 http://bitbucket.org/pypy/pypy/changeset/98bd8eef9a25/ Log: proper fix diff --git a/rpython/translator/platform/bsd.py b/rpython/translator/platform/bsd.py --- a/rpython/translator/platform/bsd.py +++ b/rpython/translator/platform/bsd.py @@ -11,7 +11,7 @@ standalone_only = [] shared_only = [] - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/cygwin.py b/rpython/translator/platform/cygwin.py --- a/rpython/translator/platform/cygwin.py +++ b/rpython/translator/platform/cygwin.py @@ -27,7 +27,7 @@ so_ext = 'dll' exe_ext = 'exe' - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -30,13 +30,16 @@ print 'in get_rpath_flags, rel_libdirs is not fixed up',rel_libdirs return self.rpath_flags - def _args_for_shared(self, args): - if hasattr(self, '_exe_name'): - target = os.path.basename(self._exe_name) + def _args_for_shared(self, args, **kwds): + if 'exe_name' in kwds: + target_basename = kwds['exe_name'].basename else: - target = '$(TARGET)' # inside a Makefile + target_basename = '$(TARGET)' + # The default '$(TARGET)' is used inside a Makefile. Otherwise + # we get the basename of the executable we're trying to build. return (list(self.shared_only) - + ['-dynamiclib', '-install_name', '@rpath/' + target, '-undefined', 'dynamic_lookup', '-flat_namespace'] + + ['-dynamiclib', '-install_name', '@rpath/' + target_basename, + '-undefined', 'dynamic_lookup', '-flat_namespace'] + args) def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py --- a/rpython/translator/platform/linux.py +++ b/rpython/translator/platform/linux.py @@ -24,7 +24,7 @@ from rpython.translator.platform.arch import s390x cflags = s390x.update_cflags(cflags) - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/netbsd.py b/rpython/translator/platform/netbsd.py --- a/rpython/translator/platform/netbsd.py +++ b/rpython/translator/platform/netbsd.py @@ -34,7 +34,7 @@ cc = get_env("CC", "gcc") super(Netbsd, self).__init__(cc) - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _preprocess_include_dirs(self, include_dirs): diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -54,9 +54,7 @@ args = [str(ofile) for ofile in ofiles] + link_args args += ['-o', str(exe_name)] if not standalone: - self._exe_name = str(exe_name) - args = self._args_for_shared(args) - del self._exe_name # remove, otherwise __eq__() fails + args = self._args_for_shared(args, exe_name=exe_name) self._execute_c_compiler(cc, args, exe_name, cwd=str(exe_name.dirpath())) return exe_name diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -200,7 +200,7 @@ def _linkfiles(self, link_files): return list(link_files) - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['/dll'] + args def check___thread(self): @@ -555,7 +555,7 @@ cc = 'gcc' Platform.__init__(self, cc) - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _include_dirs_for_libffi(self): From pypy.commits at gmail.com Thu Mar 9 04:10:59 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 01:10:59 -0800 (PST) Subject: [pypy-commit] pypy default: Workaround for gcc Message-ID: <58c11c23.98e5190a.a9388.5f8c@mx.google.com> Author: Armin Rigo Branch: Changeset: r90603:30b1611b058f Date: 2017-03-09 10:10 +0100 http://bitbucket.org/pypy/pypy/changeset/30b1611b058f/ Log: Workaround for gcc diff --git a/rpython/jit/backend/x86/test/test_zmath.py b/rpython/jit/backend/x86/test/test_zmath.py --- a/rpython/jit/backend/x86/test/test_zmath.py +++ b/rpython/jit/backend/x86/test/test_zmath.py @@ -61,7 +61,12 @@ return -42 # ok def test_math(): - f = compile(fn, []) + # note: we use lldebug because in the normal optimizing mode, some + # calls may be completely inlined and constant-folded by the + # compiler (with -flto), e.g. atanh(0.3). They give then slightly + # different result than if they were executed at runtime. + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79973 + f = compile(fn, [], lldebug=True) res = f() if res >= 0: py.test.fail(repr(MathTests.TESTCASES[res])) From pypy.commits at gmail.com Thu Mar 9 04:21:05 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 01:21:05 -0800 (PST) Subject: [pypy-commit] pypy default: bound the list size Message-ID: <58c11e81.020f2e0a.d3781.615d@mx.google.com> Author: Armin Rigo Branch: Changeset: r90604:74eb11789df3 Date: 2017-03-09 10:20 +0100 http://bitbucket.org/pypy/pypy/changeset/74eb11789df3/ Log: bound the list size diff --git a/rpython/memory/gctransform/test/test_shadowcolor.py b/rpython/memory/gctransform/test/test_shadowcolor.py --- a/rpython/memory/gctransform/test/test_shadowcolor.py +++ b/rpython/memory/gctransform/test/test_shadowcolor.py @@ -246,7 +246,7 @@ regalloc = allocate_registers(graph) assert summary_regalloc(regalloc) == [('a', 1)] * 2 + [('c', 0)] * 2 - at given(strategies.lists(strategies.booleans())) + at given(strategies.lists(strategies.booleans(), max_size=31)) def test_make_bitmask(boollist): index, bitmask = make_bitmask(boollist) if index is None: From pypy.commits at gmail.com Thu Mar 9 09:44:52 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 06:44:52 -0800 (PST) Subject: [pypy-commit] pypy py3.5: document why a test fail (didn't fix yet) Message-ID: <58c16a64.1f002e0a.ae306.70a7@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90605:7c8b6de602ac Date: 2017-03-09 15:44 +0100 http://bitbucket.org/pypy/pypy/changeset/7c8b6de602ac/ Log: document why a test fail (didn't fix yet) diff --git a/lib-python/3/test/test_io.py b/lib-python/3/test/test_io.py --- a/lib-python/3/test/test_io.py +++ b/lib-python/3/test/test_io.py @@ -1169,7 +1169,12 @@ b = bytearray(2*buffer_size) self.assertEqual(bufio.peek(3), b'fgh') self.assertEqual(rawio._reads, 3) - self.assertEqual(bufio.readinto1(b), 6) + self.assertEqual(bufio.readinto1(b), 6) # fails because of + # an apparent inconsistency in CPython: readinto1(), if the + # buffered amount is smaller, would always issue one raw read() + # call. This differs from read1(), which if the buffered amount + # if smaller (but more than zero), would just return it without + # any raw read() call. In PyPy both have the behavior of read1(). self.assertEqual(b[:6], b"fghjkl") self.assertEqual(rawio._reads, 4) From pypy.commits at gmail.com Thu Mar 9 10:11:53 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 07:11:53 -0800 (PST) Subject: [pypy-commit] pypy py3.5: test and fix Message-ID: <58c170b9.0a142e0a.135dc.3cc0@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90606:928d214e47b3 Date: 2017-03-09 16:10 +0100 http://bitbucket.org/pypy/pypy/changeset/928d214e47b3/ Log: test and fix diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -912,7 +912,6 @@ # from the mixin class __repr__ = interp2app(W_BufferedReader.repr_w), readable = interp2app(W_BufferedReader.readable_w), - writable = interp2app(W_BufferedReader.writable_w), seekable = interp2app(W_BufferedReader.seekable_w), seek = interp2app(W_BufferedReader.seek_w), tell = interp2app(W_BufferedReader.tell_w), @@ -954,7 +953,6 @@ # from the mixin class __repr__ = interp2app(W_BufferedWriter.repr_w), - readable = interp2app(W_BufferedWriter.readable_w), writable = interp2app(W_BufferedWriter.writable_w), seekable = interp2app(W_BufferedWriter.seekable_w), seek = interp2app(W_BufferedWriter.seek_w), diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -24,10 +24,12 @@ raises(ValueError, f.read, -2) f.close() # - raw = _io.FileIO(self.tmpfile) + raw = _io.FileIO(self.tmpfile, 'r+') f = _io.BufferedReader(raw) r = f.read(4) assert r == b"a\nb\n" + assert f.readable() is True + assert f.writable() is False f.close() def test_read_pieces(self): @@ -320,10 +322,12 @@ def test_write(self): import _io - raw = _io.FileIO(self.tmpfile, 'w') + raw = _io.FileIO(self.tmpfile, 'w+') f = _io.BufferedWriter(raw) f.write(b"abcd") raises(TypeError, f.write, u"cd") + assert f.writable() is True + assert f.readable() is False f.close() assert self.readfile() == b"abcd" From pypy.commits at gmail.com Thu Mar 9 10:42:28 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 07:42:28 -0800 (PST) Subject: [pypy-commit] pypy py3.5: bump the version to 3.5.3 (the stdlib of 3.5.3 was already merged) Message-ID: <58c177e4.4f5c190a.b2790.7225@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90607:f2babb675692 Date: 2017-03-09 16:41 +0100 http://bitbucket.org/pypy/pypy/changeset/f2babb675692/ Log: bump the version to 3.5.3 (the stdlib of 3.5.3 was already merged) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -21,7 +21,7 @@ /* Version parsed out into numeric values */ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 5 -#define PY_MICRO_VERSION 2 +#define PY_MICRO_VERSION 3 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -6,7 +6,7 @@ from pypy.interpreter import gateway #XXX # the release serial 42 is not in range(16) -CPYTHON_VERSION = (3, 5, 2, "final", 0) +CPYTHON_VERSION = (3, 5, 3, "final", 0) #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h From pypy.commits at gmail.com Thu Mar 9 10:56:58 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 07:56:58 -0800 (PST) Subject: [pypy-commit] pypy py3.5: link pylifecycle in the compilation process Message-ID: <58c17b4a.55582e0a.a7339.73dc@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90608:a946964a3d8a Date: 2017-03-09 16:56 +0100 http://bitbucket.org/pypy/pypy/changeset/a946964a3d8a/ Log: link pylifecycle in the compilation process diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1337,6 +1337,7 @@ source_dir / "complexobject.c", source_dir / "import.c", source_dir / "_warnings.c", + source_dir / "pylifecycle.c", ] def build_eci(code, use_micronumpy=False, translating=False): diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -133,6 +133,7 @@ #include "pysignals.h" #include "pythread.h" #include "traceback.h" +#include "pylifecycle.h" /* Missing definitions */ #include "missing.h" From pypy.commits at gmail.com Thu Mar 9 10:58:59 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 07:58:59 -0800 (PST) Subject: [pypy-commit] pypy py3.5: link genobject.py into the translation process Message-ID: <58c17bc3.db002e0a.1a0c.7446@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90609:ead90bb3fd09 Date: 2017-03-09 16:58 +0100 http://bitbucket.org/pypy/pypy/changeset/ead90bb3fd09/ Log: link genobject.py into the translation process diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -75,6 +75,7 @@ import pypy.module.cpyext.pytraceback import pypy.module.cpyext.methodobject import pypy.module.cpyext.dictproxyobject +import pypy.module.cpyext.genobject # now that all rffi_platform.Struct types are registered, configure them api.configure_types() From pypy.commits at gmail.com Thu Mar 9 11:25:38 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 08:25:38 -0800 (PST) Subject: [pypy-commit] pypy py3.5: add PyStopIterationObject Message-ID: <58c18202.98e5190a.a9388.74b0@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90610:f8357b98a4c2 Date: 2017-03-09 17:24 +0100 http://bitbucket.org/pypy/pypy/changeset/f8357b98a4c2/ Log: add PyStopIterationObject diff --git a/pypy/module/cpyext/include/pyerrors.h b/pypy/module/cpyext/include/pyerrors.h --- a/pypy/module/cpyext/include/pyerrors.h +++ b/pypy/module/cpyext/include/pyerrors.h @@ -36,6 +36,11 @@ PyAPI_FUNC(int) PyOS_snprintf(char *str, size_t size, const char *format, ...); PyAPI_FUNC(int) PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va); +typedef struct { + PyObject_HEAD /* xxx PyException_HEAD in CPython */ + PyObject *value; +} PyStopIterationObject; + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -4,13 +4,47 @@ from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror from pypy.interpreter import pytraceback from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING +from pypy.module.cpyext.api import PyObjectFields, cpython_struct +from pypy.module.cpyext.api import bootstrap_function, slot_function +from pypy.module.cpyext.pyobject import make_typedescr from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning +from pypy.module.exceptions.interp_exceptions import W_StopIteration from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, make_ref, from_ref, Py_DecRef) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from rpython.rlib import rposix, jit +PyStopIterationObjectStruct = lltype.ForwardReference() +PyStopIterationObject = lltype.Ptr(PyStopIterationObjectStruct) +PyStopIterationObjectFields = PyObjectFields + \ + (("value", PyObject), ) +cpython_struct("PyStopIterationObject", PyStopIterationObjectFields, + PyStopIterationObjectStruct) + + at bootstrap_function +def init_stopiterationobject(space): + "Type description of PyStopIterationObject" + make_typedescr(W_StopIteration.typedef, + basestruct=PyStopIterationObject.TO, + attach=stopiteration_attach, + dealloc=stopiteration_dealloc) + +def stopiteration_attach(space, py_obj, w_obj, w_userdata=None): + py_stopiteration = rffi.cast(PyStopIterationObject, py_obj) + assert isinstance(w_obj, W_StopIteration) + # note: assumes that w_value is read-only; changes on one side won't + # be reflected on the other side + py_stopiteration.c_value = make_ref(space, w_obj.w_value) + + at slot_function([PyObject], lltype.Void) +def stopiteration_dealloc(space, py_obj): + py_stopiteration = rffi.cast(PyStopIterationObject, py_obj) + Py_DecRef(space, py_stopiteration.c_value) + from pypy.module.cpyext.object import _dealloc + _dealloc(space, py_obj) + + @cpython_api([PyObject, PyObject], lltype.Void) def PyErr_SetObject(space, w_type, w_value): """This function is similar to PyErr_SetString() but lets you specify an diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py --- a/pypy/module/cpyext/test/test_pyerrors.py +++ b/pypy/module/cpyext/test/test_pyerrors.py @@ -443,3 +443,15 @@ module.test() assert len(l) == 1 assert "foo 42 bar" in str(l[0]) + + def test_StopIteration_value(self): + module = self.import_extension('foo', [ + ("test", "METH_O", + ''' + PyObject *o = ((PyStopIterationObject *)args)->value; + Py_INCREF(o); + return o; + '''), + ]) + res = module.test(StopIteration("foo!")) + assert res == "foo!" From pypy.commits at gmail.com Thu Mar 9 11:32:31 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 08:32:31 -0800 (PST) Subject: [pypy-commit] pypy py3.5: add dummy (non-)implementations Message-ID: <58c1839f.02612e0a.4824.7814@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90611:d81cac9fe8ad Date: 2017-03-09 17:31 +0100 http://bitbucket.org/pypy/pypy/changeset/d81cac9fe8ad/ Log: add dummy (non-)implementations diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -240,6 +240,7 @@ /* dummy GC macros */ #define _PyGC_FINALIZED(o) 1 +#define PyType_IS_GC(tp) 1 /* Utility macro to help write tp_traverse functions. * To use this macro, the tp_traverse function must name its arguments diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -2077,3 +2077,12 @@ it causes an exception to immediately be thrown; this is used for the throw() methods of generator objects.""" raise NotImplementedError + + at cpython_api([PyObject], rffi.INT_real, error=-1) +def PyObject_CallFinalizerFromDealloc(space, self): + """Call tp_finalize() once, taking care of reference counting. + + NOT IMPLEMENTED: might need to keep the PyPy object alive even while + tp_dealloc() is called, which the GC doesn't do so far. + """ + raise NotImplementedError From pypy.commits at gmail.com Thu Mar 9 12:11:56 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 09:11:56 -0800 (PST) Subject: [pypy-commit] pypy py3.5: improve error message Message-ID: <58c18cdc.0528190a.a2485.78c6@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90612:92ab105ad8e7 Date: 2017-03-09 18:11 +0100 http://bitbucket.org/pypy/pypy/changeset/92ab105ad8e7/ Log: improve error message diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -957,6 +957,9 @@ assert C.__name__ == 'A' assert C.__qualname__ == 'C' + e = raises(TypeError, type, 'D', (), {'__qualname__': 42}) + assert str(e.value) == "type __qualname__ must be a str, not int" + def test_compare(self): class A(object): pass diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -197,6 +197,10 @@ if self.flag_heaptype: w_qualname = self.dict_w.pop('__qualname__', None) if w_qualname is not None: + if not space.isinstance_w(w_qualname, space.w_unicode): + raise oefmt(space.w_TypeError, + "type __qualname__ must be a str, not %T", + w_qualname) self.qualname = space.unicode_w(w_qualname) else: self.qualname = self.getname(space) From pypy.commits at gmail.com Thu Mar 9 12:19:25 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 09:19:25 -0800 (PST) Subject: [pypy-commit] pypy py3.5: issue #2491: cpyext types are allowed to have non-strings as Message-ID: <58c18e9d.42152e0a.1c26e.08b4@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90613:69629e27d040 Date: 2017-03-09 18:18 +0100 http://bitbucket.org/pypy/pypy/changeset/69629e27d040/ Log: issue #2491: cpyext types are allowed to have non-strings as '__qualname__'. Used for meta-types maybe? Untested diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -494,10 +494,10 @@ minsize = rffi.sizeof(PyObject.TO) new_layout = (pto.c_tp_basicsize > minsize or pto.c_tp_itemsize > 0) + self.flag_cpytype = True W_TypeObject.__init__(self, space, name, bases_w or [space.w_object], dict_w, force_new_layout=new_layout, is_heaptype=flag_heaptype) - self.flag_cpytype = True # if a sequence or a mapping, then set the flag to force it if pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_item: self.flag_map_or_seq = 'S' diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -168,6 +168,9 @@ # used to cache the type's __new__ function w_new_function = None + # set to True by cpyext _before_ it even calls __init__() below + flag_cpytype = False + @dont_look_inside def __init__(self, space, name, bases_w, dict_w, overridetypedef=None, force_new_layout=False, @@ -183,7 +186,6 @@ self.w_doc = space.w_None self.weak_subclasses = [] self.flag_heaptype = is_heaptype - self.flag_cpytype = False self.flag_abstract = False self.flag_sequence_bug_compat = False self.flag_map_or_seq = '?' # '?' means "don't know, check otherwise" @@ -194,18 +196,16 @@ else: layout = setup_user_defined_type(self, force_new_layout) self.layout = layout + self.qualname = self.getname(space) if self.flag_heaptype: w_qualname = self.dict_w.pop('__qualname__', None) if w_qualname is not None: - if not space.isinstance_w(w_qualname, space.w_unicode): + if space.isinstance_w(w_qualname, space.w_unicode): + self.qualname = space.unicode_w(w_qualname) + elif not self.flag_cpytype: raise oefmt(space.w_TypeError, "type __qualname__ must be a str, not %T", w_qualname) - self.qualname = space.unicode_w(w_qualname) - else: - self.qualname = self.getname(space) - else: - self.qualname = self.getname(space) if not is_mro_purely_of_types(self.mro_w): pass From pypy.commits at gmail.com Fri Mar 10 01:46:40 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 22:46:40 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Move PyObject_CallFinalizerFromDealloc() to a no-op stub Message-ID: <58c24bd0.6911190a.38795.9049@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90614:6e674eb4be9f Date: 2017-03-10 07:46 +0100 http://bitbucket.org/pypy/pypy/changeset/6e674eb4be9f/ Log: Move PyObject_CallFinalizerFromDealloc() to a no-op stub diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -618,6 +618,7 @@ 'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree', 'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free', + 'PyObject_CallFinalizerFromDealloc', ] TYPES = {} FORWARD_DECLS = [] @@ -1338,6 +1339,7 @@ source_dir / "import.c", source_dir / "_warnings.c", source_dir / "pylifecycle.c", + source_dir / "object.c", ] def build_eci(code, use_micronumpy=False, translating=False): diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -275,6 +275,9 @@ #define PyObject_Del PyObject_Free #define PyObject_DEL PyObject_Free +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); +#endif /* PyPy internal ----------------------------------- */ diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/src/object.c @@ -0,0 +1,15 @@ + +#include "Python.h" + +int +PyObject_CallFinalizerFromDealloc(PyObject *self) +{ + /* STUB */ + if (self->ob_type->tp_finalize) { + fprintf(stderr, "WARNING: PyObject_CallFinalizerFromDealloc() " + "not implemented (objects of type '%s')\n", + self->ob_type->tp_name); + self->ob_type->tp_finalize = NULL; /* only once */ + } + return 0; +} diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -2077,12 +2077,3 @@ it causes an exception to immediately be thrown; this is used for the throw() methods of generator objects.""" raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=-1) -def PyObject_CallFinalizerFromDealloc(space, self): - """Call tp_finalize() once, taking care of reference counting. - - NOT IMPLEMENTED: might need to keep the PyPy object alive even while - tp_dealloc() is called, which the GC doesn't do so far. - """ - raise NotImplementedError From pypy.commits at gmail.com Fri Mar 10 01:48:52 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 22:48:52 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58c24c54.42452e0a.175b1.92c1@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90615:8bbee7cc5217 Date: 2017-03-10 07:48 +0100 http://bitbucket.org/pypy/pypy/changeset/8bbee7cc5217/ Log: hg merge default diff --git a/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h b/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h --- a/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h +++ b/pypy/module/cpyext/include/_numpypy/numpy/npy_common.h @@ -1,8 +1,8 @@ #ifndef _NPY_COMMON_H_ #define _NPY_COMMON_H_ -typedef Py_intptr_t npy_intp; -typedef Py_uintptr_t npy_uintp; +typedef long npy_intp; +typedef unsigned long npy_uintp; typedef PY_LONG_LONG npy_longlong; typedef unsigned PY_LONG_LONG npy_ulonglong; typedef unsigned char npy_bool; diff --git a/rpython/jit/backend/x86/test/test_zmath.py b/rpython/jit/backend/x86/test/test_zmath.py --- a/rpython/jit/backend/x86/test/test_zmath.py +++ b/rpython/jit/backend/x86/test/test_zmath.py @@ -61,7 +61,12 @@ return -42 # ok def test_math(): - f = compile(fn, []) + # note: we use lldebug because in the normal optimizing mode, some + # calls may be completely inlined and constant-folded by the + # compiler (with -flto), e.g. atanh(0.3). They give then slightly + # different result than if they were executed at runtime. + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79973 + f = compile(fn, [], lldebug=True) res = f() if res >= 0: py.test.fail(repr(MathTests.TESTCASES[res])) diff --git a/rpython/memory/gctransform/test/test_shadowcolor.py b/rpython/memory/gctransform/test/test_shadowcolor.py --- a/rpython/memory/gctransform/test/test_shadowcolor.py +++ b/rpython/memory/gctransform/test/test_shadowcolor.py @@ -246,7 +246,7 @@ regalloc = allocate_registers(graph) assert summary_regalloc(regalloc) == [('a', 1)] * 2 + [('c', 0)] * 2 - at given(strategies.lists(strategies.booleans())) + at given(strategies.lists(strategies.booleans(), max_size=31)) def test_make_bitmask(boollist): index, bitmask = make_bitmask(boollist) if index is None: diff --git a/rpython/translator/platform/bsd.py b/rpython/translator/platform/bsd.py --- a/rpython/translator/platform/bsd.py +++ b/rpython/translator/platform/bsd.py @@ -11,7 +11,7 @@ standalone_only = [] shared_only = [] - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/cygwin.py b/rpython/translator/platform/cygwin.py --- a/rpython/translator/platform/cygwin.py +++ b/rpython/translator/platform/cygwin.py @@ -27,7 +27,7 @@ so_ext = 'dll' exe_ext = 'exe' - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -30,13 +30,16 @@ print 'in get_rpath_flags, rel_libdirs is not fixed up',rel_libdirs return self.rpath_flags - def _args_for_shared(self, args): - if hasattr(self, '_exe_name'): - target = os.path.basename(self._exe_name) + def _args_for_shared(self, args, **kwds): + if 'exe_name' in kwds: + target_basename = kwds['exe_name'].basename else: - target = '$(TARGET)' # inside a Makefile + target_basename = '$(TARGET)' + # The default '$(TARGET)' is used inside a Makefile. Otherwise + # we get the basename of the executable we're trying to build. return (list(self.shared_only) - + ['-dynamiclib', '-install_name', '@rpath/' + target, '-undefined', 'dynamic_lookup', '-flat_namespace'] + + ['-dynamiclib', '-install_name', '@rpath/' + target_basename, + '-undefined', 'dynamic_lookup', '-flat_namespace'] + args) def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py --- a/rpython/translator/platform/linux.py +++ b/rpython/translator/platform/linux.py @@ -24,7 +24,7 @@ from rpython.translator.platform.arch import s390x cflags = s390x.update_cflags(cflags) - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _include_dirs_for_libffi(self): diff --git a/rpython/translator/platform/netbsd.py b/rpython/translator/platform/netbsd.py --- a/rpython/translator/platform/netbsd.py +++ b/rpython/translator/platform/netbsd.py @@ -34,7 +34,7 @@ cc = get_env("CC", "gcc") super(Netbsd, self).__init__(cc) - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _preprocess_include_dirs(self, include_dirs): diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -54,9 +54,7 @@ args = [str(ofile) for ofile in ofiles] + link_args args += ['-o', str(exe_name)] if not standalone: - self._exe_name = str(exe_name) - args = self._args_for_shared(args) - del self._exe_name # remove, otherwise __eq__() fails + args = self._args_for_shared(args, exe_name=exe_name) self._execute_c_compiler(cc, args, exe_name, cwd=str(exe_name.dirpath())) return exe_name diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -200,7 +200,7 @@ def _linkfiles(self, link_files): return list(link_files) - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['/dll'] + args def check___thread(self): @@ -555,7 +555,7 @@ cc = 'gcc' Platform.__init__(self, cc) - def _args_for_shared(self, args): + def _args_for_shared(self, args, **kwds): return ['-shared'] + args def _include_dirs_for_libffi(self): From pypy.commits at gmail.com Fri Mar 10 02:34:48 2017 From: pypy.commits at gmail.com (arigo) Date: Thu, 09 Mar 2017 23:34:48 -0800 (PST) Subject: [pypy-commit] pypy default: cffi issue: unsure why, but ffi.new() didn't invoke add_memory_pressure. Message-ID: <58c25718.1f002e0a.ae306.93e5@mx.google.com> Author: Armin Rigo Branch: Changeset: r90616:d920a7a7a107 Date: 2017-03-10 08:34 +0100 http://bitbucket.org/pypy/pypy/changeset/d920a7a7a107/ Log: cffi issue: unsure why, but ffi.new() didn't invoke add_memory_pressure. diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py --- a/pypy/module/_cffi_backend/allocator.py +++ b/pypy/module/_cffi_backend/allocator.py @@ -4,6 +4,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rgc class W_Allocator(W_Root): @@ -20,10 +21,12 @@ if self.w_alloc is None: if self.should_clear_after_alloc: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=True) + flavor='raw', zero=True, + add_memory_pressure=True) else: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=False) + flavor='raw', zero=False, + add_memory_pressure=True) return cdataobj.W_CDataNewStd(space, ptr, ctype, length) else: w_raw_cdata = space.call_function(self.w_alloc, @@ -50,6 +53,7 @@ if self.w_free is not None: res.w_free = self.w_free res.register_finalizer(space) + rgc.add_memory_pressure(datasize) return res @unwrap_spec(w_init=WrappedDefault(None)) From pypy.commits at gmail.com Fri Mar 10 03:05:19 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 00:05:19 -0800 (PST) Subject: [pypy-commit] cffi default: Document the missing add_memory_pressure in PyPy <= 5.6 Message-ID: <58c25e3f.5ed7190a.1d79f.9978@mx.google.com> Author: Armin Rigo Branch: Changeset: r2897:0cfa29f0cd5e Date: 2017-03-10 08:37 +0100 http://bitbucket.org/cffi/cffi/changeset/0cfa29f0cd5e/ Log: Document the missing add_memory_pressure in PyPy <= 5.6 diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -501,23 +501,31 @@ # then replace `p = ffi.new("char[]", bigsize)` with: p = new_nonzero("char[]", bigsize) -Note anyway that it might be a better idea to use explicit calls to -``lib.malloc()`` and ``lib.free()``, because the memory returned by -``new()`` or ``new_allocator()()`` is only freed when the garbage -collector runs (i.e. not always instantly after the reference to the -object goes away, particularly but not only on PyPy). Example:: +**NOTE:** the following is a general warning that applies particularly +(but not only) to PyPy versions 5.6 or older (PyPy > 5.6 attempts to +account for the memory returned by ``ffi.new()`` or a custom allocator; +and CPython uses reference counting). If you do large allocations, then +there is no hard guarantee about when the memory will be freed. You +should avoid both ``new()`` and ``new_allocator()()`` if you want to be +sure that the memory is promptly released, e.g. before you allocate more +of it. - ffibuilder.cdef(""" - void *malloc(size_t size); - void free(void *ptr); - """) +An alternative is to declare and call the C ``malloc()`` and ``free()`` +functions, or some variant like ``mmap()`` and ``munmap()``. Then you +control exactly when the memory is allocated and freed. For example, +add these two lines to your existing ``ffibuilder.cdef()``:: - # then in your code: - p = lib.malloc(bigsize) - try: - ... - finally: - lib.free(p) + void *malloc(size_t size); + void free(void *ptr); + +and then call these two functions manually:: + + p = lib.malloc(bigsize) + try: + my_array = ffi.cast("some_other_type_than_void*", p) + ... + finally: + lib.free(p) ffi.init_once() diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -57,6 +57,10 @@ compare equal to ``ffi.cast("unsigned int", -1)``: it compares smaller, because ``-1 < 4294967295``. +* PyPy: ``ffi.new()`` and ``ffi.new_allocator()()`` did not record + "memory pressure", causing the GC to run too infrequently if you call + ``ffi.new()`` very often and/or with large arrays. Fixed in PyPy 5.7. + v1.9 ==== From pypy.commits at gmail.com Fri Mar 10 03:05:21 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 00:05:21 -0800 (PST) Subject: [pypy-commit] cffi default: Write a paragraph about __pypy__.add_memory_pressure() Message-ID: <58c25e41.15502e0a.cc5af.990a@mx.google.com> Author: Armin Rigo Branch: Changeset: r2898:f6e56069e9ab Date: 2017-03-10 09:05 +0100 http://bitbucket.org/cffi/cffi/changeset/f6e56069e9ab/ Log: Write a paragraph about __pypy__.add_memory_pressure() diff --git a/doc/source/using.rst b/doc/source/using.rst --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -452,6 +452,59 @@ with ``int foo();`` really means ``int foo(void);``.) +Memory pressure (PyPy) +---------------------- + +This paragraph applies only to PyPy, because its garbage collector (GC) +is different from CPython's. It is very common in C code to have pairs +of functions, one which performs memory allocations or acquires other +resources, and the other which frees them again. Depending on how you +structure your Python code, the freeing function is only called when the +GC decides a particular (Python) object can be freed. This occurs +notably in these cases: + +* If you use a ``__del__()`` method to call the freeing function. + +* If you use ``ffi.gc()``. + +* This does not occur if you call the freeing function at a + deterministic time, like in a regular ``try: finally:`` block. It + does however occur *inside a generator---* if the generator is not + explicitly exhausted but forgotten at a ``yield`` point, then the code + in the enclosing ``finally`` block is only invoked at the next GC. + +In these cases, you may have to use the built-in function +``__pypy__.add_memory_pressure(n)``. Its argument ``n`` is an estimate +of how much memory pressure to add. For example, if the pair of C +functions that we are talking about is ``malloc(n)`` and ``free()`` or +similar, you would call ``__pypy__.add_memory_pressure(n)`` after +``malloc(n)``. Doing so is not always a complete answer to the problem, +but it makes the next GC occur earlier, which is often enough. + +The same applies if the memory allocations are indirect, e.g. the C +function allocates some internal data structures. In that case, call +``__pypy__.add_memory_pressure(n)`` with an argument ``n`` that is an +rough estimation. Knowing the exact size is not important, and memory +pressure doesn't have to be manually brought down again after calling +the freeing function. If you are writing wrappers for the allocating / +freeing pair of functions, you should probably call +``__pypy__.add_memory_pressure()`` in the former even if the user may +invoke the latter at a known point with a ``finally:`` block. + +In case this solution is not sufficient, or if the acquired resource is +not memory but something else more limited (like file descriptors), then +there is no better way than restructuring your code to make sure the +freeing function is called at a known point and not indirectly by the +GC. + +Note that in PyPy <= 5.6 the discussion above also applies to +``ffi.new()``. In more recent versions of PyPy, both ``ffi.new()`` and +``ffi.new_allocator()()`` automatically account for the memory pressure +they create. (In case you need to support both older and newer PyPy's, +try calling ``__pypy__.add_memory_pressure()`` anyway; it is better to +overestimate than not account for the memory pressure.) + + .. _extern-python: .. _`extern "Python"`: From pypy.commits at gmail.com Fri Mar 10 03:37:26 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 00:37:26 -0800 (PST) Subject: [pypy-commit] pypy py3.5: issue #2494: @builtinfy some functions Message-ID: <58c265c6.65012e0a.68b17.9c1e@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90617:6f28fc42ee45 Date: 2017-03-10 09:37 +0100 http://bitbucket.org/pypy/pypy/changeset/6f28fc42ee45/ Log: issue #2494: @builtinfy some functions diff --git a/lib_pypy/_hashlib/__init__.py b/lib_pypy/_hashlib/__init__.py --- a/lib_pypy/_hashlib/__init__.py +++ b/lib_pypy/_hashlib/__init__.py @@ -4,6 +4,10 @@ from _cffi_ssl._stdssl.utility import (_str_to_ffi_buffer, _bytes_with_len, _str_from_buf) +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + def new(name, string=b''): h = Hash(name) h.update(string) @@ -132,6 +136,7 @@ # shortcut functions def make_new_hash(name, funcname): + @builtinify def new_hash(string=b''): return new(name, string) new_hash.__name__ = funcname @@ -142,6 +147,7 @@ globals()[_newname] = make_new_hash(_name, _newname) if hasattr(lib, 'PKCS5_PBKDF2_HMAC'): + @builtinify def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): if not isinstance(hash_name, str): raise TypeError("expected 'str' for name, but got %s" % type(hash_name)) diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py --- a/lib_pypy/_ssl/__init__.py +++ b/lib_pypy/_ssl/__init__.py @@ -1,3 +1,12 @@ from _cffi_ssl._stdssl import (_PROTOCOL_NAMES, _OPENSSL_API_VERSION, _test_decode_cert, _SSLContext) from _cffi_ssl._stdssl import * + + +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + +RAND_add = builtinify(RAND_add) +RAND_bytes = builtinify(RAND_bytes) +RAND_egd = builtinify(RAND_egd) +RAND_pseudo_bytes = builtinify(RAND_pseudo_bytes) From pypy.commits at gmail.com Fri Mar 10 04:04:05 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 01:04:05 -0800 (PST) Subject: [pypy-commit] pypy default: Bump the minimum OS/X version to 10.7 and get __thread. Message-ID: <58c26c05.cf4d2e0a.6f66f.93da@mx.google.com> Author: Armin Rigo Branch: Changeset: r90618:df3467fd878a Date: 2017-03-10 10:03 +0100 http://bitbucket.org/pypy/pypy/changeset/df3467fd878a/ Log: Bump the minimum OS/X version to 10.7 and get __thread. diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -25,9 +25,9 @@ IS_64_BITS = sys.maxint > 2147483647 SUPPORT__THREAD = ( # whether the particular C compiler supports __thread - sys.platform.startswith("linux")) # Linux works - # OS/X doesn't work, because we still target 10.5/10.6 and the - # minimum required version is 10.7. Windows doesn't work. Please + sys.platform.startswith("linux") or # Linux works + sys.platform.startswith("darwin")) # OS/X >= 10.7 works + # Windows doesn't work. Please # add other platforms here if it works on them. MAINDIR = os.path.dirname(os.path.dirname(__file__)) diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -6,10 +6,10 @@ # # Although Intel 32bit is supported since Apple Mac OS X 10.4, (and PPC since, ever) # the @rpath handling used in Darwin._args_for_shared is only availabe -# since 10.5, so we use that as minimum requirement. Bumped to 10.6 -# because 10.11 does not ship with 10.5 versions of libs +# since 10.5, so we use that as minimum requirement. Bumped to 10.7 +# to allow the use of thread-local in __thread in C. # -DARWIN_VERSION_MIN = '-mmacosx-version-min=10.6' +DARWIN_VERSION_MIN = '-mmacosx-version-min=10.7' class Darwin(posix.BasePosix): name = "darwin" From pypy.commits at gmail.com Fri Mar 10 06:23:45 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 03:23:45 -0800 (PST) Subject: [pypy-commit] pypy default: skip this test on older CPython 2.7.x Message-ID: <58c28cc1.09512e0a.fce61.a07b@mx.google.com> Author: Armin Rigo Branch: Changeset: r90619:46d13fbc30fe Date: 2017-03-10 12:23 +0100 http://bitbucket.org/pypy/pypy/changeset/46d13fbc30fe/ Log: skip this test on older CPython 2.7.x diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -980,6 +980,9 @@ assert module.size_of_instances(bar) >= size def test_app_cant_subclass_two_types(self): + import sys + if sys.version_info < (2, 7, 9): + skip("crashes on CPython (2.7.5 crashes, 2.7.9 is ok)") module = self.import_module(name='foo') try: class bar(module.fooType, module.UnicodeSubtype): From pypy.commits at gmail.com Fri Mar 10 07:32:55 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 04:32:55 -0800 (PST) Subject: [pypy-commit] pypy default: Hypothesis tests for `map(f, l)` where f() mutates `l` Message-ID: <58c29cf7.44002e0a.2ee55.a1bd@mx.google.com> Author: Armin Rigo Branch: Changeset: r90620:d042bd84c29f Date: 2017-03-10 13:32 +0100 http://bitbucket.org/pypy/pypy/changeset/d042bd84c29f/ Log: Hypothesis tests for `map(f, l)` where f() mutates `l` diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -313,3 +313,33 @@ assert type(max(1, 1.0, 1L)) is int assert type(max(1.0, 1L, 1)) is float assert type(max(1L, 1, 1.0)) is long + + +try: + from hypothesis import given, strategies, example +except ImportError: + pass +else: + @given(lst=strategies.lists(strategies.integers())) + def test_map_hypothesis(space, lst): + print lst + w_lst = space.appexec([space.wrap(lst[:])], """(lst): + def change(n): + if n & 3 == 1: + lst.pop(0) + elif n & 3 == 2: + lst.append(100) + return n * 2 + return map(change, lst) + """) + expected = [] + i = 0 + while i < len(lst): + n = lst[i] + if n & 3 == 1: + lst.pop(0) + elif n & 3 == 2: + lst.append(100) + expected.append(n * 2) + i += 1 + assert space.unwrap(w_lst) == expected From pypy.commits at gmail.com Fri Mar 10 08:01:33 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 05:01:33 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Issue #2495: forgot two calls to checksignals() after we get an EINTR Message-ID: <58c2a3ad.14542e0a.fc9ed.a407@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90621:4af601050e87 Date: 2017-03-10 14:01 +0100 http://bitbucket.org/pypy/pypy/changeset/4af601050e87/ Log: Issue #2495: forgot two calls to checksignals() after we get an EINTR diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py --- a/pypy/module/_multiprocessing/interp_semaphore.py +++ b/pypy/module/_multiprocessing/interp_semaphore.py @@ -372,6 +372,7 @@ except OSError as e: if e.errno == errno.EINTR: # again + _check_signals(space) continue elif e.errno in (errno.EAGAIN, errno.ETIMEDOUT): return False diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -469,6 +469,7 @@ break # normal path if rposix.get_saved_errno() != EINTR: raise exception_from_saved_errno(space, space.w_OSError) + space.getexecutioncontext().checksignals() secs = end_time - timeutils.monotonic(space) # retry if secs <= 0.0: break From pypy.commits at gmail.com Fri Mar 10 08:04:54 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 05:04:54 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58c2a476.45cc190a.f6828.9f01@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90623:2a55aab6ecce Date: 2017-03-10 14:04 +0100 http://bitbucket.org/pypy/pypy/changeset/2a55aab6ecce/ Log: hg merge default diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -636,3 +636,33 @@ assert max([], default=None) == None raises(TypeError, max, 1, default=0) raises(TypeError, max, default=1) + + +try: + from hypothesis import given, strategies, example +except ImportError: + pass +else: + @given(lst=strategies.lists(strategies.integers())) + def test_map_hypothesis(space, lst): + print lst + w_lst = space.appexec([space.wrap(lst[:])], """(lst): + def change(n): + if n & 3 == 1: + lst.pop(0) + elif n & 3 == 2: + lst.append(100) + return n * 2 + return list(map(change, lst)) + """) + expected = [] + i = 0 + while i < len(lst): + n = lst[i] + if n & 3 == 1: + lst.pop(0) + elif n & 3 == 2: + lst.append(100) + expected.append(n * 2) + i += 1 + assert space.unwrap(w_lst) == expected diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py --- a/pypy/module/_cffi_backend/allocator.py +++ b/pypy/module/_cffi_backend/allocator.py @@ -4,6 +4,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rgc class W_Allocator(W_Root): @@ -20,10 +21,12 @@ if self.w_alloc is None: if self.should_clear_after_alloc: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=True) + flavor='raw', zero=True, + add_memory_pressure=True) else: ptr = lltype.malloc(rffi.CCHARP.TO, datasize, - flavor='raw', zero=False) + flavor='raw', zero=False, + add_memory_pressure=True) return cdataobj.W_CDataNewStd(space, ptr, ctype, length) else: w_raw_cdata = space.call_function(self.w_alloc, @@ -50,6 +53,7 @@ if self.w_free is not None: res.w_free = self.w_free res.register_finalizer(space) + rgc.add_memory_pressure(datasize) return res @unwrap_spec(w_init=WrappedDefault(None)) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -955,6 +955,9 @@ assert module.size_of_instances(bar) >= size def test_app_cant_subclass_two_types(self): + import sys + if sys.version_info < (2, 7, 9): + skip("crashes on CPython (2.7.5 crashes, 2.7.9 is ok)") module = self.import_module(name='foo') try: class bar(module.fooType, module.UnicodeSubtype): diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -25,9 +25,9 @@ IS_64_BITS = sys.maxint > 2147483647 SUPPORT__THREAD = ( # whether the particular C compiler supports __thread - sys.platform.startswith("linux")) # Linux works - # OS/X doesn't work, because we still target 10.5/10.6 and the - # minimum required version is 10.7. Windows doesn't work. Please + sys.platform.startswith("linux") or # Linux works + sys.platform.startswith("darwin")) # OS/X >= 10.7 works + # Windows doesn't work. Please # add other platforms here if it works on them. MAINDIR = os.path.dirname(os.path.dirname(__file__)) diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -6,10 +6,10 @@ # # Although Intel 32bit is supported since Apple Mac OS X 10.4, (and PPC since, ever) # the @rpath handling used in Darwin._args_for_shared is only availabe -# since 10.5, so we use that as minimum requirement. Bumped to 10.6 -# because 10.11 does not ship with 10.5 versions of libs +# since 10.5, so we use that as minimum requirement. Bumped to 10.7 +# to allow the use of thread-local in __thread in C. # -DARWIN_VERSION_MIN = '-mmacosx-version-min=10.6' +DARWIN_VERSION_MIN = '-mmacosx-version-min=10.7' class Darwin(posix.BasePosix): name = "darwin" From pypy.commits at gmail.com Fri Mar 10 08:03:21 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 05:03:21 -0800 (PST) Subject: [pypy-commit] pypy default: backport 4af601050e87 Message-ID: <58c2a419.5d532e0a.c1ef8.a7b1@mx.google.com> Author: Armin Rigo Branch: Changeset: r90622:61d3de1e7c9a Date: 2017-03-10 14:02 +0100 http://bitbucket.org/pypy/pypy/changeset/61d3de1e7c9a/ Log: backport 4af601050e87 diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py --- a/pypy/module/_multiprocessing/interp_semaphore.py +++ b/pypy/module/_multiprocessing/interp_semaphore.py @@ -362,6 +362,7 @@ except OSError as e: if e.errno == errno.EINTR: # again + _check_signals(space) continue elif e.errno in (errno.EAGAIN, errno.ETIMEDOUT): return False From pypy.commits at gmail.com Fri Mar 10 09:03:22 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 06:03:22 -0800 (PST) Subject: [pypy-commit] cffi default: Improve the error message Message-ID: <58c2b22a.421a190a.a0807.a62a@mx.google.com> Author: Armin Rigo Branch: Changeset: r2899:181e00f1c6e6 Date: 2017-03-10 15:03 +0100 http://bitbucket.org/cffi/cffi/changeset/181e00f1c6e6/ Log: Improve the error message diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -756,21 +756,27 @@ def _load_backend_lib(backend, name, flags): + import os if name is None: if sys.platform != "win32": return backend.load_library(None, flags) name = "c" # Windows: load_library(None) fails, but this works # (backward compatibility hack only) - try: - if '.' not in name and '/' not in name: - raise OSError("library not found: %r" % (name,)) - return backend.load_library(name, flags) - except OSError: - import ctypes.util - path = ctypes.util.find_library(name) - if path is None: - raise # propagate the original OSError - return backend.load_library(path, flags) + first_error = None + if '.' in name or '/' in name or os.sep in name: + try: + return backend.load_library(name, flags) + except OSError as e: + first_error = e + import ctypes.util + path = ctypes.util.find_library(name) + if path is None: + msg = ("ctypes.util.find_library() did not manage " + "to locate a library called %r" % (name,)) + if first_error is not None: + msg = "%s. Additionally, %s" % (first_error, msg) + raise OSError(msg) + return backend.load_library(path, flags) def _make_ffi_library(ffi, libname, flags): backend = ffi._backend From pypy.commits at gmail.com Fri Mar 10 09:37:02 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Fri, 10 Mar 2017 06:37:02 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Add macros to log duration measurements using a timing event Message-ID: <58c2ba0e.1a4c2e0a.2baa3.aa6f@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2024:39dec57efb62 Date: 2017-03-06 11:36 +0100 http://bitbucket.org/pypy/stmgc/changeset/39dec57efb62/ Log: Add macros to log duration measurements using a timing event diff --git a/c8/stm/marker.c b/c8/stm/marker.c --- a/c8/stm/marker.c +++ b/c8/stm/marker.c @@ -3,9 +3,6 @@ # include "core.h" // silence flymake #endif -#define payload(marker) stm_timing_event_payload_data_t data = { &marker }; \ - stm_timing_event_payload_t payload = { \ - STM_EVENT_PAYLOAD_MARKER, data }; static bool marker_fetch(stm_thread_local_t *tl, stm_loc_marker_t *out_marker) { @@ -95,16 +92,18 @@ stm_loc_marker_t marker; marker_fetch_obj_write(start, contention, &marker); - payload(marker) + stm_marker_payload(marker) stmcb_timing_event(STM_SEGMENT->running_thread, - STM_CONTENTION_WRITE_READ, &payload); + STM_CONTENTION_WRITE_READ, + &stm_marker_payload); } static void _timing_become_inevitable(void) { stm_loc_marker_t marker; marker_fetch(STM_SEGMENT->running_thread, &marker); - payload(marker) + stm_marker_payload(marker) stmcb_timing_event(STM_SEGMENT->running_thread, - STM_BECOME_INEVITABLE, &payload); + STM_BECOME_INEVITABLE, + &stm_marker_payload); } diff --git a/c8/stm/marker.h b/c8/stm/marker.h --- a/c8/stm/marker.h +++ b/c8/stm/marker.h @@ -16,6 +16,12 @@ #define timing_become_inevitable() \ (timing_enabled() ? _timing_become_inevitable() : (void)0) +#define stm_marker_payload(marker) \ + stm_timing_event_payload_data_t stm_marker_data = \ + { .loc_marker = &marker }; \ + stm_timing_event_payload_t stm_marker_payload = \ + { STM_EVENT_PAYLOAD_MARKER, stm_marker_data }; + static inline void emit_wait(stm_thread_local_t *tl, enum stm_event_e event) { diff --git a/c8/stm/prof.c b/c8/stm/prof.c --- a/c8/stm/prof.c +++ b/c8/stm/prof.c @@ -42,8 +42,8 @@ struct timespec *duration = payload->data.duration; buf.extra_length = sprintf(buf.extra, "s%un%u", - duration->tv_sec, - duration->tv_nsec); + (uint32_t)duration->tv_sec, + (uint32_t)duration->tv_nsec); } } diff --git a/c8/stm/timing.h b/c8/stm/timing.h --- a/c8/stm/timing.h +++ b/c8/stm/timing.h @@ -14,3 +14,17 @@ stop.tv_sec - start.tv_sec, \ stop.tv_nsec - start.tv_nsec \ }; + +#define stm_duration_payload(duration) \ + stm_timing_event_payload_data_t stm_duration_data = \ + { .duration = &duration }; \ + stm_timing_event_payload_t stm_duration_payload = \ + { STM_EVENT_PAYLOAD_DURATION, stm_duration_data }; + +#define publish_event(event) \ + stmcb_timing_event(STM_SEGMENT->running_thread, event, &stm_duration_payload); + +#define stop_timer_and_publish(event) stop_timer() \ + get_duration() \ + stm_duration_payload(duration) \ + publish_event(event) From pypy.commits at gmail.com Fri Mar 10 09:37:04 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Fri, 10 Mar 2017 06:37:04 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Instrument write barrier slowpath Message-ID: <58c2ba10.5125190a.15189.a74a@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2025:39847b4aa093 Date: 2017-03-06 12:23 +0100 http://bitbucket.org/pypy/stmgc/changeset/39847b4aa093/ Log: Instrument write barrier slowpath diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -959,7 +959,9 @@ __attribute__((flatten)) void _stm_write_slowpath(object_t *obj) { + start_timer() write_slowpath_common(obj, /* mark_card */ false); + stop_timer_and_publish(STM_DURATION_WRITE_BARRIER) } diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -15,6 +15,7 @@ #include #include #include "list.h" +#include "timing.h" /************************************************************/ diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -22,6 +22,7 @@ #include "stm/detach.h" #include "stm/hashtable.h" #include "stm/queue.h" +#include "stm/timing.h" #include "stm/misc.c" #include "stm/list.c" diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -642,7 +642,6 @@ the given position and with the given maximum length. */ typedef int (*stm_expand_marker_fn)(char *seg_base, stm_loc_marker_t *marker, char *output, int output_size); -/* TODO generalize expand function */ int stm_set_timing_log(const char *profiling_file_name, int fork_mode, stm_expand_marker_fn expand_marker); From pypy.commits at gmail.com Fri Mar 10 09:37:07 2017 From: pypy.commits at gmail.com (tob...@masterthesis-vm) Date: Fri, 10 Mar 2017 06:37:07 -0800 (PST) Subject: [pypy-commit] stmgc c8-efficient-serial-execution: Adapt test interface of stmgc and fix crash if timing events are not enabled Message-ID: <58c2ba13.43142e0a.13044.745d@mx.google.com> Author: tobias at masterthesis-vm Branch: c8-efficient-serial-execution Changeset: r2026:3d8a3ec72e8a Date: 2017-03-06 17:02 +0100 http://bitbucket.org/pypy/stmgc/changeset/3d8a3ec72e8a/ Log: Adapt test interface of stmgc and fix crash if timing events are not enabled diff --git a/c8/stm/timing.h b/c8/stm/timing.h --- a/c8/stm/timing.h +++ b/c8/stm/timing.h @@ -22,7 +22,9 @@ { STM_EVENT_PAYLOAD_DURATION, stm_duration_data }; #define publish_event(event) \ - stmcb_timing_event(STM_SEGMENT->running_thread, event, &stm_duration_payload); + (timing_enabled() ? \ + stmcb_timing_event(STM_SEGMENT->running_thread, event, &stm_duration_payload) : \ + (void)0); #define stop_timer_and_publish(event) stop_timer() \ get_duration() \ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -180,7 +180,7 @@ }; typedef union { stm_loc_marker_t *loc_marker; - uint32_t duration; + struct timespec *duration; } stm_timing_event_payload_data_t; /* Wrapper for payload holding data type and data. */ typedef struct { From pypy.commits at gmail.com Fri Mar 10 10:14:36 2017 From: pypy.commits at gmail.com (mattip) Date: Fri, 10 Mar 2017 07:14:36 -0800 (PST) Subject: [pypy-commit] pypy default: fix for cpython2 (arigato) Message-ID: <58c2c2dc.011c190a.e4c93.ae32@mx.google.com> Author: Matti Picus Branch: Changeset: r90624:727fd362a2e6 Date: 2017-03-10 17:13 +0200 http://bitbucket.org/pypy/pypy/changeset/727fd362a2e6/ Log: fix for cpython2 (arigato) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1165,7 +1165,7 @@ Base1->tp_basicsize = sizeof(PyHeapTypeObject); Base2->tp_basicsize = sizeof(PyHeapTypeObject); Base12->tp_basicsize = sizeof(PyHeapTypeObject); - #ifndef PYPY_VERSION /* PyHeapTypeObject has no ht_qualname on PyPy */ + #ifndef PYPY_VERSION /* PyHeapTypeObject has no ht_qualname nor ht_name on PyPy */ #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 { PyObject * dummyname = PyBytes_FromString("dummy name"); @@ -1173,6 +1173,13 @@ ((PyHeapTypeObject*)Base2)->ht_qualname = dummyname; ((PyHeapTypeObject*)Base12)->ht_qualname = dummyname; } + #elif PY_MAJOR_VERSION == 2 + { + PyObject * dummyname = PyBytes_FromString("dummy name"); + ((PyHeapTypeObject*)Base1)->ht_name = dummyname; + ((PyHeapTypeObject*)Base2)->ht_name = dummyname; + ((PyHeapTypeObject*)Base12)->ht_name = dummyname; + } #endif #endif Base1->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; From pypy.commits at gmail.com Fri Mar 10 10:44:45 2017 From: pypy.commits at gmail.com (tobweber) Date: Fri, 10 Mar 2017 07:44:45 -0800 (PST) Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Update duration logging events with additional contributors Message-ID: <58c2c9ed.cf4d2e0a.6f66f.a790@mx.google.com> Author: Tobias Weber Branch: c8-overheads-instrumentation Changeset: r2027:4e133ea2cf02 Date: 2017-03-10 16:43 +0100 http://bitbucket.org/pypy/stmgc/changeset/4e133ea2cf02/ Log: Update duration logging events with additional contributors diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -961,7 +961,7 @@ void _stm_write_slowpath(object_t *obj) { start_timer() write_slowpath_common(obj, /* mark_card */ false); - stop_timer_and_publish(STM_DURATION_WRITE_BARRIER) + stop_timer_and_publish(STM_DURATION_WRITE_SLOWPATH) } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -575,11 +575,14 @@ STM_GC_MAJOR_DONE, /* execution duration profiling events */ - STM_DURATION_WRITE_BARRIER, + STM_DURATION_WRITE_GC_ONLY, + STM_DURATION_WRITE_SLOWPATH, STM_DURATION_VALIDATION, - STM_DURATION_COMMIT, + STM_DURATION_COMMIT_MINOR_GC, + STM_DURATION_COMMIT_ALL, STM_DURATION_MINOR_GC, - STM_DURATION_MAJOR_GC, + STM_DURATION_MAJOR_GC_LOG_ONLY, + STM_DURATION_MAJOR_GC_FULL, _STM_EVENT_N }; @@ -597,11 +600,14 @@ "gc major start", \ "gc major done", \ /* names of duration events */ \ + "duration of minor gc due to write", \ "duration of write slowpath", \ "duration of validation", \ - "duration of commit", \ + "duration of minor gc due to commit", \ + "duration of commit except minor gc", \ "duration of minor gc", \ - "duration of major gc" + "duration of major gc doing log clean up only",\ + "duration of full major gc" /* The markers pushed in the shadowstack are an odd number followed by a regular object pointer. */ From pypy.commits at gmail.com Fri Mar 10 11:12:03 2017 From: pypy.commits at gmail.com (rlamy) Date: Fri, 10 Mar 2017 08:12:03 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Kill test that does not make sense on py3 (due to namespace packages) Message-ID: <58c2d053.6911190a.38795.ac66@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90625:3d62b4040cbd Date: 2017-03-10 16:10 +0000 http://bitbucket.org/pypy/pypy/changeset/3d62b4040cbd/ Log: Kill test that does not make sense on py3 (due to namespace packages) diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -158,10 +158,6 @@ p.ensure(dir=True) p.join("__init__.py").mksymlinkto(os.devnull) - p = root.join("onlypyw") - p.ensure(dir=True) - p.join("__init__.pyw") - return str(root) @@ -776,11 +772,6 @@ finally: os.rmdir(name) - def test_dir_with_only_pyw(self): - def imp(): - import onlypyw - raises(ImportError, imp) - @pytest.mark.skipif(not hasattr(py.path.local, "mksymlinkto"), reason="requires symlinks") def test_dev_null_init_file(self): import devnullpkg From pypy.commits at gmail.com Sat Mar 11 02:42:02 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 23:42:02 -0800 (PST) Subject: [pypy-commit] pypy default: Merged in fniephaus/pypy-1/fniephaus/fix-typo-1488123166752 (pull request #519) Message-ID: <58c3aa4a.1c10190a.8b50d.c8cc@mx.google.com> Author: Armin Rigo Branch: Changeset: r90627:24f529c9697d Date: 2017-03-11 07:41 +0000 http://bitbucket.org/pypy/pypy/changeset/24f529c9697d/ Log: Merged in fniephaus/pypy-1/fniephaus/fix-typo-1488123166752 (pull request #519) Fix typo diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -202,12 +202,12 @@ ARGS = FUNC.ARGS if NON_VOID_ARGS != [T for T in ARGS if T is not lltype.Void]: raise Exception( - "in operation %r: caling a function with signature %r, " + "in operation %r: calling a function with signature %r, " "but passing actual arguments (ignoring voids) of types %r" % (op, FUNC, NON_VOID_ARGS)) if RESULT != FUNC.RESULT: raise Exception( - "in operation %r: caling a function with signature %r, " + "in operation %r: calling a function with signature %r, " "but the actual return type is %r" % (op, FUNC, RESULT)) # ok # get the 'elidable' and 'loopinvariant' flags from the function object From pypy.commits at gmail.com Sat Mar 11 02:42:08 2017 From: pypy.commits at gmail.com (fniephaus) Date: Fri, 10 Mar 2017 23:42:08 -0800 (PST) Subject: [pypy-commit] pypy fniephaus/fix-typo-1488123166752: Fix typo Message-ID: <58c3aa50.0da5190a.3c072.ca2c@mx.google.com> Author: Fabio Niephaus Branch: fniephaus/fix-typo-1488123166752 Changeset: r90626:7cba052b934f Date: 2017-02-26 15:33 +0000 http://bitbucket.org/pypy/pypy/changeset/7cba052b934f/ Log: Fix typo diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -202,12 +202,12 @@ ARGS = FUNC.ARGS if NON_VOID_ARGS != [T for T in ARGS if T is not lltype.Void]: raise Exception( - "in operation %r: caling a function with signature %r, " + "in operation %r: calling a function with signature %r, " "but passing actual arguments (ignoring voids) of types %r" % (op, FUNC, NON_VOID_ARGS)) if RESULT != FUNC.RESULT: raise Exception( - "in operation %r: caling a function with signature %r, " + "in operation %r: calling a function with signature %r, " "but the actual return type is %r" % (op, FUNC, RESULT)) # ok # get the 'elidable' and 'loopinvariant' flags from the function object From pypy.commits at gmail.com Sat Mar 11 02:59:12 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 10 Mar 2017 23:59:12 -0800 (PST) Subject: [pypy-commit] pypy default: import cffi/181e00f1c6e6 Message-ID: <58c3ae50.c35c2e0a.a1d29.c610@mx.google.com> Author: Armin Rigo Branch: Changeset: r90628:ec2b0693556e Date: 2017-03-11 08:58 +0100 http://bitbucket.org/pypy/pypy/changeset/ec2b0693556e/ Log: import cffi/181e00f1c6e6 diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -756,21 +756,27 @@ def _load_backend_lib(backend, name, flags): + import os if name is None: if sys.platform != "win32": return backend.load_library(None, flags) name = "c" # Windows: load_library(None) fails, but this works # (backward compatibility hack only) - try: - if '.' not in name and '/' not in name: - raise OSError("library not found: %r" % (name,)) - return backend.load_library(name, flags) - except OSError: - import ctypes.util - path = ctypes.util.find_library(name) - if path is None: - raise # propagate the original OSError - return backend.load_library(path, flags) + first_error = None + if '.' in name or '/' in name or os.sep in name: + try: + return backend.load_library(name, flags) + except OSError as e: + first_error = e + import ctypes.util + path = ctypes.util.find_library(name) + if path is None: + msg = ("ctypes.util.find_library() did not manage " + "to locate a library called %r" % (name,)) + if first_error is not None: + msg = "%s. Additionally, %s" % (first_error, msg) + raise OSError(msg) + return backend.load_library(path, flags) def _make_ffi_library(ffi, libname, flags): backend = ffi._backend diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -34,6 +34,9 @@ r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') _r_star_const_space = re.compile( # matches "* const " r"[*]\s*((const|volatile|restrict)\b\s*)+") +_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+" + r"\.\.\.") +_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.") def _get_parser(): global _parser_cache @@ -180,6 +183,10 @@ assert csource[p:p+3] == '...' csource = '%s __dotdotdot%d__ %s' % (csource[:p], number, csource[p+3:]) + # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__" + csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource) + # Replace "float ..." or "double..." with "__dotdotdotfloat__" + csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) # Replace all remaining "..." with the same name, "__dotdotdot__", # which is declared with a typedef for the purpose of C parsing. return csource.replace('...', ' __dotdotdot__ '), macros @@ -252,7 +259,8 @@ typenames += sorted(ctn) # csourcelines = ['typedef int %s;' % typename for typename in typenames] - csourcelines.append('typedef int __dotdotdot__;') + csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,' + ' __dotdotdot__;') csourcelines.append(csource) csource = '\n'.join(csourcelines) if lock is not None: @@ -311,6 +319,8 @@ for decl in iterator: if decl.name == '__dotdotdot__': break + else: + assert 0 # try: self._inside_extern_python = '__cffi_extern_python_stop' @@ -322,15 +332,15 @@ raise CDefError("typedef does not declare any name", decl) quals = 0 - if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) - and decl.type.type.names[-1] == '__dotdotdot__'): + if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and + decl.type.type.names[-1].startswith('__dotdotdot')): realtype = self._get_unknown_type(decl) elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and isinstance(decl.type.type.type, pycparser.c_ast.IdentifierType) and - decl.type.type.type.names == ['__dotdotdot__']): - realtype = model.unknown_ptr_type(decl.name) + decl.type.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_ptr_type(decl) else: realtype, quals = self._get_type_and_quals( decl.type, name=decl.name, partial_length_ok=True) @@ -832,24 +842,25 @@ def _get_unknown_type(self, decl): typenames = decl.type.type.names - assert typenames[-1] == '__dotdotdot__' - if len(typenames) == 1: + if typenames == ['__dotdotdot__']: return model.unknown_type(decl.name) - if (typenames[:-1] == ['float'] or - typenames[:-1] == ['double']): - # not for 'long double' so far - result = model.UnknownFloatType(decl.name) - else: - for t in typenames[:-1]: - if t not in ['int', 'short', 'long', 'signed', - 'unsigned', 'char']: - raise FFIError(':%d: bad usage of "..."' % - decl.coord.line) - result = model.UnknownIntegerType(decl.name) + if typenames == ['__dotdotdotint__']: + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef int... %s'" % decl.name + return model.UnknownIntegerType(decl.name) - if self._uses_new_feature is None: - self._uses_new_feature = "'typedef %s... %s'" % ( - ' '.join(typenames[:-1]), decl.name) + if typenames == ['__dotdotdotfloat__']: + # note: not for 'long double' so far + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef float... %s'" % decl.name + return model.UnknownFloatType(decl.name) - return result + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) + + def _get_unknown_ptr_type(self, decl): + if decl.type.type.type.names == ['__dotdotdot__']: + return model.unknown_ptr_type(decl.name) + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py @@ -2247,10 +2247,10 @@ assert str(e.value) == ("feature not supported with ffi.verify(), but only " "with ffi.set_source(): 'typedef int... t1'") ffi = FFI() - ffi.cdef("typedef unsigned long... t1;") + ffi.cdef("typedef double ... t1;") e = py.test.raises(VerificationError, ffi.verify, "") assert str(e.value) == ("feature not supported with ffi.verify(), but only " - "with ffi.set_source(): 'typedef unsigned long... t1'") + "with ffi.set_source(): 'typedef float... t1'") def test_const_fields(): ffi = FFI() From pypy.commits at gmail.com Sat Mar 11 06:46:25 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 03:46:25 -0800 (PST) Subject: [pypy-commit] pypy default: fix Message-ID: <58c3e391.55582e0a.a7339.cf9f@mx.google.com> Author: Armin Rigo Branch: Changeset: r90629:886fcb2b78f4 Date: 2017-03-11 12:46 +0100 http://bitbucket.org/pypy/pypy/changeset/886fcb2b78f4/ Log: fix diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -423,7 +423,7 @@ guard_false(i114, descr=...) --TICK-- i123 = arraylen_gc(p67, descr=) - i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=) + i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero_mpressure__Signed), 6, descr=) check_memory_error(i119) raw_store(i119, 0, i160, descr=) raw_store(i119, 2, i160, descr=) From pypy.commits at gmail.com Sat Mar 11 07:58:48 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 04:58:48 -0800 (PST) Subject: [pypy-commit] cffi default: mention static linking of the produced C file Message-ID: <58c3f488.48572e0a.87d22.dcb0@mx.google.com> Author: Armin Rigo Branch: Changeset: r2900:410ddef6d046 Date: 2017-03-11 13:58 +0100 http://bitbucket.org/cffi/cffi/changeset/410ddef6d046/ Log: mention static linking of the produced C file diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst --- a/doc/source/embedding.rst +++ b/doc/source/embedding.rst @@ -4,26 +4,44 @@ .. contents:: -You can use CFFI to generate a ``.so/.dll/.dylib`` which exports the -API of your choice to any C application that wants to link with this -``.so/.dll/.dylib``. +You can use CFFI to generate C code which exports the API of your choice +to any C application that wants to link with this C code. This API, +which you define yourself, ends up as the API of a ``.so/.dll/.dylib`` +library---or you can statically link it within a larger application. The general idea is as follows: -* You write and execute a Python script, which produces a - ``.so/.dll/.dylib`` file with the API of your choice. The script - also gives some Python code to be "frozen" inside the ``.so``. +* You write and execute a Python script, which produces a ``.c`` file + with the API of your choice (and optionally compile it into a + ``.so/.dll/.dylib``). The script also gives some Python code to be + "frozen" inside the ``.so``. -* At runtime, the C application loads this ``.so/.dll/.dylib`` without - having to know that it was produced by Python and CFFI. +* At runtime, the C application loads this ``.so/.dll/.dylib`` (or is + statically linked with the ``.c`` source) without having to know that + it was produced from Python and CFFI. * The first time a C function is called, Python is initialized and the frozen Python code is executed. -* The frozen Python code attaches Python functions that implement the +* The frozen Python code defines more Python functions that implement the C functions of your API, which are then used for all subsequent C function calls. +Possible use cases: + +* Exposing a library written in Python directly to C/C++ programs. + +* Using Python to make a "plug-in" for an existing program that is + already written to load them. + +* Using Python to implement part of a larger C/C++ application (with + static linking). + +* Writing a small C/C++ wrapper around Python, hiding the fact that the + application is actually written in Python (to make a custom + command-line interface; for distribution purposes; or simply to make + it a bit harder to reverse-engineer the application). + One of the goals of this approach is to be entirely independent from the CPython C API: no ``Py_Initialize()`` nor ``PyRun_SimpleString()`` nor even ``PyObject``. It works identically on CPython and PyPy. @@ -188,8 +206,11 @@ ``ffibuilder.emit_c_code("foo.c")`` and compile the resulting ``foo.c`` file using other means. CFFI's compilation logic is based on the standard library ``distutils`` package, which is really developed - and tested for the purpose of making CPython extension modules, not - other DLLs. + and tested for the purpose of making CPython extension modules; it + might not always be appropriate for making general DLLs. Also, just + getting the C code is what you need if you do not want to make a + stand-alone ``.so/.dll/.dylib`` file: this C file can be compiled + and statically linked as part of a larger application. More reading From pypy.commits at gmail.com Sat Mar 11 08:03:35 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 05:03:35 -0800 (PST) Subject: [pypy-commit] cffi default: move the 'use cases' before the 'how-to' section Message-ID: <58c3f5a7.50152e0a.10dc8.65c5@mx.google.com> Author: Armin Rigo Branch: Changeset: r2901:f3bf3ae4010f Date: 2017-03-11 14:03 +0100 http://bitbucket.org/cffi/cffi/changeset/f3bf3ae4010f/ Log: move the 'use cases' before the 'how-to' section diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst --- a/doc/source/embedding.rst +++ b/doc/source/embedding.rst @@ -9,6 +9,21 @@ which you define yourself, ends up as the API of a ``.so/.dll/.dylib`` library---or you can statically link it within a larger application. +Possible use cases: + +* Exposing a library written in Python directly to C/C++ programs. + +* Using Python to make a "plug-in" for an existing program that is + already written to load them. + +* Using Python to implement part of a larger C/C++ application (with + static linking). + +* Writing a small C/C++ wrapper around Python, hiding the fact that the + application is actually written in Python (to make a custom + command-line interface; for distribution purposes; or simply to make + it a bit harder to reverse-engineer the application). + The general idea is as follows: * You write and execute a Python script, which produces a ``.c`` file @@ -27,21 +42,6 @@ C functions of your API, which are then used for all subsequent C function calls. -Possible use cases: - -* Exposing a library written in Python directly to C/C++ programs. - -* Using Python to make a "plug-in" for an existing program that is - already written to load them. - -* Using Python to implement part of a larger C/C++ application (with - static linking). - -* Writing a small C/C++ wrapper around Python, hiding the fact that the - application is actually written in Python (to make a custom - command-line interface; for distribution purposes; or simply to make - it a bit harder to reverse-engineer the application). - One of the goals of this approach is to be entirely independent from the CPython C API: no ``Py_Initialize()`` nor ``PyRun_SimpleString()`` nor even ``PyObject``. It works identically on CPython and PyPy. From pypy.commits at gmail.com Sat Mar 11 08:07:55 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 05:07:55 -0800 (PST) Subject: [pypy-commit] cffi default: tweaks Message-ID: <58c3f6ab.13542e0a.25fb6.d558@mx.google.com> Author: Armin Rigo Branch: Changeset: r2902:e777d7ed61bc Date: 2017-03-11 14:07 +0100 http://bitbucket.org/cffi/cffi/changeset/e777d7ed61bc/ Log: tweaks diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst --- a/doc/source/embedding.rst +++ b/doc/source/embedding.rst @@ -13,7 +13,7 @@ * Exposing a library written in Python directly to C/C++ programs. -* Using Python to make a "plug-in" for an existing program that is +* Using Python to make a "plug-in" for an existing C/C++ program that is already written to load them. * Using Python to implement part of a larger C/C++ application (with @@ -118,6 +118,7 @@ """) ffibuilder.compile(target="plugin-1.5.*", verbose=True) + # or: ffibuilder.emit_c_code("my_plugin.c") Running the code above produces a *DLL*, i,e, a dynamically-loadable library. It is a file with the extension ``.dll`` on Windows, From pypy.commits at gmail.com Sat Mar 11 10:09:56 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 07:09:56 -0800 (PST) Subject: [pypy-commit] cffi default: pypy-c => pypy3-c Message-ID: <58c41344.58092e0a.ea952.d61e@mx.google.com> Author: Armin Rigo Branch: Changeset: r2903:4720f43ec0e0 Date: 2017-03-11 16:09 +0100 http://bitbucket.org/cffi/cffi/changeset/4720f43ec0e0/ Log: pypy-c => pypy3-c diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -570,7 +570,10 @@ # we need 'libpypy-c.{so,dylib}', which should be by # default located in 'sys.prefix/bin' for installed # systems. - pythonlib = "pypy-c" + if sys.version_info < (3,): + pythonlib = "pypy-c" + else: + pythonlib = "pypy3-c" if hasattr(sys, 'prefix'): ensure('library_dirs', os.path.join(sys.prefix, 'bin')) # On uninstalled pypy's, the libpypy-c is typically found in From pypy.commits at gmail.com Sat Mar 11 12:31:01 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 09:31:01 -0800 (PST) Subject: [pypy-commit] pypy default: import cffi/4720f43ec0e0 Message-ID: <58c43455.c819190a.7baba.dc6a@mx.google.com> Author: Armin Rigo Branch: Changeset: r90630:d4859be28252 Date: 2017-03-11 17:07 +0100 http://bitbucket.org/pypy/pypy/changeset/d4859be28252/ Log: import cffi/4720f43ec0e0 diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -570,7 +570,10 @@ # we need 'libpypy-c.{so,dylib}', which should be by # default located in 'sys.prefix/bin' for installed # systems. - pythonlib = "pypy-c" + if sys.version_info < (3,): + pythonlib = "pypy-c" + else: + pythonlib = "pypy3-c" if hasattr(sys, 'prefix'): ensure('library_dirs', os.path.join(sys.prefix, 'bin')) # On uninstalled pypy's, the libpypy-c is typically found in From pypy.commits at gmail.com Sat Mar 11 12:31:04 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 09:31:04 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Rename pypy-c -> pypy3-c and libpypy-c.so -> libpypy3-c.so. Message-ID: <58c43458.011c190a.e4c93.e1b3@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90631:17d88d201da3 Date: 2017-03-11 17:10 +0100 http://bitbucket.org/pypy/pypy/changeset/17d88d201da3/ Log: Rename pypy-c -> pypy3-c and libpypy-c.so -> libpypy3-c.so. Also attempt to rename the final "pypy" binary to "pypy3". diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py --- a/lib-python/3/subprocess.py +++ b/lib-python/3/subprocess.py @@ -1549,8 +1549,8 @@ def _pypy_install_libs_after_virtualenv(target_executable): # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv # - # PyPy 2.4.1 turned --shared on by default. This means the pypy binary - # depends on the 'libpypy-c.so' shared library to be able to run. + # We have --shared on by default. This means the pypy binary + # depends on the 'libpypy3-c.so' shared library to be able to run. # The virtualenv code existing at the time did not account for this # and would break. Try to detect that we're running under such a # virtualenv in the "Testing executable with" phase and copy the @@ -1560,7 +1560,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy3-c.so', 'libpypy3-c.dylib']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/3/venv/__init__.py b/lib-python/3/venv/__init__.py --- a/lib-python/3/venv/__init__.py +++ b/lib-python/3/venv/__init__.py @@ -230,7 +230,7 @@ # # PyPy extension: also copy the main library, not just the # small executable - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy3-c.so', 'libpypy3-c.dylib']: dest_library = os.path.join(binpath, libname) src_library = os.path.join(os.path.dirname(context.executable), libname) diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -158,7 +158,7 @@ import sys options = getattr(sys, 'pypy_translation_info', None) if options is None: - py.test.skip("not running on translated pypy " + py.test.skip("not running on translated pypy3 " "(btw, i would need options: %s)" % (ropts,)) for opt in ropts: @@ -166,7 +166,7 @@ break else: return - py.test.skip("need translated pypy with: %s, got %s" + py.test.skip("need translated pypy3 with: %s, got %s" %(ropts,options)) class LazyObjSpaceGetter(object): diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -15,13 +15,13 @@ arch = 'linux' cmd = 'wget "%s"' TAR_OPTIONS += ' --wildcards' - binfiles = "'*/bin/pypy' '*/bin/libpypy-c.so'" + binfiles = "'*/bin/pypy3' '*/bin/libpypy3-c.so'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): arch = 'osx' cmd = 'curl -O "%s"' - binfiles = "'*/bin/pypy'" + binfiles = "'*/bin/pypy3'" else: print 'Cannot determine the platform, please update this script' sys.exit(1) diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -252,7 +252,7 @@ return pypy_optiondescription def target(self, driver, args): - driver.exe_name = 'pypy-%(backend)s' + driver.exe_name = 'pypy3-%(backend)s' config = driver.config parser = self.opt_parser(config) diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -# This is pure Python code that handles the main entry point into "pypy". +# This is pure Python code that handles the main entry point into "pypy3". # See test/test_app_main. # Missing vs CPython: -b, -d, -x @@ -169,7 +169,7 @@ raise SystemExit def get_sys_executable(): - return getattr(sys, 'executable', 'pypy') + return getattr(sys, 'executable', 'pypy3') def print_help(*args): import os @@ -516,12 +516,6 @@ (not options["ignore_environment"] and os.getenv('PYTHONINSPECT'))): options["inspect"] = 1 -## We don't print the warning, because it offers no additional security -## in CPython either (http://bugs.python.org/issue14621) -## if (options["hash_randomization"] or os.getenv('PYTHONHASHSEED')): -## print >> sys.stderr, ( -## "Warning: pypy does not implement hash randomization") - if we_are_translated(): flags = [options[flag] for flag in sys_flags] sys.flags = type(sys.flags)(flags) @@ -768,7 +762,7 @@ args = (runpy._run_module_as_main, '__main__', False) break else: - # That's the normal path, "pypy stuff.py". + # That's the normal path, "pypy3 stuff.py". # We don't actually load via SourceFileLoader # because we require PyCF_ACCEPT_NULL_BYTES loader = SourceFileLoader('__main__', filename) @@ -815,7 +809,7 @@ STDLIB_WARNING = """\ debug: WARNING: Library path not found, using compiled-in sys.path. debug: WARNING: 'sys.prefix' will not be set. -debug: WARNING: Make sure the pypy binary is kept inside its tree of files. +debug: WARNING: Make sure the pypy3 binary is kept inside its tree of files. debug: WARNING: It is ok to create a symlink to it from somewhere else.""" def setup_bootstrap_path(executable): diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py --- a/pypy/interpreter/test/test_app_main.py +++ b/pypy/interpreter/test/test_app_main.py @@ -1,5 +1,5 @@ """ -Tests for the entry point of pypy-c, app_main.py. +Tests for the entry point of pypy3-c, app_main.py. """ from __future__ import with_statement import py @@ -1083,12 +1083,12 @@ from lib_pypy._pypy_interact import irc_header goal_dir = os.path.dirname(app_main) - # build a directory hierarchy like which contains both bin/pypy-c and - # lib/pypy1.2/* + # build a directory hierarchy like which contains both bin/pypy3-c and + # lib_pypy and lib-python prefix = udir.join('pathtest').ensure(dir=1) - fake_exe = 'bin/pypy-c' + fake_exe = 'bin/pypy3-c' if sys.platform == 'win32': - fake_exe = 'pypy-c.exe' + fake_exe = 'pypy3-c.exe' fake_exe = prefix.join(fake_exe).ensure(file=1) expected_path = [str(prefix.join(subdir).ensure(dir=1)) for subdir in ('lib_pypy', @@ -1119,11 +1119,11 @@ if self.tmp_dir.startswith(self.trunkdir): skip('TMPDIR is inside the PyPy source') sys.path.append(self.goal_dir) - tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c') + tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy3-c') try: os.chdir(self.tmp_dir) - # If we are running PyPy with a libpypy-c, the following + # If we are running PyPy with a libpypy3-c, the following # lines find the stdlib anyway. Otherwise, it is not found. expected_found = ( getattr(sys, 'pypy_translation_info', {}) @@ -1164,7 +1164,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy3-c') app_main.setup_bootstrap_path(pypy_c) newpath = sys.path[:] # we get at least lib_pypy @@ -1182,7 +1182,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy3-c') app_main.entry_point(pypy_c, [self.foo_py]) # assert it did not crash finally: diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py --- a/pypy/interpreter/test/test_targetpypy.py +++ b/pypy/interpreter/test/test_targetpypy.py @@ -6,7 +6,7 @@ def test_run(self): config = get_pypy_config(translating=False) entry_point = get_entry_point(config)[0] - entry_point(['pypy-c' , '-S', '-c', 'print 3']) + entry_point(['pypy3-c' , '-S', '-c', 'print 3']) def test_execute_source(space): _, d = create_entry_point(space, None) diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -67,7 +67,7 @@ with_traceback=True) space.appexec([], r"""(): import sys - sys.stderr.write('pypy version: %s.%s.%s\n' % + sys.stderr.write('pypy3 version: %s.%s.%s\n' % sys.pypy_version_info[:3]) sys.stderr.write('sys.path: %r\n' % (sys.path,)) """) diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -126,7 +126,7 @@ argv0 = space.getitem(argv, space.newint(0)) progname = space.unicode_w(argv0) else: - progname = u"pypy" + progname = u"pypy3" self.programname = rffi.unicode2wcharp(progname) lltype.render_immortal(self.programname) return self.programname diff --git a/pypy/module/pypyjit/test_pypy_c/conftest.py b/pypy/module/pypyjit/test_pypy_c/conftest.py --- a/pypy/module/pypyjit/test_pypy_c/conftest.py +++ b/pypy/module/pypyjit/test_pypy_c/conftest.py @@ -1,4 +1,4 @@ def pytest_addoption(parser): group = parser.getgroup("pypyjit options") group.addoption("--pypy", action="store", default=None, dest="pypy_c", - help="the location of the JIT enabled pypy-c") + help="the location of the JIT enabled pypy3-c") diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -116,7 +116,7 @@ in the parent directory of 'executable', and search from the 'home' entry instead of from the path to 'executable'. """ - search = 'pypy-c' if executable == '' else executable + search = 'pypy3-c' if executable == '' else executable search_pyvenv_cfg = 2 while True: dirname = resolvedirof(search) diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py --- a/pypy/module/sys/test/test_initpath.py +++ b/pypy/module/sys/test/test_initpath.py @@ -15,7 +15,7 @@ def test_find_stdlib(tmpdir): bin_dir = tmpdir.join('bin').ensure(dir=True) - pypy = bin_dir.join('pypy').ensure(file=True) + pypy = bin_dir.join('pypy3').ensure(file=True) build_hierarchy(tmpdir) path, prefix = find_stdlib(None, str(pypy)) assert prefix == tmpdir @@ -28,10 +28,10 @@ @py.test.mark.skipif('not hasattr(os, "symlink")') def test_find_stdlib_follow_symlink(tmpdir): - pypydir = tmpdir.join('opt', 'pypy-xxx') - pypy = pypydir.join('bin', 'pypy').ensure(file=True) + pypydir = tmpdir.join('opt', 'pypy3-xxx') + pypy = pypydir.join('bin', 'pypy3').ensure(file=True) build_hierarchy(pypydir) - pypy_sym = tmpdir.join('pypy_sym') + pypy_sym = tmpdir.join('pypy3_sym') os.symlink(str(pypy), str(pypy_sym)) path, prefix = find_stdlib(None, str(pypy_sym)) assert prefix == pypydir @@ -59,51 +59,51 @@ def test_find_executable(tmpdir, monkeypatch): from pypy.module.sys import initpath tmpdir = py.path.local(os.path.realpath(str(tmpdir))) - # /tmp/a/pypy - # /tmp/b/pypy + # /tmp/a/pypy3 + # /tmp/b/pypy3 # /tmp/c a = tmpdir.join('a').ensure(dir=True) b = tmpdir.join('b').ensure(dir=True) c = tmpdir.join('c').ensure(dir=True) - a.join('pypy').ensure(file=True) - b.join('pypy').ensure(file=True) + a.join('pypy3').ensure(file=True) + b.join('pypy3').ensure(file=True) # monkeypatch.setattr(os, 'access', lambda x, y: True) # if there is already a slash, don't do anything monkeypatch.chdir(tmpdir) - assert find_executable('a/pypy') == a.join('pypy') + assert find_executable('a/pypy3') == a.join('pypy3') # # if path is None, try abspath (if the file exists) monkeypatch.setenv('PATH', None) monkeypatch.chdir(a) - assert find_executable('pypy') == a.join('pypy') - monkeypatch.chdir(tmpdir) # no pypy there - assert find_executable('pypy') == '' + assert find_executable('pypy3') == a.join('pypy3') + monkeypatch.chdir(tmpdir) # no pypy3 there + assert find_executable('pypy3') == '' # # find it in path monkeypatch.setenv('PATH', str(a)) - assert find_executable('pypy') == a.join('pypy') + assert find_executable('pypy3') == a.join('pypy3') # # find it in the first dir in path monkeypatch.setenv('PATH', '%s%s%s' % (b, os.pathsep, a)) - assert find_executable('pypy') == b.join('pypy') + assert find_executable('pypy3') == b.join('pypy3') # # find it in the second, because in the first it's not there monkeypatch.setenv('PATH', '%s%s%s' % (c, os.pathsep, a)) - assert find_executable('pypy') == a.join('pypy') - # if pypy is found but it's not a file, ignore it - c.join('pypy').ensure(dir=True) - assert find_executable('pypy') == a.join('pypy') - # if pypy is found but it's not executable, ignore it + assert find_executable('pypy3') == a.join('pypy3') + # if pypy3 is found but it's not a file, ignore it + c.join('pypy3').ensure(dir=True) + assert find_executable('pypy3') == a.join('pypy3') + # if pypy3 is found but it's not executable, ignore it monkeypatch.setattr(os, 'access', lambda x, y: False) - assert find_executable('pypy') == '' + assert find_executable('pypy3') == '' # monkeypatch.setattr(os, 'access', lambda x, y: True) monkeypatch.setattr(initpath, 'we_are_translated', lambda: True) monkeypatch.setattr(initpath, '_WIN32', True) monkeypatch.setenv('PATH', str(a)) - a.join('pypy.exe').ensure(file=True) - assert find_executable('pypy') == a.join('pypy.exe') + a.join('pypy3.exe').ensure(file=True) + assert find_executable('pypy3') == a.join('pypy3.exe') def test_resolvedirof(tmpdir): assert resolvedirof('') == os.path.abspath(os.path.join(os.getcwd(), '..')) @@ -132,7 +132,7 @@ mydir = tmpdir.join('follow_pyvenv_cfg').ensure(dir=True) otherdir = tmpdir.join('otherdir').ensure(dir=True) bin_dir = mydir.join('bin').ensure(dir=True) - pypy = bin_dir.join('pypy').ensure(file=True) + pypy = bin_dir.join('pypy3').ensure(file=True) build_hierarchy(otherdir) for homedir in [otherdir, otherdir.join('bin')]: mydir.join('pyvenv.cfg').write('home = %s\n' % (homedir,)) diff --git a/pypy/sandbox/pypy_interact.py b/pypy/sandbox/pypy_interact.py --- a/pypy/sandbox/pypy_interact.py +++ b/pypy/sandbox/pypy_interact.py @@ -17,7 +17,7 @@ Note that you can get readline-like behavior with a tool like 'ledit', provided you use enough -u options: - ledit python -u pypy_interact.py pypy-c-sandbox -u + ledit python -u pypy_interact.py pypy3-c-sandbox -u """ import sys, os @@ -29,7 +29,7 @@ LIB_ROOT = os.path.dirname(os.path.dirname(pypy.__file__)) class PyPySandboxedProc(VirtualizedSandboxedProc, SimpleIOSandboxedProc): - argv0 = '/bin/pypy-c' + argv0 = '/bin/pypy3-c' virtual_cwd = '/tmp' virtual_env = {} virtual_console_isatty = True @@ -55,7 +55,7 @@ return Dir({ 'bin': Dir({ - 'pypy-c': RealFile(self.executable, mode=0111), + 'pypy3-c': RealFile(self.executable, mode=0111), 'lib-python': RealDir(os.path.join(libroot, 'lib-python'), exclude=exclude), 'lib_pypy': RealDir(os.path.join(libroot, 'lib_pypy'), @@ -126,4 +126,4 @@ sandproc.kill() if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/pypy/sandbox/test/test_pypy_interact.py b/pypy/sandbox/test/test_pypy_interact.py --- a/pypy/sandbox/test/test_pypy_interact.py +++ b/pypy/sandbox/test/test_pypy_interact.py @@ -26,9 +26,9 @@ assert_(argv[2] == 'bar', "bad argv[2]") env = os.environ.items() assert_(len(env) == 0, "empty environment expected") - assert_(argv[0] == '/bin/pypy-c', "bad argv[0]") - st = os.lstat('/bin/pypy-c') - assert_(stat.S_ISREG(st.st_mode), "bad st_mode for /bin/pypy-c") + assert_(argv[0] == '/bin/pypy3-c', "bad argv[0]") + st = os.lstat('/bin/pypy3-c') + assert_(stat.S_ISREG(st.st_mode), "bad st_mode for /bin/pypy3-c") for dirname in ['/bin/lib-python/' + VERSION, '/bin/lib_pypy']: st = os.stat(dirname) assert_(stat.S_ISDIR(st.st_mode), "bad st_mode for " + dirname) diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -69,7 +69,7 @@ if has != value: #print sys.pypy_translation_info py.test.skip("cannot runappdirect test: space needs %s = %s, "\ - "while pypy-c was built with %s" % (key, value, has)) + "while pypy3-c was built with %s" % (key, value, has)) for name in ('int', 'long', 'str', 'unicode', 'list', 'None', 'ValueError', 'OverflowError'): diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ packages PyPy, provided that it's already built. -It uses 'pypy/goal/pypy-c' and parts of the rest of the working +It uses 'pypy/goal/pypy3-c' and parts of the rest of the working copy. Usage: package.py [--options] --archive-name=pypy-VER-PLATFORM @@ -25,8 +25,7 @@ STDLIB_VER = "3" -# XXX: don't hardcode the version -POSIX_EXE = 'pypy3.5' +POSIX_EXE = 'pypy3' from pypy.tool.build_cffi_imports import (create_cffi_import_libraries, MissingDependenciesError, cffi_build_scripts) @@ -69,7 +68,7 @@ basedir = py.path.local(basedir) if not override_pypy_c: - basename = 'pypy-c' + basename = 'pypy3-c' if sys.platform == 'win32': basename += '.exe' pypy_c = basedir.join('pypy', 'goal', basename) @@ -99,10 +98,10 @@ if (sys.platform != 'win32' and # handled below not _fake and os.path.getsize(str(pypy_c)) < 500000): - # This pypy-c is very small, so it means it relies on libpypy_c.so. + # This pypy3-c is very small, so it means it relies on libpypy3_c.so. # If it would be bigger, it wouldn't. That's a hack. - libpypy_name = ('libpypy-c.so' if not sys.platform.startswith('darwin') - else 'libpypy-c.dylib') + libpypy_name = ('libpypy3-c.so' if not sys.platform.startswith('darwin') + else 'libpypy3-c.dylib') libpypy_c = pypy_c.new(basename=libpypy_name) if not libpypy_c.check(): raise PyPyCNotFound('Expected pypy to be mostly in %r, but did ' @@ -123,8 +122,8 @@ tgt = py.path.local(tgt) binaries.append((pypyw, tgt.new(purebasename=tgt.purebasename + 'w').basename)) print "Picking %s" % str(pypyw) - # Can't rename a DLL: it is always called 'libpypy-c.dll' - win_extras = ['libpypy-c.dll', 'sqlite3.dll'] + # Can't rename a DLL: it is always called 'libpypy3-c.dll' + win_extras = ['libpypy3-c.dll', 'sqlite3.dll'] if not options.no_tk: win_extras += ['tcl85.dll', 'tk85.dll'] @@ -144,7 +143,7 @@ else: print '"libs" dir with import library not found.' print 'You have to create %r' % (str(libsdir),) - print 'and copy libpypy-c.lib in there, renamed to python32.lib' + print 'and copy libpypy3-c.lib in there, renamed to python32.lib' # XXX users will complain that they cannot compile capi (cpyext) # modules for windows, also embedding pypy (i.e. in cffi) # will fail. @@ -207,14 +206,14 @@ else: open(str(archive), 'wb').close() os.chmod(str(archive), 0755) - if not _fake and not sys.platform == 'win32': - # create the pypy3 symlink - old_dir = os.getcwd() - os.chdir(str(bindir)) - try: - os.symlink(POSIX_EXE, 'pypy3') - finally: - os.chdir(old_dir) + #if not _fake and not sys.platform == 'win32': + # # create the pypy3 symlink + # old_dir = os.getcwd() + # os.chdir(str(bindir)) + # try: + # os.symlink(POSIX_EXE, 'pypy3') + # finally: + # os.chdir(old_dir) fix_permissions(pypydir) old_dir = os.getcwd() @@ -266,7 +265,7 @@ def package(*args, **kwds): import argparse if sys.platform == 'win32': - pypy_exe = 'pypy.exe' + pypy_exe = 'pypy3.exe' else: pypy_exe = POSIX_EXE parser = argparse.ArgumentParser() @@ -286,7 +285,7 @@ parser.add_argument('--nostrip', dest='nostrip', action='store_true', help='do not strip the exe, making it ~10MB larger') parser.add_argument('--rename_pypy_c', dest='pypy_c', type=str, default=pypy_exe, - help='target executable name, defaults to "pypy"') + help='target executable name, defaults to "%s"' % pypy_exe) parser.add_argument('--archive-name', dest='name', type=str, default='', help='pypy-VER-PLATFORM') parser.add_argument('--builddir', type=str, default='', @@ -294,7 +293,7 @@ parser.add_argument('--targetdir', type=str, default='', help='destination dir for archive') parser.add_argument('--override_pypy_c', type=str, default='', - help='use as pypy exe instead of pypy/goal/pypy-c') + help='use as pypy3 exe instead of pypy/goal/pypy3-c') options = parser.parse_args(args) if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"): diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -8,13 +8,13 @@ class TestPackaging: def setup_class(cls): - # make sure we have sort of pypy-c + # make sure we have sort of pypy3-c if sys.platform == 'win32': - basename = 'pypy-c.exe' - cls.rename_pypy_c = 'pypy-c' - cls.exe_name_in_archive = 'pypy-c.exe' + basename = 'pypy3-c.exe' + cls.rename_pypy_c = 'pypy3-c' + cls.exe_name_in_archive = 'pypy3-c.exe' else: - basename = 'pypy-c' + basename = 'pypy3-c' cls.rename_pypy_c = package.POSIX_EXE cls.exe_name_in_archive = os.path.join('bin', package.POSIX_EXE) cls.pypy_c = py.path.local(pypydir).join('goal', basename) @@ -96,7 +96,7 @@ bin = tmpdir.join('bin') .ensure(dir=True) file1 = tmpdir.join('file1').ensure(file=True) file2 = mydir .join('file2').ensure(file=True) - pypy = bin .join('pypy') .ensure(file=True) + pypy = bin .join('pypy3').ensure(file=True) # mydir.chmod(0700) bin.chmod(0700) From pypy.commits at gmail.com Sat Mar 11 12:31:06 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 09:31:06 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58c4345a.1099190a.fb67a.ddf4@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90632:e9f7ee0d2969 Date: 2017-03-11 17:12 +0100 http://bitbucket.org/pypy/pypy/changeset/e9f7ee0d2969/ Log: hg merge default diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -570,7 +570,10 @@ # we need 'libpypy-c.{so,dylib}', which should be by # default located in 'sys.prefix/bin' for installed # systems. - pythonlib = "pypy-c" + if sys.version_info < (3,): + pythonlib = "pypy-c" + else: + pythonlib = "pypy3-c" if hasattr(sys, 'prefix'): ensure('library_dirs', os.path.join(sys.prefix, 'bin')) # On uninstalled pypy's, the libpypy-c is typically found in @@ -756,21 +759,27 @@ def _load_backend_lib(backend, name, flags): + import os if name is None: if sys.platform != "win32": return backend.load_library(None, flags) name = "c" # Windows: load_library(None) fails, but this works # (backward compatibility hack only) - try: - if '.' not in name and '/' not in name: - raise OSError("library not found: %r" % (name,)) - return backend.load_library(name, flags) - except OSError: - import ctypes.util - path = ctypes.util.find_library(name) - if path is None: - raise # propagate the original OSError - return backend.load_library(path, flags) + first_error = None + if '.' in name or '/' in name or os.sep in name: + try: + return backend.load_library(name, flags) + except OSError as e: + first_error = e + import ctypes.util + path = ctypes.util.find_library(name) + if path is None: + msg = ("ctypes.util.find_library() did not manage " + "to locate a library called %r" % (name,)) + if first_error is not None: + msg = "%s. Additionally, %s" % (first_error, msg) + raise OSError(msg) + return backend.load_library(path, flags) def _make_ffi_library(ffi, libname, flags): backend = ffi._backend diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -34,6 +34,9 @@ r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') _r_star_const_space = re.compile( # matches "* const " r"[*]\s*((const|volatile|restrict)\b\s*)+") +_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+" + r"\.\.\.") +_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.") def _get_parser(): global _parser_cache @@ -180,6 +183,10 @@ assert csource[p:p+3] == '...' csource = '%s __dotdotdot%d__ %s' % (csource[:p], number, csource[p+3:]) + # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__" + csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource) + # Replace "float ..." or "double..." with "__dotdotdotfloat__" + csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) # Replace all remaining "..." with the same name, "__dotdotdot__", # which is declared with a typedef for the purpose of C parsing. return csource.replace('...', ' __dotdotdot__ '), macros @@ -252,7 +259,8 @@ typenames += sorted(ctn) # csourcelines = ['typedef int %s;' % typename for typename in typenames] - csourcelines.append('typedef int __dotdotdot__;') + csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,' + ' __dotdotdot__;') csourcelines.append(csource) csource = '\n'.join(csourcelines) if lock is not None: @@ -311,6 +319,8 @@ for decl in iterator: if decl.name == '__dotdotdot__': break + else: + assert 0 # try: self._inside_extern_python = '__cffi_extern_python_stop' @@ -322,15 +332,15 @@ raise CDefError("typedef does not declare any name", decl) quals = 0 - if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) - and decl.type.type.names[-1] == '__dotdotdot__'): + if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and + decl.type.type.names[-1].startswith('__dotdotdot')): realtype = self._get_unknown_type(decl) elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and isinstance(decl.type.type.type, pycparser.c_ast.IdentifierType) and - decl.type.type.type.names == ['__dotdotdot__']): - realtype = model.unknown_ptr_type(decl.name) + decl.type.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_ptr_type(decl) else: realtype, quals = self._get_type_and_quals( decl.type, name=decl.name, partial_length_ok=True) @@ -832,24 +842,25 @@ def _get_unknown_type(self, decl): typenames = decl.type.type.names - assert typenames[-1] == '__dotdotdot__' - if len(typenames) == 1: + if typenames == ['__dotdotdot__']: return model.unknown_type(decl.name) - if (typenames[:-1] == ['float'] or - typenames[:-1] == ['double']): - # not for 'long double' so far - result = model.UnknownFloatType(decl.name) - else: - for t in typenames[:-1]: - if t not in ['int', 'short', 'long', 'signed', - 'unsigned', 'char']: - raise FFIError(':%d: bad usage of "..."' % - decl.coord.line) - result = model.UnknownIntegerType(decl.name) + if typenames == ['__dotdotdotint__']: + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef int... %s'" % decl.name + return model.UnknownIntegerType(decl.name) - if self._uses_new_feature is None: - self._uses_new_feature = "'typedef %s... %s'" % ( - ' '.join(typenames[:-1]), decl.name) + if typenames == ['__dotdotdotfloat__']: + # note: not for 'long double' so far + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef float... %s'" % decl.name + return model.UnknownFloatType(decl.name) - return result + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) + + def _get_unknown_ptr_type(self, decl): + if decl.type.type.type.names == ['__dotdotdot__']: + return model.unknown_ptr_type(decl.name) + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1141,7 +1141,7 @@ Base1->tp_basicsize = sizeof(PyHeapTypeObject); Base2->tp_basicsize = sizeof(PyHeapTypeObject); Base12->tp_basicsize = sizeof(PyHeapTypeObject); - #ifndef PYPY_VERSION /* PyHeapTypeObject has no ht_qualname on PyPy */ + #ifndef PYPY_VERSION /* PyHeapTypeObject has no ht_qualname nor ht_name on PyPy */ #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 { PyObject * dummyname = PyBytes_FromString("dummy name"); @@ -1149,8 +1149,15 @@ ((PyHeapTypeObject*)Base2)->ht_qualname = dummyname; ((PyHeapTypeObject*)Base12)->ht_qualname = dummyname; } - #endif - #endif + #elif PY_MAJOR_VERSION == 2 + { + PyObject * dummyname = PyBytes_FromString("dummy name"); + ((PyHeapTypeObject*)Base1)->ht_name = dummyname; + ((PyHeapTypeObject*)Base2)->ht_name = dummyname; + ((PyHeapTypeObject*)Base12)->ht_name = dummyname; + } + #endif + #endif Base1->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; Base2->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; Base12->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE; diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -424,7 +424,7 @@ # NB. we get threads because '_hashlib' uses ffi callback/def_extern --THREAD-TICK-- i123 = arraylen_gc(p67, descr=) - i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=) + i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero_mpressure__Signed), 6, descr=) check_memory_error(i119) raw_store(i119, 0, i160, descr=) raw_store(i119, 2, i160, descr=) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py @@ -2247,10 +2247,10 @@ assert str(e.value) == ("feature not supported with ffi.verify(), but only " "with ffi.set_source(): 'typedef int... t1'") ffi = FFI() - ffi.cdef("typedef unsigned long... t1;") + ffi.cdef("typedef double ... t1;") e = py.test.raises(VerificationError, ffi.verify, "") assert str(e.value) == ("feature not supported with ffi.verify(), but only " - "with ffi.set_source(): 'typedef unsigned long... t1'") + "with ffi.set_source(): 'typedef float... t1'") def test_const_fields(): ffi = FFI() diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py --- a/rpython/jit/codewriter/call.py +++ b/rpython/jit/codewriter/call.py @@ -202,12 +202,12 @@ ARGS = FUNC.ARGS if NON_VOID_ARGS != [T for T in ARGS if T is not lltype.Void]: raise Exception( - "in operation %r: caling a function with signature %r, " + "in operation %r: calling a function with signature %r, " "but passing actual arguments (ignoring voids) of types %r" % (op, FUNC, NON_VOID_ARGS)) if RESULT != FUNC.RESULT: raise Exception( - "in operation %r: caling a function with signature %r, " + "in operation %r: calling a function with signature %r, " "but the actual return type is %r" % (op, FUNC, RESULT)) # ok # get the 'elidable' and 'loopinvariant' flags from the function object From pypy.commits at gmail.com Sat Mar 11 13:00:17 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 10:00:17 -0800 (PST) Subject: [pypy-commit] pypy py3.5: move to 'essential_modules' two modules without which 'pypy3-c' doesn't start Message-ID: <58c43b31.50162e0a.87a5d.084c@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90633:2d2fc5003a77 Date: 2017-03-11 18:59 +0100 http://bitbucket.org/pypy/pypy/changeset/2d2fc5003a77/ Log: move to 'essential_modules' two modules without which 'pypy3-c' doesn't start diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -16,7 +16,7 @@ essential_modules = set([ "exceptions", "_io", "sys", "builtins", "posix", "_warnings", - "itertools", "_frozen_importlib", "operator", + "itertools", "_frozen_importlib", "operator", "_locale", "struct", ]) if sys.platform == "win32": essential_modules.add("_winreg") @@ -33,9 +33,9 @@ # --allworkingmodules working_modules = default_modules.copy() working_modules.update([ - "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", + "_socket", "unicodedata", "mmap", "fcntl", "pwd", "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", - "zlib", "bz2", "struct", "_md5", "_minimal_curses", + "zlib", "bz2", "_md5", "_minimal_curses", "thread", "itertools", "pyexpat", "cpyext", "array", "binascii", "_multiprocessing", '_warnings', "_collections", "_multibytecodec", "_continuation", "_cffi_backend", From pypy.commits at gmail.com Sat Mar 11 13:33:44 2017 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 11 Mar 2017 10:33:44 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Skip tests that rely on __del__ to be called at interpreter shutdown. Message-ID: <58c44308.06152e0a.167e8.7636@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.5 Changeset: r90634:c56a1f35a86a Date: 2017-03-11 19:30 +0100 http://bitbucket.org/pypy/pypy/changeset/c56a1f35a86a/ Log: Skip tests that rely on __del__ to be called at interpreter shutdown. diff --git a/lib-python/3/test/test_io.py b/lib-python/3/test/test_io.py --- a/lib-python/3/test/test_io.py +++ b/lib-python/3/test/test_io.py @@ -3121,6 +3121,7 @@ t = _make_illegal_wrapper() self.assertRaises(TypeError, t.read) + @support.impl_detail("PyPy does not call __del__ at shutdown", pypy=False) def _check_create_at_shutdown(self, **kwargs): # Issue #20037: creating a TextIOWrapper at shutdown # shouldn't crash the interpreter. From pypy.commits at gmail.com Sat Mar 11 13:33:47 2017 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 11 Mar 2017 10:33:47 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Skip a test that tries to remove the unicodedata module. Message-ID: <58c4430b.1f002e0a.ae306.e3e4@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.5 Changeset: r90635:ad57bb3dff61 Date: 2017-03-11 19:14 +0100 http://bitbucket.org/pypy/pypy/changeset/ad57bb3dff61/ Log: Skip a test that tries to remove the unicodedata module. PyPy's unicodedata is builtin and always available from RPython code. diff --git a/lib-python/3/test/test_unicodedata.py b/lib-python/3/test/test_unicodedata.py --- a/lib-python/3/test/test_unicodedata.py +++ b/lib-python/3/test/test_unicodedata.py @@ -9,7 +9,7 @@ import sys import unittest import hashlib -from test.support import script_helper +from test.support import script_helper, impl_detail encoding = 'utf-8' errors = 'surrogatepass' @@ -224,6 +224,7 @@ class UnicodeMiscTest(UnicodeDatabaseTest): + @impl_detail("pypy's unicodedata module is always available", pypy=False) def test_failed_import_during_compiling(self): # Issue 4367 # Decoding \N escapes requires the unicodedata module. If it can't be From pypy.commits at gmail.com Sat Mar 11 16:27:41 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 11 Mar 2017 13:27:41 -0800 (PST) Subject: [pypy-commit] pypy default: The hack to prevent function inlining no longer works with "clang Message-ID: <58c46bcd.09532e0a.fd194.e8cd@mx.google.com> Author: Armin Rigo Branch: Changeset: r90636:fcd776f8ec36 Date: 2017-03-11 22:26 +0100 http://bitbucket.org/pypy/pypy/changeset/fcd776f8ec36/ Log: The hack to prevent function inlining no longer works with "clang -flto". Fix it by using the proper way. Even if it is compiler- dependent, we should be fine with two versions: MSVC and GCC/clang. This is an attempted fix for the OS/X tests of _continuation/greenlet/stackless. diff --git a/rpython/translator/c/src/stacklet/stacklet.c b/rpython/translator/c/src/stacklet/stacklet.c --- a/rpython/translator/c/src/stacklet/stacklet.c +++ b/rpython/translator/c/src/stacklet/stacklet.c @@ -16,6 +16,7 @@ * can redefine it to upwards growing, 1. */ #define STACK_DIRECTION 0 +#define STATIC_NOINLINE __attribute__((noinline)) static #include "src/stacklet/slp_platformselect.h" @@ -56,11 +57,6 @@ stacklet_thread_handle stack_thrd; /* the thread where the stacklet is */ }; -void *(*_stacklet_switchstack)(void*(*)(void*, void*), - void*(*)(void*, void*), void*) = NULL; -void (*_stacklet_initialstub)(struct stacklet_thread_s *, - stacklet_run_fn, void *) = NULL; - struct stacklet_thread_s { struct stacklet_s *g_stack_chain_head; /* NULL <=> running main */ char *g_current_stack_stop; @@ -252,8 +248,17 @@ return EMPTY_STACKLET_HANDLE; } -static void g_initialstub(struct stacklet_thread_s *thrd, - stacklet_run_fn run, void *run_arg) +STATIC_NOINLINE +void *_stacklet_switchstack(void *(*save_state)(void*, void*), + void *(*restore_state)(void*, void*), + void *extra) +{ + return slp_switch(save_state, restore_state, extra); +} + +STATIC_NOINLINE +void g_initialstub(struct stacklet_thread_s *thrd, + stacklet_run_fn run, void *run_arg) { struct stacklet_s *result; @@ -284,13 +289,6 @@ { struct stacklet_thread_s *thrd; - if (_stacklet_switchstack == NULL) { - /* set up the following global with an indirection, which is needed - to prevent any inlining */ - _stacklet_initialstub = g_initialstub; - _stacklet_switchstack = slp_switch; - } - thrd = malloc(sizeof(struct stacklet_thread_s)); if (thrd != NULL) memset(thrd, 0, sizeof(struct stacklet_thread_s)); @@ -311,7 +309,7 @@ thrd->g_current_stack_stop = ((char *)&stackmarker) + 1; thrd->g_current_stack_marker = (char *)&stackmarker; - _stacklet_initialstub(thrd, run, run_arg); + g_initialstub(thrd, run, run_arg); return thrd->g_source; } diff --git a/rpython/translator/c/src/stacklet/switch_x64_msvc.h b/rpython/translator/c/src/stacklet/switch_x64_msvc.h --- a/rpython/translator/c/src/stacklet/switch_x64_msvc.h +++ b/rpython/translator/c/src/stacklet/switch_x64_msvc.h @@ -5,3 +5,5 @@ void *(*restore_state)(void*, void*), void *extra); +#undef STATIC_NOINLINE +#define STATIC_NOINLINE static __declspec(noinline) diff --git a/rpython/translator/c/src/stacklet/switch_x86_msvc.h b/rpython/translator/c/src/stacklet/switch_x86_msvc.h --- a/rpython/translator/c/src/stacklet/switch_x86_msvc.h +++ b/rpython/translator/c/src/stacklet/switch_x86_msvc.h @@ -5,6 +5,9 @@ void *(*restore_state)(void*, void*), void *extra); +#undef STATIC_NOINLINE +#define STATIC_NOINLINE static __declspec(noinline) + #define WIN32_LEAN_AND_MEAN #include From pypy.commits at gmail.com Sun Mar 12 04:11:57 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 12 Mar 2017 00:11:57 -0800 (PST) Subject: [pypy-commit] pypy default: Implement _PyTraceMalloc_Track() and define a macro that can be Message-ID: <58c502cd.8f57190a.451c0.70ea@mx.google.com> Author: Armin Rigo Branch: Changeset: r90637:f0290a07535c Date: 2017-03-12 09:11 +0100 http://bitbucket.org/pypy/pypy/changeset/f0290a07535c/ Log: Implement _PyTraceMalloc_Track() and define a macro that can be tested for in CPython C extensions. See pymem.h for the exact way to say that... diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h --- a/pypy/module/cpyext/include/pymem.h +++ b/pypy/module/cpyext/include/pymem.h @@ -51,6 +51,25 @@ #define PyMem_Del PyMem_Free #define PyMem_DEL PyMem_FREE + +/* From CPython 3.6, with a different goal. _PyTraceMalloc_Track() + * is equivalent to __pypy__.add_memory_pressure(size); it works with + * or without the GIL. _PyTraceMalloc_Untrack() is an empty stub. + * You can check if these functions are available by using: + * + * #if defined(PYPY_TRACEMALLOC) || \ + * (PY_VERSION_HEX >= 0x03060000 && !defined(Py_LIMITED_API)) + */ +#define PYPY_TRACEMALLOC 1 + +typedef unsigned int _PyTraceMalloc_domain_t; + +PyAPI_FUNC(int) _PyTraceMalloc_Track(_PyTraceMalloc_domain_t domain, + uintptr_t ptr, size_t size); +PyAPI_FUNC(int) _PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, + uintptr_t ptr); + + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -475,3 +475,8 @@ with rffi.scoped_nonmovingbuffer(data) as buf: fwrite(buf, 1, count, fp) return 0 + + at cpython_api([lltype.Signed], lltype.Void) +def _PyPyGC_AddMemoryPressure(space, report): + from rpython.rlib import rgc + rgc.add_memory_pressure(report) diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c --- a/pypy/module/cpyext/src/pymem.c +++ b/pypy/module/cpyext/src/pymem.c @@ -4,3 +4,46 @@ { return malloc((n) ? (n) : 1); } + +int _PyTraceMalloc_Track(_PyTraceMalloc_domain_t domain, + uintptr_t ptr, size_t size) +{ + /* to avoid acquiring/releasing the GIL too often, only do it + if the total reported size exceeds 64KB. */ + static volatile long unreported_size = 0; + long prev, next, report; + + size += sizeof(long); + /* ^^^ to account for some alignment. Important, otherwise we'd + * collect sizes of, say, 1-bytes mallocs in 1-bytes increment */ + + retry: + report = 0; + prev = unreported_size; + next = prev + size; + if (next >= 65536) { + report = next; + next = 0; + } + if (prev != next) { +#ifdef _WIN32 + if (InterlockedCompareExchange(&unreported_size, next, prev) != prev) + goto retry; +#else + if (!__sync_bool_compare_and_swap(&unreported_size, prev, next)) + goto retry; +#endif + } + + if (report) { + PyGILState_STATE state = PyGILState_Ensure(); + _PyPyGC_AddMemoryPressure(report); + PyGILState_Release(state); + } +} + +int _PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, + uintptr_t ptr) +{ + /* nothing */ +} diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -214,6 +214,9 @@ class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): + from rpython.rlib import rgc + from pypy.interpreter import gateway + AppTestCpythonExtensionBase.setup_class.im_func(cls) tmpname = str(py.test.ensuretemp('out', dir=0)) if cls.runappdirect: @@ -221,6 +224,28 @@ else: cls.w_tmpname = cls.space.wrap(tmpname) + cls.total_mem = 0 + def add_memory_pressure(estimate): + assert estimate >= 0 + cls.total_mem += estimate + cls.orig_add_memory_pressure = [rgc.add_memory_pressure] + rgc.add_memory_pressure = add_memory_pressure + + def _reset_memory_pressure(space): + cls.total_mem = 0 + cls.w_reset_memory_pressure = cls.space.wrap( + gateway.interp2app(_reset_memory_pressure)) + + def _cur_memory_pressure(space): + return space.newint(cls.total_mem) + cls.w_cur_memory_pressure = cls.space.wrap( + gateway.interp2app(_cur_memory_pressure)) + + def teardown_class(cls): + from rpython.rlib import rgc + if hasattr(cls, 'orig_add_memory_pressure'): + [rgc.add_memory_pressure] = cls.orig_add_memory_pressure + def test_object_malloc(self): module = self.import_extension('foo', [ ("malloctest", "METH_NOARGS", @@ -323,6 +348,30 @@ a = module.empty_format('hello') assert isinstance(a, unicode) + def test_add_memory_pressure(self): + module = self.import_extension('foo', [ + ("foo", "METH_O", + """ + _PyTraceMalloc_Track(0, 0, PyInt_AsLong(args) - sizeof(long)); + Py_INCREF(Py_None); + return Py_None; + """)]) + self.reset_memory_pressure() + module.foo(42) + assert self.cur_memory_pressure() == 0 + module.foo(65000 - 42) + assert self.cur_memory_pressure() == 0 + module.foo(536) + assert self.cur_memory_pressure() == 65536 + module.foo(40000) + assert self.cur_memory_pressure() == 65536 + module.foo(40000) + assert self.cur_memory_pressure() == 65536 + 80000 + module.foo(35000) + assert self.cur_memory_pressure() == 65536 + 80000 + module.foo(35000) + assert self.cur_memory_pressure() == 65536 + 80000 + 70000 + class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase): """ PyBuffer_FillInfo populates the fields of a Py_buffer from its arguments. From pypy.commits at gmail.com Sun Mar 12 13:58:10 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 12 Mar 2017 10:58:10 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2492 Message-ID: <58c58c32.a215190a.ff681.0a2f@mx.google.com> Author: Armin Rigo Branch: Changeset: r90638:46167d4097c8 Date: 2017-03-12 18:57 +0100 http://bitbucket.org/pypy/pypy/changeset/46167d4097c8/ Log: Issue #2492 Modernize the interface of _minimal_curses. Also moves the logic to avoid any ``#include `` from anywhere except a single small C file. diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py --- a/pypy/module/_minimal_curses/fficurses.py +++ b/pypy/module/_minimal_curses/fficurses.py @@ -1,11 +1,8 @@ -""" The ffi for rpython, need to be imported for side effects +""" The ffi for rpython """ from rpython.rtyper.lltypesystem import rffi -from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.tool import rffi_platform -from rpython.rtyper.extfunc import register_external -from pypy.module._minimal_curses import interp_curses from rpython.translator.tool.cbuild import ExternalCompilationInfo # We cannot trust ncurses5-config, it's broken in various ways in @@ -58,86 +55,73 @@ eci = guess_eci() -INT = rffi.INT -INTP = lltype.Ptr(lltype.Array(INT, hints={'nolength':True})) -c_setupterm = rffi.llexternal('setupterm', [rffi.CCHARP, INT, INTP], INT, - compilation_info=eci) -c_tigetstr = rffi.llexternal('tigetstr', [rffi.CCHARP], rffi.CCHARP, - compilation_info=eci) -c_tparm = rffi.llexternal('tparm', [rffi.CCHARP, INT, INT, INT, INT, INT, - INT, INT, INT, INT], rffi.CCHARP, - compilation_info=eci) +# We should not use this 'eci' directly because it causes the #include +# of term.h to appear in all generated C sources, and term.h contains a +# poisonous quantity of #defines for common lower-case names like +# 'buttons' or 'lines' (!!!). It is basically dangerous to include +# term.h in any C source file that may contain unrelated source code. -ERR = rffi.CConstant('ERR', lltype.Signed) -OK = rffi.CConstant('OK', lltype.Signed) +include_lines = '\n'.join(['#include <%s>' % _incl for _incl in eci.includes]) +eci = eci.copy_without('includes') -def curses_setupterm(term, fd): - intp = lltype.malloc(INTP.TO, 1, flavor='raw') - err = rffi.cast(lltype.Signed, c_setupterm(term, fd, intp)) - try: - if err == ERR: - errret = rffi.cast(lltype.Signed, intp[0]) - if errret == 0: - msg = "setupterm: could not find terminal" - elif errret == -1: - msg = "setupterm: could not find terminfo database" - else: - msg = "setupterm: unknown error" - raise interp_curses.curses_error(msg) - interp_curses.module_info.setupterm_called = True - finally: - lltype.free(intp, flavor='raw') -def curses_setupterm_null_llimpl(fd): - curses_setupterm(lltype.nullptr(rffi.CCHARP.TO), fd) +eci = eci.merge(ExternalCompilationInfo( + post_include_bits=[ + "RPY_EXTERN char *rpy_curses_setupterm(char *, int);\n" + "RPY_EXTERN char *rpy_curses_tigetstr(char *);\n" + "RPY_EXTERN char *rpy_curses_tparm(char *, int, int, int, int," + " int, int, int, int, int);" + ], + separate_module_sources=[""" -def curses_setupterm_llimpl(term, fd): - ll_s = rffi.str2charp(term) - try: - curses_setupterm(ll_s, fd) - finally: - rffi.free_charp(ll_s) +%(include_lines)s -register_external(interp_curses._curses_setupterm_null, - [int], llimpl=curses_setupterm_null_llimpl, - export_name='_curses.setupterm_null') -register_external(interp_curses._curses_setupterm, - [str, int], llimpl=curses_setupterm_llimpl, - export_name='_curses.setupterm') +RPY_EXTERN +char *rpy_curses_setupterm(char *term, int fd) +{ + int errret = -42; + if (setupterm(term, fd, &errret) == ERR) { + switch (errret) { + case 0: + return "setupterm: could not find terminal"; + case -1: + return "setupterm: could not find terminfo database"; + default: + return "setupterm: unknown error"; + } + } + return NULL; +} -def check_setup_invoked(): - if not interp_curses.module_info.setupterm_called: - raise interp_curses.curses_error("must call (at least) setupterm() first") +RPY_EXTERN +char *rpy_curses_tigetstr(char *capname) +{ + char *res = tigetstr(capname); + if (res == (char *)-1) + res = NULL; + return res; +} -def tigetstr_llimpl(cap): - check_setup_invoked() - ll_cap = rffi.str2charp(cap) - try: - ll_res = c_tigetstr(ll_cap) - num = lltype.cast_ptr_to_int(ll_res) - if num == 0 or num == -1: - raise interp_curses.TermError() - res = rffi.charp2str(ll_res) - return res - finally: - rffi.free_charp(ll_cap) +RPY_EXTERN +char *rpy_curses_tparm(char *str, int x0, int x1, int x2, int x3, + int x4, int x5, int x6, int x7, int x8) +{ + return tparm(str, x0, x1, x2, x3, x4, x5, x6, x7, x8); +} -register_external(interp_curses._curses_tigetstr, [str], str, - export_name='_curses.tigetstr', llimpl=tigetstr_llimpl) +""" % globals()])) -def tparm_llimpl(s, args): - check_setup_invoked() - l = [0, 0, 0, 0, 0, 0, 0, 0, 0] - for i in range(min(len(args), 9)): - l[i] = args[i] - ll_s = rffi.str2charp(s) - # XXX nasty trick stolen from CPython - ll_res = c_tparm(ll_s, l[0], l[1], l[2], l[3], l[4], l[5], l[6], - l[7], l[8]) - rffi.free_charp(ll_s) - res = rffi.charp2str(ll_res) - return res -register_external(interp_curses._curses_tparm, [str, [int]], str, - export_name='_curses.tparm', llimpl=tparm_llimpl) +rpy_curses_setupterm = rffi.llexternal( + "rpy_curses_setupterm", [rffi.CCHARP, rffi.INT], rffi.CCHARP, + compilation_info=eci) +rpy_curses_tigetstr = rffi.llexternal( + "rpy_curses_tigetstr", [rffi.CCHARP], rffi.CCHARP, + compilation_info=eci) + +rpy_curses_tparm = rffi.llexternal( + "rpy_curses_tparm", [rffi.CCHARP, rffi.INT, rffi.INT, rffi.INT, rffi.INT, + rffi.INT, rffi.INT, rffi.INT, rffi.INT, rffi.INT], + rffi.CCHARP, + compilation_info=eci) diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -1,44 +1,24 @@ - from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import OperationError -from pypy.module._minimal_curses import _curses +from pypy.module._minimal_curses import fficurses +from rpython.rtyper.lltypesystem import lltype, rffi + class ModuleInfo: - def __init__(self): + def __init__(self, space): self.setupterm_called = False -module_info = ModuleInfo() +def check_setup_invoked(space): + if not space.fromcache(ModuleInfo).setupterm_called: + raise curses_error(space, "must call (at least) setupterm() first") -class curses_error(Exception): - def __init__(self, msg): - self.msg = msg -from rpython.annotator.classdesc import FORCE_ATTRIBUTES_INTO_CLASSES -from rpython.annotator.model import SomeString - -# this is necessary due to annmixlevel -FORCE_ATTRIBUTES_INTO_CLASSES[curses_error] = {'msg': SomeString()} - -def convert_error(space, error): - msg = error.msg +def curses_error(space, errmsg): w_module = space.getbuiltinmodule('_minimal_curses') w_exception_class = space.getattr(w_module, space.newtext('error')) - w_exception = space.call_function(w_exception_class, space.newtext(msg)) + w_exception = space.call_function(w_exception_class, space.newtext(errmsg)) return OperationError(w_exception_class, w_exception) -def _curses_setupterm_null(fd): - # NOT_RPYTHON - try: - _curses.setupterm(None, fd) - except _curses.error as e: - raise curses_error(e.args[0]) - -def _curses_setupterm(termname, fd): - # NOT_RPYTHON - try: - _curses.setupterm(termname, fd) - except _curses.error as e: - raise curses_error(e.args[0]) @unwrap_spec(fd=int) def setupterm(space, w_termname=None, fd=-1): @@ -47,48 +27,47 @@ space.newtext('stdout')) fd = space.int_w(space.call_function(space.getattr(w_stdout, space.newtext('fileno')))) - try: - if space.is_none(w_termname): - _curses_setupterm_null(fd) - else: - _curses_setupterm(space.text_w(w_termname), fd) - except curses_error as e: - raise convert_error(space, e) + if space.is_none(w_termname): + termname = None + else: + termname = space.text_w(w_termname) -class TermError(Exception): - pass + with rffi.scoped_str2charp(termname) as ll_term: + fd = rffi.cast(rffi.INT, fd) + ll_errmsg = fficurses.rpy_curses_setupterm(ll_term, fd) + if ll_errmsg: + raise curses_error(space, rffi.charp2str(ll_errmsg)) -def _curses_tigetstr(capname): - # NOT_RPYTHON - try: - res = _curses.tigetstr(capname) - except _curses.error as e: - raise curses_error(e.args[0]) - if res is None: - raise TermError - return res - -def _curses_tparm(s, args): - # NOT_RPYTHON - try: - return _curses.tparm(s, *args) - except _curses.error as e: - raise curses_error(e.args[0]) + space.fromcache(ModuleInfo).setupterm_called = True @unwrap_spec(capname='text') def tigetstr(space, capname): - try: - result = _curses_tigetstr(capname) - except TermError: - return space.w_None - except curses_error as e: - raise convert_error(space, e) - return space.newbytes(result) + check_setup_invoked(space) + with rffi.scoped_str2charp(capname) as ll_capname: + ll_result = fficurses.rpy_curses_tigetstr(ll_capname) + if ll_result: + return space.newbytes(rffi.charp2str(ll_result)) + else: + return space.w_None @unwrap_spec(s='text') def tparm(space, s, args_w): + check_setup_invoked(space) args = [space.int_w(a) for a in args_w] - try: - return space.newbytes(_curses_tparm(s, args)) - except curses_error as e: - raise convert_error(space, e) + # nasty trick stolen from CPython + x0 = args[0] if len(args) > 0 else 0 + x1 = args[1] if len(args) > 1 else 0 + x2 = args[2] if len(args) > 2 else 0 + x3 = args[3] if len(args) > 3 else 0 + x4 = args[4] if len(args) > 4 else 0 + x5 = args[5] if len(args) > 5 else 0 + x6 = args[6] if len(args) > 6 else 0 + x7 = args[7] if len(args) > 7 else 0 + x8 = args[8] if len(args) > 8 else 0 + with rffi.scoped_str2charp(s) as ll_str: + ll_result = fficurses.rpy_curses_tparm(ll_str, x0, x1, x2, x3, + x4, x5, x6, x7, x8) + if ll_result: + return space.newbytes(rffi.charp2str(ll_result)) + else: + raise curses_error(space, "tparm() returned NULL") diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py --- a/pypy/module/_minimal_curses/test/test_curses.py +++ b/pypy/module/_minimal_curses/test/test_curses.py @@ -76,19 +76,27 @@ """ def test_csetupterm(self): from rpython.translator.c.test.test_genc import compile - from pypy.module._minimal_curses import interp_curses + from rpython.rtyper.lltypesystem import lltype, rffi + from pypy.module._minimal_curses import fficurses + def runs_setupterm(): - interp_curses._curses_setupterm_null(1) + null = lltype.nullptr(rffi.CCHARP.TO) + fficurses.rpy_curses_setupterm(null, 1) fn = compile(runs_setupterm, []) fn() def test_ctgetstr(self): from rpython.translator.c.test.test_genc import compile - from pypy.module._minimal_curses import interp_curses + from rpython.rtyper.lltypesystem import lltype, rffi + from pypy.module._minimal_curses import fficurses + def runs_ctgetstr(): - interp_curses._curses_setupterm("xterm", 1) - return interp_curses._curses_tigetstr('cup') + with rffi.scoped_str2charp("xterm") as ll_term: + fficurses.rpy_curses_setupterm(ll_term, 1) + with rffi.scoped_str2charp("cup") as ll_capname: + ll = fficurses.rpy_curses_tigetstr(ll_capname) + return rffi.charp2str(ll) fn = compile(runs_ctgetstr, []) res = fn() @@ -96,11 +104,16 @@ def test_ctparm(self): from rpython.translator.c.test.test_genc import compile - from pypy.module._minimal_curses import interp_curses + from rpython.rtyper.lltypesystem import lltype, rffi + from pypy.module._minimal_curses import fficurses + def runs_tparm(): - interp_curses._curses_setupterm("xterm", 1) - cup = interp_curses._curses_tigetstr('cup') - return interp_curses._curses_tparm(cup, [5, 3]) + with rffi.scoped_str2charp("xterm") as ll_term: + fficurses.rpy_curses_setupterm(ll_term, 1) + with rffi.scoped_str2charp("cup") as ll_capname: + cup = fficurses.rpy_curses_tigetstr(ll_capname) + res = fficurses.rpy_curses_tparm(cup, 5, 3, 0, 0, 0, 0, 0, 0, 0) + return rffi.charp2str(res) fn = compile(runs_tparm, []) res = fn() diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py --- a/rpython/translator/tool/cbuild.py +++ b/rpython/translator/tool/cbuild.py @@ -334,3 +334,9 @@ d['separate_module_files'] = () d['separate_module_sources'] = () return ExternalCompilationInfo(**d) + + def copy_without(self, *names): + d = self._copy_attributes() + for name in names: + del d[name] + return ExternalCompilationInfo(**d) From pypy.commits at gmail.com Sun Mar 12 14:00:30 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 12 Mar 2017 11:00:30 -0700 (PDT) Subject: [pypy-commit] pypy default: this import for side-effects is not needed any more Message-ID: <58c58cbe.42152e0a.1c26e.9f65@mx.google.com> Author: Armin Rigo Branch: Changeset: r90639:2ad1b74a0c90 Date: 2017-03-12 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/2ad1b74a0c90/ Log: this import for side-effects is not needed any more diff --git a/pypy/module/_minimal_curses/__init__.py b/pypy/module/_minimal_curses/__init__.py --- a/pypy/module/_minimal_curses/__init__.py +++ b/pypy/module/_minimal_curses/__init__.py @@ -10,7 +10,6 @@ py.test.skip("no _curses or _minimal_curses module") # no _curses at all from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._minimal_curses import fficurses # for side effects class Module(MixedModule): From pypy.commits at gmail.com Sun Mar 12 14:03:14 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 12 Mar 2017 11:03:14 -0700 (PDT) Subject: [pypy-commit] pypy default: fix Message-ID: <58c58d62.85d5190a.5629e.0598@mx.google.com> Author: Armin Rigo Branch: Changeset: r90640:1f81f5bbc348 Date: 2017-03-12 19:02 +0100 http://bitbucket.org/pypy/pypy/changeset/1f81f5bbc348/ Log: fix diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -187,3 +187,5 @@ asmgcc---close enough that we can now make shadowstack the default even on Linux. This should remove a whole class of rare bugs introduced by asmgcc. + +.. branch: fniephaus/fix-typo-1488123166752 From pypy.commits at gmail.com Sun Mar 12 14:08:15 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 12 Mar 2017 11:08:15 -0700 (PDT) Subject: [pypy-commit] pypy default: Add lldebug=True to two compilation tests. This avoids the errors we Message-ID: <58c58e8f.8d63190a.ea185.1293@mx.google.com> Author: Armin Rigo Branch: Changeset: r90641:de9b0ab9954a Date: 2017-03-12 19:07 +0100 http://bitbucket.org/pypy/pypy/changeset/de9b0ab9954a/ Log: Add lldebug=True to two compilation tests. This avoids the errors we get with ``gcc -flto`` on Linux, at least with gcc versions 6.2.0 and 6.3.1. diff --git a/pypy/sandbox/test/test_pypy_interact.py b/pypy/sandbox/test/test_pypy_interact.py --- a/pypy/sandbox/test/test_pypy_interact.py +++ b/pypy/sandbox/test/test_pypy_interact.py @@ -74,7 +74,8 @@ def setup_module(mod): - t = Translation(mini_pypy_like_entry_point, backend='c', sandbox=True) + t = Translation(mini_pypy_like_entry_point, backend='c', sandbox=True, + lldebug=True) mod.executable = str(t.compile()) diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py --- a/rpython/translator/sandbox/test/test_sandbox.py +++ b/rpython/translator/sandbox/test/test_sandbox.py @@ -37,9 +37,9 @@ write_message(g, result, resulttype) g.flush() -def compile(f, gc='ref'): +def compile(f, gc='ref', **kwds): t = Translation(f, backend='c', sandbox=True, gc=gc, - check_str_without_nul=True) + check_str_without_nul=True, **kwds) return str(t.compile()) def run_in_subprocess(exe): @@ -198,7 +198,7 @@ l.append("x" * int(argv[2])) return int(len(l) > 1000) - exe = compile(entry_point, gc='hybrid') + exe = compile(entry_point, gc='hybrid', lldebug=True) pipe = subprocess.Popen([exe, '10', '10000'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) g = pipe.stdin From pypy.commits at gmail.com Sun Mar 12 20:27:43 2017 From: pypy.commits at gmail.com (sirtom67) Date: Sun, 12 Mar 2017 17:27:43 -0700 (PDT) Subject: [pypy-commit] cffi sirtom67/float_complex: Support for "double _Complex". Skip the test_c tests for now Message-ID: <58c5e77f.15502e0a.cc5af.1df6@mx.google.com> Author: Tom Krauss Branch: sirtom67/float_complex Changeset: r2904:2e64e82fb960 Date: 2017-03-12 19:26 -0500 http://bitbucket.org/cffi/cffi/changeset/2e64e82fb960/ Log: Support for "double _Complex". Skip the test_c tests for now since libffi returning non-sense. Add complex tests in testing/cffi1/test_recompiler.py diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2035,7 +2035,8 @@ } return PyFloat_FromDouble(value); } - if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) { + if ((cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) && + (cd->c_type->ct_size<16)) { double value = read_raw_float_data(cd->c_data, cd->c_type->ct_size); return PyComplex_FromDoubles(value, 0.0); } @@ -4104,6 +4105,16 @@ else goto bad_ffi_type; } + else if (ptypes->flags & CT_PRIMITIVE_COMPLEX) { + // as of March 2017, still no libffi support for complex + // but it fails silently. + if (strcmp(ptypes->name, "float _Complex") == 0) + ffitype = &ffi_type_complex_float; + else if (strcmp(ptypes->name, "double _Complex") == 0) + ffitype = &ffi_type_complex_double; + else + goto bad_ffi_type; + } else { switch (ptypes->size) { case 1: ffitype = &ffi_type_uint8; break; @@ -6594,6 +6605,10 @@ { return a + I*2.0*b; } +static double _Complex _testfunc25(double a, double b) +{ + return a + I*2.0*b; +} static PyObject *b__testfunc(PyObject *self, PyObject *args) { @@ -6628,6 +6643,7 @@ case 22: f = &_testfunc22; break; case 23: f = &_testfunc23; break; case 24: f = &_testfunc24; break; + case 25: f = &_testfunc25; break; default: PyErr_SetNone(PyExc_ValueError); return NULL; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -184,7 +184,7 @@ def test_complex_types(): INF = 1E200 * 1E200 - for name in ["float"]: #, "double"]: + for name in ["float", "double"]: p = new_primitive_type(name + " _Complex") assert bool(cast(p, 0)) assert bool(cast(p, INF)) @@ -1085,6 +1085,7 @@ assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192 def test_call_function_24(): + py.test.skip("libffi returning nonsense silently") BFloat = new_primitive_type("float") BFloatComplex = new_primitive_type("float _Complex") BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False) @@ -1094,6 +1095,18 @@ assert result.real == 1.25 # exact assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact +def test_call_function_25(): + py.test.skip("libffi returning nonsense silently") + BDouble = new_primitive_type("double") + BDoubleComplex = new_primitive_type("double _Complex") + BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False) + f = cast(BFunc3, _testfunc(25)) + result = f(1.25, 5.1) + assert type(result) == complex + assert result.real == 1.25 # exact + assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact + + def test_cannot_call_with_a_autocompleted_struct(): BSChar = new_primitive_type("signed char") BDouble = new_primitive_type("double") diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -110,6 +110,9 @@ PyInt_FromLong((long)x) : \ PyLong_FromLongLong((long long)x))) +#define _cffi_from_c_float__Complex(x) PyComplex_FromDoubles(crealf(x), cimagf(x)) +#define _cffi_from_c_double__Complex(x) PyComplex_FromDoubles(creal(x), cimag(x)) + #define _cffi_to_c_int(o, type) \ ((type)( \ sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -1994,6 +1994,30 @@ """) assert lib.f1(52).a == 52 +def test_function_returns_float_complex(): + ffi = FFI() + ffi.cdef("float _Complex f1(float a, float b);"); + lib = verify(ffi, "test_function_returns_float_complex", """ + #include + static float _Complex f1 (float a, float b) { return a + I*2.0*b; } + """) + result = lib.f1(1.25, 5.1) + assert type(result) == complex + assert result.real == 1.25 # exact + assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact + +def test_function_returns_double_complex(): + ffi = FFI() + ffi.cdef("double _Complex f1(double a, double b);"); + lib = verify(ffi, "test_function_returns_double_complex", """ + #include + static double _Complex f1 (double a, double b) { return a + I*2.0*b; } + """) + result = lib.f1(1.25, 5.1) + assert type(result) == complex + assert result.real == 1.25 # exact + assert result.imag == 2*5.1 # exact + def test_typedef_array_dotdotdot(): ffi = FFI() ffi.cdef(""" From pypy.commits at gmail.com Sun Mar 12 21:09:11 2017 From: pypy.commits at gmail.com (sirtom67) Date: Sun, 12 Mar 2017 18:09:11 -0700 (PDT) Subject: [pypy-commit] cffi sirtom67/float_complex: fix test (now apparently cast(p, -1.1j) == cast(p, -1.1j)) Message-ID: <58c5f137.1a4a2e0a.2be3b.1235@mx.google.com> Author: Tom Krauss Branch: sirtom67/float_complex Changeset: r2906:38562ef9d8f7 Date: 2017-03-12 20:08 -0500 http://bitbucket.org/cffi/cffi/changeset/38562ef9d8f7/ Log: fix test (now apparently cast(p, -1.1j) == cast(p, -1.1j)) Remove a printf. diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2965,7 +2965,6 @@ Py_complex value = read_raw_complex_data(cd->c_data, cd->c_type->ct_size); PyObject *op = PyComplex_FromCComplex(value); PyComplexObject *opc = (PyComplexObject *) op; - printf("%f %f\n",opc->cval.real,opc->cval.imag); return op; } PyErr_Format(PyExc_TypeError, "complex() not supported on cdata '%s'", diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -206,7 +206,7 @@ assert complex(cast(p, 1E200+3j)) == INF+3j # limited range assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range - assert cast(p, -1.1j) != cast(p, -1.1j) + assert cast(p, -1.1j) == cast(p, -1.1j) assert repr(complex(cast(p, -0.0)).real) == '-0.0' #assert repr(complex(cast(p, -0j))) == '-0j' # http://bugs.python.org/issue29602 assert complex(cast(p, b'\x09')) == 9.0 From pypy.commits at gmail.com Sun Mar 12 21:09:08 2017 From: pypy.commits at gmail.com (sirtom67) Date: Sun, 12 Mar 2017 18:09:08 -0700 (PDT) Subject: [pypy-commit] cffi sirtom67/float_complex: Merge default in. Message-ID: <58c5f134.124b2e0a.8f89d.1e6a@mx.google.com> Author: Tom Krauss Branch: sirtom67/float_complex Changeset: r2905:ab7c2855f10f Date: 2017-03-12 19:43 -0500 http://bitbucket.org/cffi/cffi/changeset/ab7c2855f10f/ Log: Merge default in. diff too long, truncating to 2000 out of 2939 lines diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -15,6 +15,16 @@ [Mailing list](https://groups.google.com/forum/#!forum/python-cffi) -To run tests under CPython, run: +Testing/development tips +------------------------ -python setup.py build_ext -i +To run tests under CPython, run:: + + pip install pytest # if you don't have py.test already + pip install pycparser + python setup.py build_ext -f -i + py.test c/ testing/ + +If you run in another directory (either the tests or another program), +you should use the environment variable ``PYTHONPATH=/path`` to point +to the location that contains the ``_cffi_backend.so`` just compiled. diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.9.2" +#define CFFI_VERSION "1.10.0" #ifdef MS_WIN32 #include @@ -460,6 +460,8 @@ static PyObject * get_field_name(CTypeDescrObject *ct, CFieldObject *cf); /* forward */ +/* returns 0 if the struct ctype is opaque, 1 if it is not, or -1 if + an exception occurs */ #define force_lazy_struct(ct) \ ((ct)->ct_stuff != NULL ? 1 : do_realize_lazy_struct(ct)) @@ -1014,8 +1016,23 @@ /*READ(data, ct->ct_size)*/ value = read_raw_unsigned_data(data, ct->ct_size); - if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) + if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) { + if (ct->ct_flags & CT_IS_BOOL) { + PyObject *x; + switch ((int)value) { + case 0: x = Py_False; break; + case 1: x = Py_True; break; + default: + PyErr_Format(PyExc_ValueError, + "got a _Bool of value %d, expected 0 or 1", + (int)value); + return NULL; + } + Py_INCREF(x); + return x; + } return PyInt_FromLong((long)value); + } else return PyLong_FromUnsignedLongLong(value); } @@ -1256,6 +1273,20 @@ } static int +must_be_array_of_zero_or_one(const char *data, Py_ssize_t n) +{ + Py_ssize_t i; + for (i = 0; i < n; i++) { + if (((unsigned char)data[i]) > 1) { + PyErr_SetString(PyExc_ValueError, + "an array of _Bool can only contain \\x00 or \\x01"); + return -1; + } + } + return 0; +} + +static int convert_array_from_object(char *data, CTypeDescrObject *ct, PyObject *init) { /* used by convert_from_object(), and also to decode lists/tuples/unicodes @@ -1302,6 +1333,9 @@ if (n != ct->ct_length) n++; srcdata = PyBytes_AS_STRING(init); + if (ctitem->ct_flags & CT_IS_BOOL) + if (must_be_array_of_zero_or_one(srcdata, n) < 0) + return -1; memcpy(data, srcdata, n); return 0; } @@ -1472,12 +1506,15 @@ unsigned PY_LONG_LONG value = _my_PyLong_AsUnsignedLongLong(init, 1); if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred()) return -1; - if (ct->ct_flags & CT_IS_BOOL) - if (value & ~1) /* value != 0 && value != 1 */ + if (ct->ct_flags & CT_IS_BOOL) { + if (value > 1ULL) /* value != 0 && value != 1 */ goto overflow; - write_raw_integer_data(buf, value, ct->ct_size); - if (value != read_raw_unsigned_data(buf, ct->ct_size)) - goto overflow; + } + else { + write_raw_integer_data(buf, value, ct->ct_size); + if (value != read_raw_unsigned_data(buf, ct->ct_size)) + goto overflow; + } write_raw_integer_data(data, value, ct->ct_size); return 0; } @@ -2047,47 +2084,97 @@ static PyObject *cdata_richcompare(PyObject *v, PyObject *w, int op) { - int res; + int v_is_ptr, w_is_ptr; PyObject *pyres; - char *v_cdata, *w_cdata; assert(CData_Check(v)); - if (!CData_Check(w)) { + + /* Comparisons involving a primitive cdata work differently than + * comparisons involving a struct/array/pointer. + * + * If v or w is a struct/array/pointer, then the other must be too + * (otherwise we return NotImplemented and leave the case to + * Python). If both are, then we compare the addresses. + * + * If v and/or w is a primitive cdata, then we convert the cdata(s) + * to regular Python objects and redo the comparison there. + */ + + v_is_ptr = !(((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY); + w_is_ptr = CData_Check(w) && + !(((CDataObject *)w)->c_type->ct_flags & CT_PRIMITIVE_ANY); + + if (v_is_ptr && w_is_ptr) { + int res; + char *v_cdata = ((CDataObject *)v)->c_data; + char *w_cdata = ((CDataObject *)w)->c_data; + + switch (op) { + case Py_EQ: res = (v_cdata == w_cdata); break; + case Py_NE: res = (v_cdata != w_cdata); break; + case Py_LT: res = (v_cdata < w_cdata); break; + case Py_LE: res = (v_cdata <= w_cdata); break; + case Py_GT: res = (v_cdata > w_cdata); break; + case Py_GE: res = (v_cdata >= w_cdata); break; + default: res = -1; + } + pyres = res ? Py_True : Py_False; + } + else if (v_is_ptr || w_is_ptr) { pyres = Py_NotImplemented; - goto done; - } - - if ((op != Py_EQ && op != Py_NE) && - ((((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY) || - (((CDataObject *)w)->c_type->ct_flags & CT_PRIMITIVE_ANY))) - goto Error; - - v_cdata = ((CDataObject *)v)->c_data; - w_cdata = ((CDataObject *)w)->c_data; - - switch (op) { - case Py_EQ: res = (v_cdata == w_cdata); break; - case Py_NE: res = (v_cdata != w_cdata); break; - case Py_LT: res = (v_cdata < w_cdata); break; - case Py_LE: res = (v_cdata <= w_cdata); break; - case Py_GT: res = (v_cdata > w_cdata); break; - case Py_GE: res = (v_cdata >= w_cdata); break; - default: res = -1; - } - pyres = res ? Py_True : Py_False; - done: + } + else { + PyObject *aa[2]; + int i; + + aa[0] = v; Py_INCREF(v); + aa[1] = w; Py_INCREF(w); + pyres = NULL; + + for (i = 0; i < 2; i++) { + v = aa[i]; + if (!CData_Check(v)) + continue; + w = convert_to_object(((CDataObject *)v)->c_data, + ((CDataObject *)v)->c_type); + if (w == NULL) + goto error; + if (CData_Check(w)) { + Py_DECREF(w); + PyErr_Format(PyExc_NotImplementedError, + "cannot use in a comparison", + ((CDataObject *)v)->c_type->ct_name); + goto error; + } + aa[i] = w; + Py_DECREF(v); + } + pyres = PyObject_RichCompare(aa[0], aa[1], op); + error: + Py_DECREF(aa[1]); + Py_DECREF(aa[0]); + return pyres; + } + Py_INCREF(pyres); return pyres; - - Error: - PyErr_SetString(PyExc_TypeError, - "cannot do comparison on a primitive cdata"); - return NULL; -} - -static long cdata_hash(CDataObject *cd) -{ - return _Py_HashPointer(cd->c_data); +} + +static long cdata_hash(CDataObject *v) +{ + if (((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY) { + PyObject *vv = convert_to_object(((CDataObject *)v)->c_data, + ((CDataObject *)v)->c_type); + if (vv == NULL) + return -1; + if (!CData_Check(vv)) { + long hash = PyObject_Hash(vv); + Py_DECREF(vv); + return hash; + } + Py_DECREF(vv); + } + return _Py_HashPointer(v->c_data); } static Py_ssize_t @@ -2470,11 +2557,26 @@ return _cdata_add_or_sub(v, w, -1); } +static void +_cdata_attr_errmsg(char *errmsg, CDataObject *cd, PyObject *attr) +{ + char *text; + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + return; + PyErr_Clear(); + text = PyText_AsUTF8(attr); + if (text == NULL) + return; + PyErr_Format(PyExc_AttributeError, errmsg, cd->c_type->ct_name, text); +} + static PyObject * cdata_getattro(CDataObject *cd, PyObject *attr) { CFieldObject *cf; CTypeDescrObject *ct = cd->c_type; + char *errmsg = "cdata '%s' has no attribute '%s'"; + PyObject *x; if (ct->ct_flags & CT_POINTER) ct = ct->ct_itemdescr; @@ -2506,14 +2608,19 @@ return new_simple_cdata(data, (CTypeDescrObject *)cf->cf_type->ct_stuff); } + errmsg = "cdata '%s' has no field '%s'"; break; case -1: return NULL; default: + errmsg = "cdata '%s' points to an opaque type: cannot read fields"; break; } } - return PyObject_GenericGetAttr((PyObject *)cd, attr); + x = PyObject_GenericGetAttr((PyObject *)cd, attr); + if (x == NULL) + _cdata_attr_errmsg(errmsg, cd, attr); + return x; } static int @@ -2521,6 +2628,8 @@ { CFieldObject *cf; CTypeDescrObject *ct = cd->c_type; + char *errmsg = "cdata '%s' has no attribute '%s'"; + int x; if (ct->ct_flags & CT_POINTER) ct = ct->ct_itemdescr; @@ -2540,14 +2649,19 @@ return -1; } } + errmsg = "cdata '%s' has no field '%s'"; break; case -1: return -1; default: + errmsg = "cdata '%s' points to an opaque type: cannot write fields"; break; } } - return PyObject_GenericSetAttr((PyObject *)cd, attr, value); + x = PyObject_GenericSetAttr((PyObject *)cd, attr, value); + if (x < 0) + _cdata_attr_errmsg(errmsg, cd, attr); + return x; } static PyObject * @@ -2597,6 +2711,10 @@ length = PyBytes_GET_SIZE(init) + 1; #else *output_data = PyBytes_AS_STRING(init); + if (ctitem->ct_flags & CT_IS_BOOL) + if (must_be_array_of_zero_or_one(*output_data, + PyBytes_GET_SIZE(init)) < 0) + return -1; return 0; #endif } @@ -3761,19 +3879,14 @@ CTypeDescrObject *ct; char *funcname; void *funcptr; - int ok; if (!PyArg_ParseTuple(args, "O!s:load_function", &CTypeDescr_Type, &ct, &funcname)) return NULL; - ok = 0; - if (ct->ct_flags & CT_FUNCTIONPTR) - ok = 1; - if ((ct->ct_flags & CT_POINTER) && (ct->ct_itemdescr->ct_flags & CT_VOID)) - ok = 1; - if (!ok) { - PyErr_Format(PyExc_TypeError, "function cdata expected, got '%s'", + if (!(ct->ct_flags & (CT_FUNCTIONPTR | CT_POINTER | CT_ARRAY))) { + PyErr_Format(PyExc_TypeError, + "function or pointer or array cdata expected, got '%s'", ct->ct_name); return NULL; } @@ -3781,12 +3894,15 @@ funcptr = dlsym(dlobj->dl_handle, funcname); if (funcptr == NULL) { const char *error = dlerror(); - PyErr_Format(PyExc_KeyError, - "function '%s' not found in library '%s': %s", + PyErr_Format(PyExc_AttributeError, + "function/symbol '%s' not found in library '%s': %s", funcname, dlobj->dl_name, error); return NULL; } + if ((ct->ct_flags & CT_ARRAY) && ct->ct_length < 0) { + ct = (CTypeDescrObject *)ct->ct_stuff; + } return new_simple_cdata(funcptr, ct); } @@ -5890,7 +6006,8 @@ if (cd->c_type->ct_itemdescr != NULL && cd->c_type->ct_itemdescr->ct_flags & (CT_PRIMITIVE_CHAR | CT_PRIMITIVE_SIGNED | - CT_PRIMITIVE_UNSIGNED)) { + CT_PRIMITIVE_UNSIGNED) && + !(cd->c_type->ct_itemdescr->ct_flags & CT_IS_BOOL)) { Py_ssize_t length = maxlen; if (cd->c_data == NULL) { PyObject *s = cdata_repr(cd); @@ -6058,7 +6175,8 @@ /* Note: we never pick case 6 if sizeof(int) == sizeof(long), so that case 6 below can assume that the 'unsigned int' result would always fit in a 'signed long'. */ - if (itemsize == sizeof(unsigned long)) casenum = 7; + if (ctitem->ct_flags & CT_IS_BOOL) casenum = 11; + else if (itemsize == sizeof(unsigned long)) casenum = 7; else if (itemsize == sizeof(unsigned int)) casenum = 6; else if (itemsize == sizeof(unsigned short)) casenum = 5; else if (itemsize == sizeof(unsigned char)) casenum = 4; @@ -6091,6 +6209,13 @@ case 8: x = PyFloat_FromDouble(*(float *)src); break; case 9: x = PyFloat_FromDouble(*(double *)src); break; case 10: x = new_simple_cdata(*(char **)src, ctitem); break; + case 11: + switch (*(unsigned char *)src) { + case 0: x = Py_False; Py_INCREF(x); break; + case 1: x = Py_True; Py_INCREF(x); break; + default: x = convert_to_object(src, ctitem); /* error */ + } + break; } if (x == NULL) { Py_DECREF(result); @@ -6102,8 +6227,10 @@ return result; } -static PyObject *b_buffer(PyObject *self, PyObject *args, PyObject *kwds) -{ +static PyObject * +b_buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + /* this is the constructor of the type implemented in minibuffer.h */ CDataObject *cd; Py_ssize_t size = -1; static char *keywords[] = {"cdata", "size", NULL}; @@ -6738,7 +6865,6 @@ {"getcname", b_getcname, METH_VARARGS}, {"string", (PyCFunction)b_string, METH_VARARGS | METH_KEYWORDS}, {"unpack", (PyCFunction)b_unpack, METH_VARARGS | METH_KEYWORDS}, - {"buffer", (PyCFunction)b_buffer, METH_VARARGS | METH_KEYWORDS}, {"get_errno", b_get_errno, METH_NOARGS}, {"set_errno", b_set_errno, METH_O}, {"newp_handle", b_newp_handle, METH_VARARGS}, @@ -7042,6 +7168,10 @@ INITERROR; } + Py_INCREF(&MiniBuffer_Type); + if (PyModule_AddObject(m, "buffer", (PyObject *)&MiniBuffer_Type) < 0) + INITERROR; + init_cffi_tls(); if (PyErr_Occurred()) INITERROR; diff --git a/c/call_python.c b/c/call_python.c --- a/c/call_python.c +++ b/c/call_python.c @@ -177,7 +177,7 @@ #if (defined(WITH_THREAD) && !defined(_MSC_VER) && \ !defined(__amd64__) && !defined(__x86_64__) && \ !defined(__i386__) && !defined(__i386)) -# if defined(__GNUC__) +# if defined(HAVE_SYNC_SYNCHRONIZE) # define read_barrier() __sync_synchronize() # elif defined(_AIX) # define read_barrier() __lwsync() diff --git a/c/cffi1_module.c b/c/cffi1_module.c --- a/c/cffi1_module.c +++ b/c/cffi1_module.c @@ -45,6 +45,9 @@ if (PyDict_SetItemString(FFI_Type.tp_dict, "CData", (PyObject *)&CData_Type) < 0) return -1; + if (PyDict_SetItemString(FFI_Type.tp_dict, "buffer", + (PyObject *)&MiniBuffer_Type) < 0) + return -1; for (i = 0; all_dlopen_flags[i].name != NULL; i++) { x = PyInt_FromLong(all_dlopen_flags[i].value); diff --git a/c/ffi_obj.c b/c/ffi_obj.c --- a/c/ffi_obj.c +++ b/c/ffi_obj.c @@ -475,19 +475,6 @@ #define ffi_unpack b_unpack /* ffi_unpack() => b_unpack() from _cffi_backend.c */ -PyDoc_STRVAR(ffi_buffer_doc, -"Return a read-write buffer object that references the raw C data\n" -"pointed to by the given 'cdata'. The 'cdata' must be a pointer or an\n" -"array. Can be passed to functions expecting a buffer, or directly\n" -"manipulated with:\n" -"\n" -" buf[:] get a copy of it in a regular string, or\n" -" buf[idx] as a single character\n" -" buf[:] = ...\n" -" buf[idx] = ... change the content"); - -#define ffi_buffer b_buffer /* ffi_buffer() => b_buffer() - from _cffi_backend.c */ PyDoc_STRVAR(ffi_offsetof_doc, "Return the offset of the named field inside the given structure or\n" @@ -1085,7 +1072,6 @@ static PyMethodDef ffi_methods[] = { {"addressof", (PyCFunction)ffi_addressof, METH_VARARGS, ffi_addressof_doc}, {"alignof", (PyCFunction)ffi_alignof, METH_O, ffi_alignof_doc}, - {"buffer", (PyCFunction)ffi_buffer, METH_VKW, ffi_buffer_doc}, {"def_extern", (PyCFunction)ffi_def_extern, METH_VKW, ffi_def_extern_doc}, {"callback", (PyCFunction)ffi_callback, METH_VKW, ffi_callback_doc}, {"cast", (PyCFunction)ffi_cast, METH_VARARGS, ffi_cast_doc}, diff --git a/c/minibuffer.h b/c/minibuffer.h --- a/c/minibuffer.h +++ b/c/minibuffer.h @@ -155,6 +155,81 @@ return 0; } +static PyObject * +mb_richcompare(PyObject *self, PyObject *other, int op) +{ + Py_ssize_t self_size, other_size; + Py_buffer self_bytes, other_bytes; + PyObject *res; + Py_ssize_t minsize; + int cmp, rc; + + /* Bytes can be compared to anything that supports the (binary) + buffer API. Except that a comparison with Unicode is always an + error, even if the comparison is for equality. */ + rc = PyObject_IsInstance(self, (PyObject*)&PyUnicode_Type); + if (!rc) + rc = PyObject_IsInstance(other, (PyObject*)&PyUnicode_Type); + if (rc < 0) + return NULL; + if (rc) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + if (PyObject_GetBuffer(self, &self_bytes, PyBUF_SIMPLE) != 0) { + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + + } + self_size = self_bytes.len; + + if (PyObject_GetBuffer(other, &other_bytes, PyBUF_SIMPLE) != 0) { + PyErr_Clear(); + PyBuffer_Release(&self_bytes); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + + } + other_size = other_bytes.len; + + if (self_size != other_size && (op == Py_EQ || op == Py_NE)) { + /* Shortcut: if the lengths differ, the objects differ */ + cmp = (op == Py_NE); + } + else { + minsize = self_size; + if (other_size < minsize) + minsize = other_size; + + cmp = memcmp(self_bytes.buf, other_bytes.buf, minsize); + /* In ISO C, memcmp() guarantees to use unsigned bytes! */ + + if (cmp == 0) { + if (self_size < other_size) + cmp = -1; + else if (self_size > other_size) + cmp = 1; + } + + switch (op) { + case Py_LT: cmp = cmp < 0; break; + case Py_LE: cmp = cmp <= 0; break; + case Py_EQ: cmp = cmp == 0; break; + case Py_NE: cmp = cmp != 0; break; + case Py_GT: cmp = cmp > 0; break; + case Py_GE: cmp = cmp >= 0; break; + } + } + + res = cmp ? Py_True : Py_False; + PyBuffer_Release(&self_bytes); + PyBuffer_Release(&other_bytes); + Py_INCREF(res); + return res; +} + #if PY_MAJOR_VERSION >= 3 /* pfffffffffffff pages of copy-paste from listobject.c */ static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item) @@ -238,6 +313,22 @@ # define MINIBUF_TPFLAGS (Py_TPFLAGS_HAVE_GETCHARBUFFER | Py_TPFLAGS_HAVE_NEWBUFFER) #endif +PyDoc_STRVAR(ffi_buffer_doc, +"ffi.buffer(cdata[, byte_size]):\n" +"Return a read-write buffer object that references the raw C data\n" +"pointed to by the given 'cdata'. The 'cdata' must be a pointer or an\n" +"array. Can be passed to functions expecting a buffer, or directly\n" +"manipulated with:\n" +"\n" +" buf[:] get a copy of it in a regular string, or\n" +" buf[idx] as a single character\n" +" buf[:] = ...\n" +" buf[idx] = ... change the content"); + +static PyObject * /* forward, implemented in _cffi_backend.c */ +b_buffer_new(PyTypeObject *type, PyObject *args, PyObject *kwds); + + static PyTypeObject MiniBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_cffi_backend.buffer", @@ -268,11 +359,25 @@ &mb_as_buffer, /* tp_as_buffer */ (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | MINIBUF_TPFLAGS), /* tp_flags */ - 0, /* tp_doc */ + ffi_buffer_doc, /* tp_doc */ (traverseproc)mb_traverse, /* tp_traverse */ (inquiry)mb_clear, /* tp_clear */ - 0, /* tp_richcompare */ + (richcmpfunc)mb_richcompare, /* tp_richcompare */ offsetof(MiniBufferObj, mb_weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + b_buffer_new, /* tp_new */ + 0, /* tp_free */ }; static PyObject *minibuffer_new(char *data, Py_ssize_t size, diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -12,9 +12,9 @@ # ____________________________________________________________ import sys -assert __version__ == "1.9.2", ("This test_c.py file is for testing a version" - " of cffi that differs from the one that we" - " get from 'import _cffi_backend'") +assert __version__ == "1.10.0", ("This test_c.py file is for testing a version" + " of cffi that differs from the one that we" + " get from 'import _cffi_backend'") if sys.version_info < (3,): type_or_class = "type" mandatory_b_prefix = '' @@ -27,6 +27,7 @@ .replace(r'\\U', r'\U')) u = U() str2bytes = str + strict_compare = False else: type_or_class = "class" long = int @@ -38,6 +39,7 @@ bitem2bchr = bytechr u = "" str2bytes = lambda s: bytes(s, "ascii") + strict_compare = True def size_of_int(): BInt = new_primitive_type("int") @@ -106,11 +108,11 @@ x = cast(p, -66 + (1<<199)*256) assert repr(x) == "" assert int(x) == -66 - assert (x == cast(p, -66)) is False - assert (x != cast(p, -66)) is True + assert (x == cast(p, -66)) is True + assert (x != cast(p, -66)) is False q = new_primitive_type("short") - assert (x == cast(q, -66)) is False - assert (x != cast(q, -66)) is True + assert (x == cast(q, -66)) is True + assert (x != cast(q, -66)) is False def test_sizeof_type(): py.test.raises(TypeError, sizeof, 42.5) @@ -175,7 +177,7 @@ assert float(cast(p, 1.1)) != 1.1 # rounding error assert float(cast(p, 1E200)) == INF # limited range - assert cast(p, -1.1) != cast(p, -1.1) + assert cast(p, -1.1) == cast(p, -1.1) assert repr(float(cast(p, -0.0))) == '-0.0' assert float(cast(p, b'\x09')) == 9.0 assert float(cast(p, u+'\x09')) == 9.0 @@ -219,7 +221,7 @@ p = new_primitive_type("char") assert bool(cast(p, 'A')) is True assert bool(cast(p, '\x00')) is False # since 1.7 - assert cast(p, '\x00') != cast(p, -17*256) + assert cast(p, '\x00') == cast(p, -17*256) assert int(cast(p, 'A')) == 65 assert long(cast(p, 'A')) == 65 assert type(int(cast(p, 'A'))) is int @@ -376,29 +378,12 @@ x = find_and_load_library(None) BVoidP = new_pointer_type(new_void_type()) assert x.load_function(BVoidP, 'strcpy') - py.test.raises(KeyError, x.load_function, + py.test.raises(AttributeError, x.load_function, BVoidP, 'xxx_this_function_does_not_exist') # the next one is from 'libm', not 'libc', but we assume # that it is already loaded too, so it should work assert x.load_function(BVoidP, 'sqrt') -def test_hash_differences(): - BChar = new_primitive_type("char") - BInt = new_primitive_type("int") - BFloat = new_primitive_type("float") - for i in range(1, 20): - x1 = cast(BChar, chr(i)) - x2 = cast(BInt, i) - if hash(x1) != hash(x2): - break - else: - raise AssertionError("hashes are equal") - for i in range(1, 20): - if hash(cast(BFloat, i)) != hash(float(i)): - break - else: - raise AssertionError("hashes are equal") - def test_no_len_on_nonarray(): p = new_primitive_type("int") py.test.raises(TypeError, len, cast(p, 42)) @@ -748,8 +733,14 @@ BInt = new_primitive_type("int") BStruct = new_struct_type("struct foo") BStructPtr = new_pointer_type(BStruct) - p = cast(BStructPtr, 0) - py.test.raises(AttributeError, "p.a1") # opaque + p = cast(BStructPtr, 42) + e = py.test.raises(AttributeError, "p.a1") # opaque + assert str(e.value) == ("cdata 'struct foo *' points to an opaque type: " + "cannot read fields") + e = py.test.raises(AttributeError, "p.a1 = 10") # opaque + assert str(e.value) == ("cdata 'struct foo *' points to an opaque type: " + "cannot write fields") + complete_struct_or_union(BStruct, [('a1', BInt, -1), ('a2', BInt, -1)]) p = newp(BStructPtr, None) @@ -760,8 +751,29 @@ assert s.a2 == 123 py.test.raises(OverflowError, "s.a1 = sys.maxsize+1") assert s.a1 == 0 - py.test.raises(AttributeError, "p.foobar") - py.test.raises(AttributeError, "s.foobar") + e = py.test.raises(AttributeError, "p.foobar") + assert str(e.value) == "cdata 'struct foo *' has no field 'foobar'" + e = py.test.raises(AttributeError, "p.foobar = 42") + assert str(e.value) == "cdata 'struct foo *' has no field 'foobar'" + e = py.test.raises(AttributeError, "s.foobar") + assert str(e.value) == "cdata 'struct foo' has no field 'foobar'" + e = py.test.raises(AttributeError, "s.foobar = 42") + assert str(e.value) == "cdata 'struct foo' has no field 'foobar'" + j = cast(BInt, 42) + e = py.test.raises(AttributeError, "j.foobar") + assert str(e.value) == "cdata 'int' has no attribute 'foobar'" + e = py.test.raises(AttributeError, "j.foobar = 42") + assert str(e.value) == "cdata 'int' has no attribute 'foobar'" + j = cast(new_pointer_type(BInt), 42) + e = py.test.raises(AttributeError, "j.foobar") + assert str(e.value) == "cdata 'int *' has no attribute 'foobar'" + e = py.test.raises(AttributeError, "j.foobar = 42") + assert str(e.value) == "cdata 'int *' has no attribute 'foobar'" + pp = newp(new_pointer_type(BStructPtr), p) + e = py.test.raises(AttributeError, "pp.a1") + assert str(e.value) == "cdata 'struct foo * *' has no attribute 'a1'" + e = py.test.raises(AttributeError, "pp.a1 = 42") + assert str(e.value) == "cdata 'struct foo * *' has no attribute 'a1'" def test_union_instance(): BInt = new_primitive_type("int") @@ -896,6 +908,15 @@ py.test.raises(OverflowError, f, 128, 0) py.test.raises(OverflowError, f, 0, 128) +def test_call_function_0_pretend_bool_result(): + BSignedChar = new_primitive_type("signed char") + BBool = new_primitive_type("_Bool") + BFunc0 = new_function_type((BSignedChar, BSignedChar), BBool, False) + f = cast(BFunc0, _testfunc(0)) + assert f(40, -39) is True + assert f(40, -40) is False + py.test.raises(ValueError, f, 40, 2) + def test_call_function_1(): BInt = new_primitive_type("int") BLong = new_primitive_type("long") @@ -1058,6 +1079,17 @@ res = f(b"foo") assert res == 1000 * ord(b'f') +def test_call_function_23_bool_array(): + # declaring the function as int(_Bool*) + BBool = new_primitive_type("_Bool") + BBoolP = new_pointer_type(BBool) + BInt = new_primitive_type("int") + BFunc23 = new_function_type((BBoolP,), BInt, False) + f = cast(BFunc23, _testfunc(23)) + res = f(b"\x01\x01") + assert res == 1000 + py.test.raises(ValueError, f, b"\x02\x02") + def test_cannot_pass_struct_with_array_of_length_0(): BInt = new_primitive_type("int") BArray0 = new_array_type(new_pointer_type(BInt), 0) @@ -2237,12 +2269,17 @@ BVoidP = new_pointer_type(new_void_type()) p = newp(BIntP, 123) q = cast(BInt, 124) - py.test.raises(TypeError, "p < q") - py.test.raises(TypeError, "p <= q") assert (p == q) is False assert (p != q) is True - py.test.raises(TypeError, "p > q") - py.test.raises(TypeError, "p >= q") + assert (q == p) is False + assert (q != p) is True + if strict_compare: + py.test.raises(TypeError, "p < q") + py.test.raises(TypeError, "p <= q") + py.test.raises(TypeError, "q < p") + py.test.raises(TypeError, "q <= p") + py.test.raises(TypeError, "p > q") + py.test.raises(TypeError, "p >= q") r = cast(BVoidP, p) assert (p < r) is False assert (p <= r) is True @@ -2275,6 +2312,7 @@ buf = buffer(c) assert repr(buf).startswith('<_cffi_backend.buffer object at 0x') assert bytes(buf) == b"hi there\x00" + assert type(buf) is buffer if sys.version_info < (3,): assert str(buf) == "hi there\x00" assert unicode(buf) == u+"hi there\x00" @@ -2651,13 +2689,38 @@ py.test.raises(OverflowError, newp, BBoolP, 2) py.test.raises(OverflowError, newp, BBoolP, -1) BCharP = new_pointer_type(new_primitive_type("char")) - p = newp(BCharP, b'X') + p = newp(BCharP, b'\x01') q = cast(BBoolP, p) - assert q[0] == ord(b'X') + assert q[0] is True + p = newp(BCharP, b'\x00') + q = cast(BBoolP, p) + assert q[0] is False py.test.raises(TypeError, string, cast(BBool, False)) BDouble = new_primitive_type("double") assert int(cast(BBool, cast(BDouble, 0.1))) == 1 assert int(cast(BBool, cast(BDouble, 0.0))) == 0 + BBoolA = new_array_type(BBoolP, None) + p = newp(BBoolA, b'\x01\x00') + assert p[0] is True + assert p[1] is False + +def test_bool_forbidden_cases(): + BBool = new_primitive_type("_Bool") + BBoolP = new_pointer_type(BBool) + BBoolA = new_array_type(BBoolP, None) + BCharP = new_pointer_type(new_primitive_type("char")) + p = newp(BCharP, b'X') + q = cast(BBoolP, p) + py.test.raises(ValueError, "q[0]") + py.test.raises(TypeError, newp, BBoolP, b'\x00') + assert newp(BBoolP, 0)[0] is False + assert newp(BBoolP, 1)[0] is True + py.test.raises(OverflowError, newp, BBoolP, 2) + py.test.raises(OverflowError, newp, BBoolP, -1) + py.test.raises(ValueError, newp, BBoolA, b'\x00\x01\x02') + py.test.raises(OverflowError, newp, BBoolA, [0, 1, 2]) + py.test.raises(TypeError, string, newp(BBoolP, 1)) + py.test.raises(TypeError, string, newp(BBoolA, [1])) def test_typeoffsetof(): BChar = new_primitive_type("char") @@ -3697,7 +3760,7 @@ ("int16_t", [-2**15, 2**15-1]), ("int32_t", [-2**31, 2**31-1]), ("int64_t", [-2**63, 2**63-1]), - ("_Bool", [0, 1]), + ("_Bool", [False, True]), ("float", [0.0, 10.5]), ("double", [12.34, 56.78]), ]: @@ -3767,7 +3830,7 @@ def test_char_pointer_conversion(): import warnings - assert __version__.startswith(("1.8", "1.9")), ( + assert __version__.startswith(("1.8", "1.9", "1.10")), ( "consider turning the warning into an error") BCharP = new_pointer_type(new_primitive_type("char")) BIntP = new_pointer_type(new_primitive_type("int")) @@ -3790,3 +3853,87 @@ assert len(w) == 2 # check that the warnings are associated with lines in this file assert w[1].lineno == w[0].lineno + 4 + +def test_primitive_comparison(): + def assert_eq(a, b): + assert (a == b) is True + assert (b == a) is True + assert (a != b) is False + assert (b != a) is False + assert (a < b) is False + assert (a <= b) is True + assert (a > b) is False + assert (a >= b) is True + assert (b < a) is False + assert (b <= a) is True + assert (b > a) is False + assert (b >= a) is True + assert hash(a) == hash(b) + def assert_lt(a, b, check_hash=True): + assert (a == b) is False + assert (b == a) is False + assert (a != b) is True + assert (b != a) is True + assert (a < b) is True + assert (a <= b) is True + assert (a > b) is False + assert (a >= b) is False + assert (b < a) is False + assert (b <= a) is False + assert (b > a) is True + assert (b >= a) is True + if check_hash: + assert hash(a) != hash(b) # (or at least, it is unlikely) + def assert_gt(a, b, check_hash=True): + assert_lt(b, a, check_hash) + def assert_ne(a, b): + assert (a == b) is False + assert (b == a) is False + assert (a != b) is True + assert (b != a) is True + if strict_compare: + py.test.raises(TypeError, "a < b") + py.test.raises(TypeError, "a <= b") + py.test.raises(TypeError, "a > b") + py.test.raises(TypeError, "a >= b") + py.test.raises(TypeError, "b < a") + py.test.raises(TypeError, "b <= a") + py.test.raises(TypeError, "b > a") + py.test.raises(TypeError, "b >= a") + elif a < b: + assert_lt(a, b) + else: + assert_lt(b, a) + assert_eq(5, 5) + assert_lt(3, 5) + assert_ne('5', 5) + # + t1 = new_primitive_type("char") + t2 = new_primitive_type("int") + t3 = new_primitive_type("unsigned char") + t4 = new_primitive_type("unsigned int") + t5 = new_primitive_type("float") + t6 = new_primitive_type("double") + assert_eq(cast(t1, 65), b'A') + assert_lt(cast(t1, 64), b'\x99') + assert_gt(cast(t1, 200), b'A') + assert_ne(cast(t1, 65), 65) + assert_eq(cast(t2, -25), -25) + assert_lt(cast(t2, -25), -24) + assert_gt(cast(t2, -25), -26) + assert_eq(cast(t3, 65), 65) + assert_ne(cast(t3, 65), b'A') + assert_ne(cast(t3, 65), cast(t1, 65)) + assert_gt(cast(t4, -1), -1, check_hash=False) + assert_gt(cast(t4, -1), cast(t2, -1), check_hash=False) + assert_gt(cast(t4, -1), 99999) + assert_eq(cast(t4, -1), 256 ** size_of_int() - 1) + assert_eq(cast(t5, 3.0), 3) + assert_eq(cast(t5, 3.5), 3.5) + assert_lt(cast(t5, 3.3), 3.3) # imperfect rounding + assert_eq(cast(t6, 3.3), 3.3) + assert_eq(cast(t5, 3.5), cast(t6, 3.5)) + assert_lt(cast(t5, 3.1), cast(t6, 3.1)) # imperfect rounding + assert_eq(cast(t5, 7.0), cast(t3, 7)) + assert_lt(cast(t5, 3.1), 3.101) + assert_gt(cast(t5, 3.1), 3) diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -1,11 +1,11 @@ __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError', 'FFIError'] -from .api import FFI, CDefError, FFIError -from .ffiplatform import VerificationError, VerificationMissing +from .api import FFI +from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.9.2" -__version_info__ = (1, 9, 2) +__version__ = "1.10.0" +__version_info__ = (1, 10, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.9.2" + "\ncompiled with cffi version: 1.10.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -1,5 +1,7 @@ import sys, types from .lock import allocate_lock +from .error import CDefError +from . import model try: callable @@ -15,17 +17,6 @@ basestring = str -class FFIError(Exception): - pass - -class CDefError(Exception): - def __str__(self): - try: - line = 'line %d: ' % (self.args[1].coord.line,) - except (AttributeError, TypeError, IndexError): - line = '' - return '%s%s' % (line, self.args[0]) - class FFI(object): r''' @@ -49,7 +40,6 @@ """Create an FFI instance. The 'backend' argument is used to select a non-default backend, mostly for tests. """ - from . import cparser, model if backend is None: # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with # _cffi_backend.so compiled. @@ -70,6 +60,7 @@ # 'backend=backend_ctypes.CTypesBackend()', but don't # rely on it! It's probably not going to work well.) + from . import cparser self._backend = backend self._lock = allocate_lock() self._parser = cparser.Parser() @@ -102,6 +93,7 @@ # ctypes backend: attach these constants to the instance self.NULL = self.cast(self.BVoidP, 0) self.CData, self.CType = backend._get_types() + self.buffer = backend.buffer def cdef(self, csource, override=False, packed=False): """Parse the given C source. This registers all declared functions, @@ -221,7 +213,7 @@ def offsetof(self, cdecl, *fields_or_indexes): """Return the offset of the named field inside the given - structure or array, which must be given as a C type name. + structure or array, which must be given as a C type name. You can give several field names in case of nested structures. You can also give numeric values which correspond to array items, in case of an array type. @@ -309,7 +301,7 @@ return self._backend.string(cdata, maxlen) def unpack(self, cdata, length): - """Unpack an array of C data of the given length, + """Unpack an array of C data of the given length, returning a Python string/unicode/list. If 'cdata' is a pointer to 'char', returns a byte string. @@ -325,18 +317,18 @@ """ return self._backend.unpack(cdata, length) - def buffer(self, cdata, size=-1): - """Return a read-write buffer object that references the raw C data - pointed to by the given 'cdata'. The 'cdata' must be a pointer or - an array. Can be passed to functions expecting a buffer, or directly - manipulated with: - - buf[:] get a copy of it in a regular string, or - buf[idx] as a single character - buf[:] = ... - buf[idx] = ... change the content - """ - return self._backend.buffer(cdata, size) + #def buffer(self, cdata, size=-1): + # """Return a read-write buffer object that references the raw C data + # pointed to by the given 'cdata'. The 'cdata' must be a pointer or + # an array. Can be passed to functions expecting a buffer, or directly + # manipulated with: + # + # buf[:] get a copy of it in a regular string, or + # buf[idx] as a single character + # buf[:] = ... + # buf[idx] = ... change the content + # """ + # note that 'buffer' is a type, set on this instance by __init__ def from_buffer(self, python_buffer): """Return a that points to the data of the @@ -461,7 +453,6 @@ return self._backend.getwinerror(code) def _pointer_to(self, ctype): - from . import model with self._lock: return model.pointer_cache(self, ctype) @@ -471,7 +462,12 @@ field or array item in the structure or array, recursively in case of nested structures. """ - ctype = self._backend.typeof(cdata) + try: + ctype = self._backend.typeof(cdata) + except TypeError: + if '__addressof__' in type(cdata).__dict__: + return type(cdata).__addressof__(cdata, *fields_or_indexes) + raise if fields_or_indexes: ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes) else: @@ -574,7 +570,10 @@ # we need 'libpypy-c.{so,dylib}', which should be by # default located in 'sys.prefix/bin' for installed # systems. - pythonlib = "pypy-c" + if sys.version_info < (3,): + pythonlib = "pypy-c" + else: + pythonlib = "pypy3-c" if hasattr(sys, 'prefix'): ensure('library_dirs', os.path.join(sys.prefix, 'bin')) # On uninstalled pypy's, the libpypy-c is typically found in @@ -603,11 +602,15 @@ ensure('extra_link_args', '/MANIFEST') def set_source(self, module_name, source, source_extension='.c', **kwds): + import os if hasattr(self, '_assigned_source'): raise ValueError("set_source() cannot be called several times " "per ffi object") if not isinstance(module_name, basestring): raise TypeError("'module_name' must be a string") + if os.sep in module_name or (os.altsep and os.altsep in module_name): + raise ValueError("'module_name' must not contain '/': use a dotted " + "name to make a 'package.module' location") self._assigned_source = (str(module_name), source, source_extension, kwds) @@ -756,24 +759,29 @@ def _load_backend_lib(backend, name, flags): + import os if name is None: if sys.platform != "win32": return backend.load_library(None, flags) name = "c" # Windows: load_library(None) fails, but this works # (backward compatibility hack only) - try: - if '.' not in name and '/' not in name: - raise OSError("library not found: %r" % (name,)) - return backend.load_library(name, flags) - except OSError: - import ctypes.util - path = ctypes.util.find_library(name) - if path is None: - raise # propagate the original OSError - return backend.load_library(path, flags) + first_error = None + if '.' in name or '/' in name or os.sep in name: + try: + return backend.load_library(name, flags) + except OSError as e: + first_error = e + import ctypes.util + path = ctypes.util.find_library(name) + if path is None: + msg = ("ctypes.util.find_library() did not manage " + "to locate a library called %r" % (name,)) + if first_error is not None: + msg = "%s. Additionally, %s" % (first_error, msg) + raise OSError(msg) + return backend.load_library(path, flags) def _make_ffi_library(ffi, libname, flags): - import os backend = ffi._backend backendlib = _load_backend_lib(backend, libname, flags) # @@ -781,10 +789,7 @@ key = 'function ' + name tp, _ = ffi._parser._declarations[key] BType = ffi._get_cached_btype(tp) - try: - value = backendlib.load_function(BType, name) - except KeyError as e: - raise AttributeError('%s: %s' % (name, e)) + value = backendlib.load_function(BType, name) library.__dict__[name] = value # def accessor_variable(name): @@ -797,6 +802,21 @@ lambda self: read_variable(BType, name), lambda self, value: write_variable(BType, name, value))) # + def addressof_var(name): + try: + return addr_variables[name] + except KeyError: + with ffi._lock: + if name not in addr_variables: + key = 'variable ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + if BType.kind != 'array': + BType = model.pointer_cache(ffi, BType) + p = backendlib.load_function(BType, name) + addr_variables[name] = p + return addr_variables[name] + # def accessor_constant(name): raise NotImplementedError("non-integer constant '%s' cannot be " "accessed from a dlopen() library" % (name,)) @@ -806,12 +826,12 @@ # accessors = {} accessors_version = [False] + addr_variables = {} # def update_accessors(): if accessors_version[0] is ffi._cdef_version: return # - from . import model for key, (tp, _) in ffi._parser._declarations.items(): if not isinstance(tp, model.EnumType): tag, name = key.split(' ', 1) @@ -857,6 +877,18 @@ with ffi._lock: update_accessors() return accessors.keys() + def __addressof__(self, name): + if name in library.__dict__: + return library.__dict__[name] + if name in FFILibrary.__dict__: + return addressof_var(name) + make_accessor(name) + if name in library.__dict__: + return library.__dict__[name] + if name in FFILibrary.__dict__: + return addressof_var(name) + raise AttributeError("cffi library has no function or " + "global variable named '%s'" % (name,)) # if libname is not None: try: diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py --- a/cffi/backend_ctypes.py +++ b/cffi/backend_ctypes.py @@ -112,11 +112,20 @@ def _make_cmp(name): cmpfunc = getattr(operator, name) def cmp(self, other): - if isinstance(other, CTypesData): + v_is_ptr = not isinstance(self, CTypesGenericPrimitive) + w_is_ptr = (isinstance(other, CTypesData) and + not isinstance(other, CTypesGenericPrimitive)) + if v_is_ptr and w_is_ptr: return cmpfunc(self._convert_to_address(None), other._convert_to_address(None)) + elif v_is_ptr or w_is_ptr: + return NotImplemented else: - return NotImplemented + if isinstance(self, CTypesGenericPrimitive): + self = self._value + if isinstance(other, CTypesGenericPrimitive): + other = other._value + return cmpfunc(self, other) cmp.func_name = name return cmp @@ -128,7 +137,7 @@ __ge__ = _make_cmp('__ge__') def __hash__(self): - return hash(type(self)) ^ hash(self._convert_to_address(None)) + return hash(self._convert_to_address(None)) def _to_string(self, maxlen): raise TypeError("string(): %r" % (self,)) @@ -137,14 +146,8 @@ class CTypesGenericPrimitive(CTypesData): __slots__ = [] - def __eq__(self, other): - return self is other - - def __ne__(self, other): - return self is not other - def __hash__(self): - return object.__hash__(self) + return hash(self._value) def _get_own_repr(self): return repr(self._from_ctypes(self._value)) diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py --- a/cffi/cffi_opcode.py +++ b/cffi/cffi_opcode.py @@ -1,3 +1,4 @@ +from .error import VerificationError class CffiOp(object): def __init__(self, op, arg): @@ -19,7 +20,6 @@ % (self.arg,)) return format_four_bytes(value) if isinstance(self.arg, str): - from .ffiplatform import VerificationError raise VerificationError("cannot emit to Python: %r" % (self.arg,)) return format_four_bytes((self.arg << 8) | self.op) diff --git a/cffi/commontypes.py b/cffi/commontypes.py --- a/cffi/commontypes.py +++ b/cffi/commontypes.py @@ -1,5 +1,6 @@ import sys -from . import api, model +from . import model +from .error import FFIError COMMON_TYPES = {} @@ -31,11 +32,11 @@ elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES: result, quals = model.PrimitiveType(cdecl), 0 elif cdecl == 'set-unicode-needed': - raise api.FFIError("The Windows type %r is only available after " - "you call ffi.set_unicode()" % (commontype,)) + raise FFIError("The Windows type %r is only available after " + "you call ffi.set_unicode()" % (commontype,)) else: if commontype == cdecl: - raise api.FFIError( + raise FFIError( "Unsupported type: %r. Please look at " "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations " "and file an issue if you think this type should really " diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -1,5 +1,6 @@ -from . import api, model +from . import model from .commontypes import COMMON_TYPES, resolve_common_type +from .error import FFIError, CDefError try: from . import _pycparser as pycparser except ImportError: @@ -33,6 +34,9 @@ r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') _r_star_const_space = re.compile( # matches "* const " r"[*]\s*((const|volatile|restrict)\b\s*)+") +_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+" + r"\.\.\.") +_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.") def _get_parser(): global _parser_cache @@ -113,7 +117,7 @@ # grouping variant closing = csource.find('}', endpos) if closing < 0: - raise api.CDefError("'extern \"Python\" {': no '}' found") + raise CDefError("'extern \"Python\" {': no '}' found") if csource.find('{', endpos + 1, closing) >= 0: raise NotImplementedError("cannot use { } inside a block " "'extern \"Python\" { ... }'") @@ -123,7 +127,7 @@ # non-grouping variant semicolon = csource.find(';', endpos) if semicolon < 0: - raise api.CDefError("'extern \"Python\": no ';' found") + raise CDefError("'extern \"Python\": no ';' found") parts.append(csource[endpos:semicolon+1]) csource = csource[semicolon+1:] parts.append(' void __cffi_extern_python_stop;') @@ -179,6 +183,10 @@ assert csource[p:p+3] == '...' csource = '%s __dotdotdot%d__ %s' % (csource[:p], number, csource[p+3:]) + # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__" + csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource) + # Replace "float ..." or "double..." with "__dotdotdotfloat__" + csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) # Replace all remaining "..." with the same name, "__dotdotdot__", # which is declared with a typedef for the purpose of C parsing. return csource.replace('...', ' __dotdotdot__ '), macros @@ -251,7 +259,8 @@ typenames += sorted(ctn) # csourcelines = ['typedef int %s;' % typename for typename in typenames] - csourcelines.append('typedef int __dotdotdot__;') + csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,' + ' __dotdotdot__;') csourcelines.append(csource) csource = '\n'.join(csourcelines) if lock is not None: @@ -288,7 +297,7 @@ msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: msg = 'parse error\n%s' % (msg,) - raise api.CDefError(msg) + raise CDefError(msg) def parse(self, csource, override=False, packed=False, dllexport=False): prev_options = self._options @@ -310,6 +319,8 @@ for decl in iterator: if decl.name == '__dotdotdot__': break + else: + assert 0 # try: self._inside_extern_python = '__cffi_extern_python_stop' @@ -318,18 +329,18 @@ self._parse_decl(decl) elif isinstance(decl, pycparser.c_ast.Typedef): if not decl.name: - raise api.CDefError("typedef does not declare any name", - decl) + raise CDefError("typedef does not declare any name", + decl) quals = 0 - if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) - and decl.type.type.names[-1] == '__dotdotdot__'): + if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and + decl.type.type.names[-1].startswith('__dotdotdot')): realtype = self._get_unknown_type(decl) elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and isinstance(decl.type.type.type, pycparser.c_ast.IdentifierType) and - decl.type.type.type.names == ['__dotdotdot__']): - realtype = model.unknown_ptr_type(decl.name) + decl.type.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_ptr_type(decl) else: realtype, quals = self._get_type_and_quals( decl.type, name=decl.name, partial_length_ok=True) @@ -337,8 +348,8 @@ elif decl.__class__.__name__ == 'Pragma': pass # skip pragma, only in pycparser 2.15 else: - raise api.CDefError("unrecognized construct", decl) - except api.FFIError as e: + raise CDefError("unrecognized construct", decl) + except FFIError as e: msg = self._convert_pycparser_error(e, csource) if msg: e.args = (e.args[0] + "\n *** Err: %s" % msg,) @@ -348,7 +359,7 @@ if key in self._int_constants: if self._int_constants[key] == val: return # ignore identical double declarations - raise api.FFIError( + raise FFIError( "multiple declarations of constant: %s" % (key,)) self._int_constants[key] = val @@ -375,7 +386,7 @@ elif value == '...': self._declare('macro ' + key, value) else: - raise api.CDefError( + raise CDefError( 'only supports one of the following syntax:\n' ' #define %s ... (literally dot-dot-dot)\n' ' #define %s NUMBER (with NUMBER an integer' @@ -410,8 +421,8 @@ elif isinstance(node, pycparser.c_ast.Enum): self._get_struct_union_enum_type('enum', node) elif not decl.name: - raise api.CDefError("construct does not declare any variable", - decl) + raise CDefError("construct does not declare any variable", + decl) # if decl.name: tp, quals = self._get_type_and_quals(node, @@ -438,7 +449,7 @@ self._inside_extern_python = decl.name else: if self._inside_extern_python !='__cffi_extern_python_stop': - raise api.CDefError( + raise CDefError( "cannot declare constants or " "variables with 'extern \"Python\"'") if (quals & model.Q_CONST) and not tp.is_array_type: @@ -454,7 +465,7 @@ assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): - raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) + raise CDefError("unknown identifier '%s'" % (exprnode.name,)) return self._get_type_and_quals(exprnode.type) def _declare(self, name, obj, included=False, quals=0): @@ -463,7 +474,7 @@ if prevobj is obj and prevquals == quals: return if not self._options.get('override'): - raise api.FFIError( + raise FFIError( "multiple declarations of %s (for interactive usage, " "try cdef(xx, override=True))" % (name,)) assert '__dotdotdot__' not in name.split() @@ -551,7 +562,7 @@ if ident == 'void': return model.void_type, quals if ident == '__dotdotdot__': - raise api.FFIError(':%d: bad usage of "..."' % + raise FFIError(':%d: bad usage of "..."' % typenode.coord.line) tp0, quals0 = resolve_common_type(self, ident) return tp0, (quals | quals0) @@ -583,14 +594,14 @@ return self._get_struct_union_enum_type('union', typenode, name, nested=True), 0 # - raise api.FFIError(":%d: bad or unsupported type declaration" % + raise FFIError(":%d: bad or unsupported type declaration" % typenode.coord.line) def _parse_function_type(self, typenode, funcname=None): params = list(getattr(typenode.args, 'params', [])) for i, arg in enumerate(params): if not hasattr(arg, 'type'): - raise api.CDefError("%s arg %d: unknown type '%s'" + raise CDefError("%s arg %d: unknown type '%s'" " (if you meant to use the old C syntax of giving" " untyped arguments, it is not supported)" % (funcname or 'in expression', i + 1, @@ -604,7 +615,7 @@ if ellipsis: params.pop() if not params: - raise api.CDefError( + raise CDefError( "%s: a function with only '(...)' as argument" " is not correct C" % (funcname or 'in expression')) args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type)) @@ -705,7 +716,7 @@ return tp # if tp.fldnames is not None: - raise api.CDefError("duplicate declaration of struct %s" % name) + raise CDefError("duplicate declaration of struct %s" % name) fldnames = [] fldtypes = [] fldbitsize = [] @@ -749,7 +760,7 @@ def _make_partial(self, tp, nested): if not isinstance(tp, model.StructOrUnion): - raise api.CDefError("%s cannot be partial" % (tp,)) + raise CDefError("%s cannot be partial" % (tp,)) if not tp.has_c_name() and not nested: raise NotImplementedError("%s is partial but has no C name" %(tp,)) tp.partial = True @@ -769,7 +780,7 @@ len(s) == 3 or (len(s) == 4 and s[1] == "\\")): return ord(s[-2]) else: - raise api.CDefError("invalid constant %r" % (s,)) + raise CDefError("invalid constant %r" % (s,)) # if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and exprnode.op == '+'): @@ -788,12 +799,12 @@ if partial_length_ok: self._partial_length = True return '...' - raise api.FFIError(":%d: unsupported '[...]' here, cannot derive " - "the actual array length in this context" - % exprnode.coord.line) + raise FFIError(":%d: unsupported '[...]' here, cannot derive " + "the actual array length in this context" + % exprnode.coord.line) # - raise api.FFIError(":%d: unsupported expression: expected a " - "simple numeric constant" % exprnode.coord.line) + raise FFIError(":%d: unsupported expression: expected a " + "simple numeric constant" % exprnode.coord.line) def _build_enum_type(self, explicit_name, decls): if decls is not None: @@ -831,24 +842,25 @@ def _get_unknown_type(self, decl): typenames = decl.type.type.names - assert typenames[-1] == '__dotdotdot__' - if len(typenames) == 1: + if typenames == ['__dotdotdot__']: return model.unknown_type(decl.name) - if (typenames[:-1] == ['float'] or - typenames[:-1] == ['double']): - # not for 'long double' so far - result = model.UnknownFloatType(decl.name) - else: - for t in typenames[:-1]: - if t not in ['int', 'short', 'long', 'signed', - 'unsigned', 'char']: - raise api.FFIError(':%d: bad usage of "..."' % - decl.coord.line) - result = model.UnknownIntegerType(decl.name) + if typenames == ['__dotdotdotint__']: + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef int... %s'" % decl.name + return model.UnknownIntegerType(decl.name) - if self._uses_new_feature is None: - self._uses_new_feature = "'typedef %s... %s'" % ( - ' '.join(typenames[:-1]), decl.name) + if typenames == ['__dotdotdotfloat__']: + # note: not for 'long double' so far + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef float... %s'" % decl.name + return model.UnknownFloatType(decl.name) - return result + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) + + def _get_unknown_ptr_type(self, decl): + if decl.type.type.type.names == ['__dotdotdot__']: + return model.unknown_ptr_type(decl.name) + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) diff --git a/cffi/error.py b/cffi/error.py new file mode 100644 --- /dev/null +++ b/cffi/error.py @@ -0,0 +1,20 @@ + +class FFIError(Exception): + pass + +class CDefError(Exception): + def __str__(self): + try: + line = 'line %d: ' % (self.args[1].coord.line,) + except (AttributeError, TypeError, IndexError): + line = '' + return '%s%s' % (line, self.args[0]) + +class VerificationError(Exception): + """ An error raised when verification fails + """ + +class VerificationMissing(Exception): + """ An error raised when incomplete structures are passed into + cdef, but no verification has been done + """ diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py --- a/cffi/ffiplatform.py +++ b/cffi/ffiplatform.py @@ -1,14 +1,5 @@ import sys, os - - -class VerificationError(Exception): - """ An error raised when verification fails - """ - -class VerificationMissing(Exception): - """ An error raised when incomplete structures are passed into - cdef, but no verification has been done - """ +from .error import VerificationError LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', diff --git a/cffi/model.py b/cffi/model.py --- a/cffi/model.py +++ b/cffi/model.py @@ -1,8 +1,8 @@ -import types, sys +import types import weakref from .lock import allocate_lock - +from .error import CDefError, VerificationError, VerificationMissing # type qualifiers Q_CONST = 0x01 @@ -39,7 +39,6 @@ replace_with = qualify(quals, replace_with) result = result.replace('&', replace_with) if '$' in result: - from .ffiplatform import VerificationError raise VerificationError( "cannot generate '%s' in %s: unknown type name" % (self._get_c_name(), context)) @@ -225,9 +224,8 @@ is_raw_function = True def build_backend_type(self, ffi, finishlist): - from . import api - raise api.CDefError("cannot render the type %r: it is a function " - "type, not a pointer-to-function type" % (self,)) + raise CDefError("cannot render the type %r: it is a function " + "type, not a pointer-to-function type" % (self,)) def as_function_pointer(self): return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi) @@ -309,9 +307,8 @@ def build_backend_type(self, ffi, finishlist): if self.length == '...': - from . import api - raise api.CDefError("cannot render the type %r: unknown length" % - (self,)) + raise CDefError("cannot render the type %r: unknown length" % + (self,)) self.item.get_cached_btype(ffi, finishlist) # force the item BType BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) @@ -457,13 +454,11 @@ self.completed = 2 def _verification_error(self, msg): - from .ffiplatform import VerificationError raise VerificationError(msg) def check_not_partial(self): if self.partial and self.fixedlayout is None: - from . import ffiplatform - raise ffiplatform.VerificationMissing(self._get_c_name()) + raise VerificationMissing(self._get_c_name()) def build_backend_type(self, ffi, finishlist): self.check_not_partial() @@ -501,8 +496,7 @@ def check_not_partial(self): if self.partial and not self.partial_resolved: - from . import ffiplatform - raise ffiplatform.VerificationMissing(self._get_c_name()) + raise VerificationMissing(self._get_c_name()) def build_backend_type(self, ffi, finishlist): self.check_not_partial() @@ -516,7 +510,6 @@ if self.baseinttype is not None: return self.baseinttype.get_cached_btype(ffi, finishlist) # - from . import api if self.enumvalues: smallest_value = min(self.enumvalues) largest_value = max(self.enumvalues) @@ -551,8 +544,8 @@ if (smallest_value >= ((-1) << (8*size2-1)) and largest_value < (1 << (8*size2-sign))): return btype2 - raise api.CDefError("%s values don't all fit into either 'long' " - "or 'unsigned long'" % self._get_c_name()) + raise CDefError("%s values don't all fit into either 'long' " + "or 'unsigned long'" % self._get_c_name()) def unknown_type(name, structname=None): if structname is None: diff --git a/cffi/recompiler.py b/cffi/recompiler.py --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -1,5 +1,6 @@ import os, sys, io from . import ffiplatform, model +from .error import VerificationError from .cffi_opcode import * VERSION = "0x2601" @@ -211,7 +212,7 @@ method = getattr(self, '_generate_cpy_%s_%s' % (kind, step_name)) except AttributeError: - raise ffiplatform.VerificationError( + raise VerificationError( "not implemented in recompile(): %r" % name) try: self._current_quals = quals @@ -354,12 +355,12 @@ included_module_name, included_source = ( ffi_to_include._assigned_source[:2]) except AttributeError: - raise ffiplatform.VerificationError( + raise VerificationError( "ffi object %r includes %r, but the latter has not " "been prepared with set_source()" % ( self.ffi, ffi_to_include,)) if included_source is None: - raise ffiplatform.VerificationError( + raise VerificationError( "not implemented yet: ffi.include() of a Python-based " "ffi inside a C-based ffi") prnt(' "%s",' % (included_module_name,)) @@ -391,6 +392,10 @@ prnt() # # the init function + prnt('#ifdef __GNUC__') + prnt('# pragma GCC visibility push(default) /* for -fvisibility= */') + prnt('#endif') + prnt() prnt('#ifdef PYPY_VERSION') prnt('PyMODINIT_FUNC') prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,)) @@ -429,6 +434,10 @@ self.module_name, version)) prnt('}') prnt('#endif') + prnt() + prnt('#ifdef __GNUC__') + prnt('# pragma GCC visibility pop') + prnt('#endif') def _to_py(self, x): if isinstance(x, str): @@ -456,12 +465,12 @@ included_module_name, included_source = ( ffi_to_include._assigned_source[:2]) except AttributeError: - raise ffiplatform.VerificationError( + raise VerificationError( "ffi object %r includes %r, but the latter has not " "been prepared with set_source()" % ( self.ffi, ffi_to_include,)) if included_source is not None: - raise ffiplatform.VerificationError( + raise VerificationError( "not implemented yet: ffi.include() of a C-based " "ffi inside a Python-based ffi") prnt('from %s import ffi as _ffi%d' % (included_module_name, i)) @@ -831,7 +840,7 @@ prnt(' { %s = &p->%s; (void)tmp; }' % ( ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), fname)) - except ffiplatform.VerificationError as e: + except VerificationError as e: prnt(' /* %s */' % str(e)) # cannot verify it, ignore prnt('}') prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname)) @@ -994,7 +1003,7 @@ def _generate_cpy_const(self, is_int, name, tp=None, category='const', check_value=None): if (category, name) in self._seen_constants: - raise ffiplatform.VerificationError( + raise VerificationError( "duplicate declaration of %s '%s'" % (category, name)) self._seen_constants.add((category, name)) # @@ -1093,7 +1102,7 @@ def _generate_cpy_macro_ctx(self, tp, name): if tp == '...': if self.target_is_python: - raise ffiplatform.VerificationError( + raise VerificationError( "cannot use the syntax '...' in '#define %s ...' when " "using the ABI mode" % (name,)) check_value = None @@ -1226,7 +1235,7 @@ def _generate_cpy_extern_python_ctx(self, tp, name): if self.target_is_python: - raise ffiplatform.VerificationError( + raise VerificationError( "cannot use 'extern \"Python\"' in the ABI mode") if tp.ellipsis: raise NotImplementedError("a vararg function is extern \"Python\"") @@ -1307,7 +1316,7 @@ if tp.length is None: self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index) elif tp.length == '...': - raise ffiplatform.VerificationError( + raise VerificationError( "type %s badly placed: the '...' array length can only be " "used on global arrays or on fields of structures" % ( str(tp).replace('/*...*/', '...'),)) diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py --- a/cffi/vengine_cpy.py +++ b/cffi/vengine_cpy.py @@ -2,7 +2,8 @@ # DEPRECATED: implementation for ffi.verify() # import sys, imp -from . import model, ffiplatform +from . import model +from .error import VerificationError class VCPythonEngine(object): @@ -155,7 +156,7 @@ self.verifier.modulefilename) except ImportError as e: error = "importing %r: %s" % (self.verifier.modulefilename, e) - raise ffiplatform.VerificationError(error) + raise VerificationError(error) finally: if hasattr(sys, "setdlopenflags"): sys.setdlopenflags(previous_flags) @@ -185,7 +186,7 @@ def __dir__(self): return FFILibrary._cffi_dir + list(self.__dict__) library = FFILibrary() - if module._cffi_setup(lst, ffiplatform.VerificationError, library): + if module._cffi_setup(lst, VerificationError, library): import warnings warnings.warn("reimporting %r might overwrite older definitions" % (self.verifier.get_module_name())) @@ -212,7 +213,7 @@ method = getattr(self, '_generate_cpy_%s_%s' % (kind, step_name)) except AttributeError: - raise ffiplatform.VerificationError( + raise VerificationError( "not implemented in verify(): %r" % name) try: method(tp, realname) @@ -485,7 +486,7 @@ prnt(' { %s = &p->%s; (void)tmp; }' % ( ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), fname)) - except ffiplatform.VerificationError as e: + except VerificationError as e: prnt(' /* %s */' % str(e)) # cannot verify it, ignore prnt('}') prnt('static PyObject *') @@ -550,7 +551,7 @@ # check that the layout sizes and offsets match the real ones def check(realvalue, expectedvalue, msg): if realvalue != expectedvalue: - raise ffiplatform.VerificationError( + raise VerificationError( "%s (we have %d, but C compiler says %d)" % (msg, expectedvalue, realvalue)) ffi = self.ffi @@ -771,7 +772,7 @@ BItemType = self.ffi._get_cached_btype(tp.item) length, rest = divmod(size, self.ffi.sizeof(BItemType)) if rest != 0: - raise ffiplatform.VerificationError( + raise VerificationError( "bad size: %r does not seem to be an array of %s" % (name, tp.item)) tp = tp.resolve_length(length) diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py --- a/cffi/vengine_gen.py +++ b/cffi/vengine_gen.py @@ -4,7 +4,8 @@ import sys, os import types -from . import model, ffiplatform From pypy.commits at gmail.com Mon Mar 13 02:45:50 2017 From: pypy.commits at gmail.com (arigo) Date: Sun, 12 Mar 2017 23:45:50 -0700 (PDT) Subject: [pypy-commit] pypy default: Skip test in -A Message-ID: <58c6401e.51a0190a.f743a.1635@mx.google.com> Author: Armin Rigo Branch: Changeset: r90642:87e9652421d1 Date: 2017-03-13 07:45 +0100 http://bitbucket.org/pypy/pypy/changeset/87e9652421d1/ Log: Skip test in -A diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -224,22 +224,27 @@ else: cls.w_tmpname = cls.space.wrap(tmpname) - cls.total_mem = 0 - def add_memory_pressure(estimate): - assert estimate >= 0 - cls.total_mem += estimate - cls.orig_add_memory_pressure = [rgc.add_memory_pressure] - rgc.add_memory_pressure = add_memory_pressure + if not cls.runappdirect: + cls.total_mem = 0 + def add_memory_pressure(estimate): + assert estimate >= 0 + cls.total_mem += estimate + cls.orig_add_memory_pressure = [rgc.add_memory_pressure] + rgc.add_memory_pressure = add_memory_pressure - def _reset_memory_pressure(space): - cls.total_mem = 0 - cls.w_reset_memory_pressure = cls.space.wrap( - gateway.interp2app(_reset_memory_pressure)) + def _reset_memory_pressure(space): + cls.total_mem = 0 + cls.w_reset_memory_pressure = cls.space.wrap( + gateway.interp2app(_reset_memory_pressure)) - def _cur_memory_pressure(space): - return space.newint(cls.total_mem) - cls.w_cur_memory_pressure = cls.space.wrap( - gateway.interp2app(_cur_memory_pressure)) + def _cur_memory_pressure(space): + return space.newint(cls.total_mem) + cls.w_cur_memory_pressure = cls.space.wrap( + gateway.interp2app(_cur_memory_pressure)) + else: + def _skip_test(*ignored): + pytest.skip("not for -A testing") + cls.w_reset_memory_pressure = _skip_test def teardown_class(cls): from rpython.rlib import rgc @@ -349,6 +354,7 @@ assert isinstance(a, unicode) def test_add_memory_pressure(self): + self.reset_memory_pressure() # for the potential skip module = self.import_extension('foo', [ ("foo", "METH_O", """ From pypy.commits at gmail.com Mon Mar 13 03:23:36 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 00:23:36 -0700 (PDT) Subject: [pypy-commit] pypy default: Check in here test runners for the lib-python and the pypyjit steps Message-ID: <58c648f8.14542e0a.fc9ed.21c0@mx.google.com> Author: Armin Rigo Branch: Changeset: r90643:6790d94fb46a Date: 2017-03-13 08:23 +0100 http://bitbucket.org/pypy/pypy/changeset/6790d94fb46a/ Log: Check in here test runners for the lib-python and the pypyjit steps of buildbot. Will then fix buildbot to execute these. It allows us to customize the steps on py3.5. If you are running buildbot on other branches, you need to copy these two files too (or merge default). diff --git a/testrunner/lib_python_tests.py b/testrunner/lib_python_tests.py new file mode 100755 --- /dev/null +++ b/testrunner/lib_python_tests.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +""" +This is what the buildbot runs to execute the lib-python tests +on top of pypy-c. +""" + +import sys, os +import subprocess + +rootdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) + +popen = subprocess.Popen( + [sys.executable, "pypy/test_all.py", + "--pypy=pypy/goal/pypy-c", + "--timeout=3600", + "--resultlog=cpython.log", "lib-python"], + cwd=rootdir) + +try: + ret = popen.wait() +except KeyboardInterrupt: + popen.kill() + print "\ninterrupted" + ret = 1 + +sys.exit(ret) diff --git a/testrunner/pypyjit_tests.py b/testrunner/pypyjit_tests.py new file mode 100755 --- /dev/null +++ b/testrunner/pypyjit_tests.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +""" +This is what the buildbot runs to execute the pypyjit tests +on top of pypy-c. +""" + +import sys, os +import subprocess + +rootdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) + +popen = subprocess.Popen( + ["pypy/goal/pypy-c", "pypy/test_all.py", + "--resultlog=pypyjit_new.log", + "pypy/module/pypyjit/test_pypy_c"], + cwd=rootdir) + +try: + ret = popen.wait() +except KeyboardInterrupt: + popen.kill() + print "\ninterrupted" + ret = 1 + +sys.exit(ret) From pypy.commits at gmail.com Mon Mar 13 03:45:29 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 00:45:29 -0700 (PDT) Subject: [pypy-commit] pypy default: fixes, also add app_level_tests.py Message-ID: <58c64e19.8173190a.8d21e.25e2@mx.google.com> Author: Armin Rigo Branch: Changeset: r90644:fd8af983f881 Date: 2017-03-13 08:45 +0100 http://bitbucket.org/pypy/pypy/changeset/fd8af983f881/ Log: fixes, also add app_level_tests.py diff --git a/testrunner/app_level_tests.py b/testrunner/app_level_tests.py new file mode 100755 --- /dev/null +++ b/testrunner/app_level_tests.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +""" +This is what the buildbot runs to execute the app-level tests +on top of pypy-c. +""" + +import sys, os +import subprocess + +rootdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) +os.environ['PYTHONPATH'] = rootdir +os.environ['PYTEST_PLUGINS'] = '' + +popen = subprocess.Popen( + [sys.executable, "testrunner/runner.py", + "--logfile=pytest-A.log", + "--config=pypy/pytest-A.cfg", + "--config=pypy/pytest-A.py", + "--config=~/machine-A_cfg.py", + "--root=pypy", "--timeout=3600", + ] + sys.argv[1:], + cwd=rootdir) + +try: + ret = popen.wait() +except KeyboardInterrupt: + popen.kill() + print "\ninterrupted" + ret = 1 + +sys.exit(ret) diff --git a/testrunner/lib_python_tests.py b/testrunner/lib_python_tests.py old mode 100755 new mode 100644 --- a/testrunner/lib_python_tests.py +++ b/testrunner/lib_python_tests.py @@ -8,12 +8,15 @@ import subprocess rootdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) +os.environ['PYTHONPATH'] = rootdir +os.environ['PYTEST_PLUGINS'] = '' popen = subprocess.Popen( [sys.executable, "pypy/test_all.py", - "--pypy=pypy/goal/pypy-c", - "--timeout=3600", - "--resultlog=cpython.log", "lib-python"], + "--pypy=pypy/goal/pypy-c", + "--timeout=3600", + "--resultlog=cpython.log", "lib-python", + ] + sys.argv[1:], cwd=rootdir) try: diff --git a/testrunner/pypyjit_tests.py b/testrunner/pypyjit_tests.py old mode 100755 new mode 100644 --- a/testrunner/pypyjit_tests.py +++ b/testrunner/pypyjit_tests.py @@ -8,11 +8,14 @@ import subprocess rootdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) +os.environ['PYTHONPATH'] = rootdir +os.environ['PYTEST_PLUGINS'] = '' popen = subprocess.Popen( ["pypy/goal/pypy-c", "pypy/test_all.py", - "--resultlog=pypyjit_new.log", - "pypy/module/pypyjit/test_pypy_c"], + "--resultlog=pypyjit_new.log", + "pypy/module/pypyjit/test_pypy_c", + ] + sys.argv[1:], cwd=rootdir) try: From pypy.commits at gmail.com Mon Mar 13 03:50:37 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 00:50:37 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58c64f4d.02a2190a.2625e.215d@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90647:7c1314187e2f Date: 2017-03-13 08:50 +0100 http://bitbucket.org/pypy/pypy/changeset/7c1314187e2f/ Log: hg merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -200,3 +200,5 @@ asmgcc---close enough that we can now make shadowstack the default even on Linux. This should remove a whole class of rare bugs introduced by asmgcc. + +.. branch: fniephaus/fix-typo-1488123166752 diff --git a/pypy/module/_minimal_curses/__init__.py b/pypy/module/_minimal_curses/__init__.py --- a/pypy/module/_minimal_curses/__init__.py +++ b/pypy/module/_minimal_curses/__init__.py @@ -10,7 +10,6 @@ py.test.skip("no _curses or _minimal_curses module") # no _curses at all from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._minimal_curses import fficurses # for side effects class Module(MixedModule): diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py --- a/pypy/module/_minimal_curses/fficurses.py +++ b/pypy/module/_minimal_curses/fficurses.py @@ -1,11 +1,8 @@ -""" The ffi for rpython, need to be imported for side effects +""" The ffi for rpython """ from rpython.rtyper.lltypesystem import rffi -from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.tool import rffi_platform -from rpython.rtyper.extfunc import register_external -from pypy.module._minimal_curses import interp_curses from rpython.translator.tool.cbuild import ExternalCompilationInfo # We cannot trust ncurses5-config, it's broken in various ways in @@ -58,86 +55,73 @@ eci = guess_eci() -INT = rffi.INT -INTP = lltype.Ptr(lltype.Array(INT, hints={'nolength':True})) -c_setupterm = rffi.llexternal('setupterm', [rffi.CCHARP, INT, INTP], INT, - compilation_info=eci) -c_tigetstr = rffi.llexternal('tigetstr', [rffi.CCHARP], rffi.CCHARP, - compilation_info=eci) -c_tparm = rffi.llexternal('tparm', [rffi.CCHARP, INT, INT, INT, INT, INT, - INT, INT, INT, INT], rffi.CCHARP, - compilation_info=eci) +# We should not use this 'eci' directly because it causes the #include +# of term.h to appear in all generated C sources, and term.h contains a +# poisonous quantity of #defines for common lower-case names like +# 'buttons' or 'lines' (!!!). It is basically dangerous to include +# term.h in any C source file that may contain unrelated source code. -ERR = rffi.CConstant('ERR', lltype.Signed) -OK = rffi.CConstant('OK', lltype.Signed) +include_lines = '\n'.join(['#include <%s>' % _incl for _incl in eci.includes]) +eci = eci.copy_without('includes') -def curses_setupterm(term, fd): - intp = lltype.malloc(INTP.TO, 1, flavor='raw') - err = rffi.cast(lltype.Signed, c_setupterm(term, fd, intp)) - try: - if err == ERR: - errret = rffi.cast(lltype.Signed, intp[0]) - if errret == 0: - msg = "setupterm: could not find terminal" - elif errret == -1: - msg = "setupterm: could not find terminfo database" - else: - msg = "setupterm: unknown error" - raise interp_curses.curses_error(msg) - interp_curses.module_info.setupterm_called = True - finally: - lltype.free(intp, flavor='raw') -def curses_setupterm_null_llimpl(fd): - curses_setupterm(lltype.nullptr(rffi.CCHARP.TO), fd) +eci = eci.merge(ExternalCompilationInfo( + post_include_bits=[ + "RPY_EXTERN char *rpy_curses_setupterm(char *, int);\n" + "RPY_EXTERN char *rpy_curses_tigetstr(char *);\n" + "RPY_EXTERN char *rpy_curses_tparm(char *, int, int, int, int," + " int, int, int, int, int);" + ], + separate_module_sources=[""" -def curses_setupterm_llimpl(term, fd): - ll_s = rffi.str2charp(term) - try: - curses_setupterm(ll_s, fd) - finally: - rffi.free_charp(ll_s) +%(include_lines)s -register_external(interp_curses._curses_setupterm_null, - [int], llimpl=curses_setupterm_null_llimpl, - export_name='_curses.setupterm_null') -register_external(interp_curses._curses_setupterm, - [str, int], llimpl=curses_setupterm_llimpl, - export_name='_curses.setupterm') +RPY_EXTERN +char *rpy_curses_setupterm(char *term, int fd) +{ + int errret = -42; + if (setupterm(term, fd, &errret) == ERR) { + switch (errret) { + case 0: + return "setupterm: could not find terminal"; + case -1: + return "setupterm: could not find terminfo database"; + default: + return "setupterm: unknown error"; + } + } + return NULL; +} -def check_setup_invoked(): - if not interp_curses.module_info.setupterm_called: - raise interp_curses.curses_error("must call (at least) setupterm() first") +RPY_EXTERN +char *rpy_curses_tigetstr(char *capname) +{ + char *res = tigetstr(capname); + if (res == (char *)-1) + res = NULL; + return res; +} -def tigetstr_llimpl(cap): - check_setup_invoked() - ll_cap = rffi.str2charp(cap) - try: - ll_res = c_tigetstr(ll_cap) - num = lltype.cast_ptr_to_int(ll_res) - if num == 0 or num == -1: - raise interp_curses.TermError() - res = rffi.charp2str(ll_res) - return res - finally: - rffi.free_charp(ll_cap) +RPY_EXTERN +char *rpy_curses_tparm(char *str, int x0, int x1, int x2, int x3, + int x4, int x5, int x6, int x7, int x8) +{ + return tparm(str, x0, x1, x2, x3, x4, x5, x6, x7, x8); +} -register_external(interp_curses._curses_tigetstr, [str], str, - export_name='_curses.tigetstr', llimpl=tigetstr_llimpl) +""" % globals()])) -def tparm_llimpl(s, args): - check_setup_invoked() - l = [0, 0, 0, 0, 0, 0, 0, 0, 0] - for i in range(min(len(args), 9)): - l[i] = args[i] - ll_s = rffi.str2charp(s) - # XXX nasty trick stolen from CPython - ll_res = c_tparm(ll_s, l[0], l[1], l[2], l[3], l[4], l[5], l[6], - l[7], l[8]) - rffi.free_charp(ll_s) - res = rffi.charp2str(ll_res) - return res -register_external(interp_curses._curses_tparm, [str, [int]], str, - export_name='_curses.tparm', llimpl=tparm_llimpl) +rpy_curses_setupterm = rffi.llexternal( + "rpy_curses_setupterm", [rffi.CCHARP, rffi.INT], rffi.CCHARP, + compilation_info=eci) +rpy_curses_tigetstr = rffi.llexternal( + "rpy_curses_tigetstr", [rffi.CCHARP], rffi.CCHARP, + compilation_info=eci) + +rpy_curses_tparm = rffi.llexternal( + "rpy_curses_tparm", [rffi.CCHARP, rffi.INT, rffi.INT, rffi.INT, rffi.INT, + rffi.INT, rffi.INT, rffi.INT, rffi.INT, rffi.INT], + rffi.CCHARP, + compilation_info=eci) diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -1,44 +1,24 @@ - from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import OperationError -from pypy.module._minimal_curses import _curses +from pypy.module._minimal_curses import fficurses +from rpython.rtyper.lltypesystem import lltype, rffi + class ModuleInfo: - def __init__(self): + def __init__(self, space): self.setupterm_called = False -module_info = ModuleInfo() +def check_setup_invoked(space): + if not space.fromcache(ModuleInfo).setupterm_called: + raise curses_error(space, "must call (at least) setupterm() first") -class curses_error(Exception): - def __init__(self, msg): - self.msg = msg -from rpython.annotator.classdesc import FORCE_ATTRIBUTES_INTO_CLASSES -from rpython.annotator.model import SomeString - -# this is necessary due to annmixlevel -FORCE_ATTRIBUTES_INTO_CLASSES[curses_error] = {'msg': SomeString()} - -def convert_error(space, error): - msg = error.msg +def curses_error(space, errmsg): w_module = space.getbuiltinmodule('_minimal_curses') w_exception_class = space.getattr(w_module, space.newtext('error')) - w_exception = space.call_function(w_exception_class, space.newtext(msg)) + w_exception = space.call_function(w_exception_class, space.newtext(errmsg)) return OperationError(w_exception_class, w_exception) -def _curses_setupterm_null(fd): - # NOT_RPYTHON - try: - _curses.setupterm(None, fd) - except _curses.error as e: - raise curses_error(e.args[0]) - -def _curses_setupterm(termname, fd): - # NOT_RPYTHON - try: - _curses.setupterm(termname, fd) - except _curses.error as e: - raise curses_error(e.args[0]) @unwrap_spec(fd=int) def setupterm(space, w_termname=None, fd=-1): @@ -47,48 +27,47 @@ space.newtext('stdout')) fd = space.int_w(space.call_function(space.getattr(w_stdout, space.newtext('fileno')))) - try: - if space.is_none(w_termname): - _curses_setupterm_null(fd) - else: - _curses_setupterm(space.text_w(w_termname), fd) - except curses_error as e: - raise convert_error(space, e) + if space.is_none(w_termname): + termname = None + else: + termname = space.text_w(w_termname) -class TermError(Exception): - pass + with rffi.scoped_str2charp(termname) as ll_term: + fd = rffi.cast(rffi.INT, fd) + ll_errmsg = fficurses.rpy_curses_setupterm(ll_term, fd) + if ll_errmsg: + raise curses_error(space, rffi.charp2str(ll_errmsg)) -def _curses_tigetstr(capname): - # NOT_RPYTHON - try: - res = _curses.tigetstr(capname) - except _curses.error as e: - raise curses_error(e.args[0]) - if res is None: - raise TermError - return res - -def _curses_tparm(s, args): - # NOT_RPYTHON - try: - return _curses.tparm(s, *args) - except _curses.error as e: - raise curses_error(e.args[0]) + space.fromcache(ModuleInfo).setupterm_called = True @unwrap_spec(capname='text') def tigetstr(space, capname): - try: - result = _curses_tigetstr(capname) - except TermError: - return space.w_None - except curses_error as e: - raise convert_error(space, e) - return space.newbytes(result) + check_setup_invoked(space) + with rffi.scoped_str2charp(capname) as ll_capname: + ll_result = fficurses.rpy_curses_tigetstr(ll_capname) + if ll_result: + return space.newbytes(rffi.charp2str(ll_result)) + else: + return space.w_None @unwrap_spec(s='bufferstr') def tparm(space, s, args_w): + check_setup_invoked(space) args = [space.int_w(a) for a in args_w] - try: - return space.newbytes(_curses_tparm(s, args)) - except curses_error as e: - raise convert_error(space, e) + # nasty trick stolen from CPython + x0 = args[0] if len(args) > 0 else 0 + x1 = args[1] if len(args) > 1 else 0 + x2 = args[2] if len(args) > 2 else 0 + x3 = args[3] if len(args) > 3 else 0 + x4 = args[4] if len(args) > 4 else 0 + x5 = args[5] if len(args) > 5 else 0 + x6 = args[6] if len(args) > 6 else 0 + x7 = args[7] if len(args) > 7 else 0 + x8 = args[8] if len(args) > 8 else 0 + with rffi.scoped_str2charp(s) as ll_str: + ll_result = fficurses.rpy_curses_tparm(ll_str, x0, x1, x2, x3, + x4, x5, x6, x7, x8) + if ll_result: + return space.newbytes(rffi.charp2str(ll_result)) + else: + raise curses_error(space, "tparm() returned NULL") diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py --- a/pypy/module/_minimal_curses/test/test_curses.py +++ b/pypy/module/_minimal_curses/test/test_curses.py @@ -76,19 +76,27 @@ """ def test_csetupterm(self): from rpython.translator.c.test.test_genc import compile - from pypy.module._minimal_curses import interp_curses + from rpython.rtyper.lltypesystem import lltype, rffi + from pypy.module._minimal_curses import fficurses + def runs_setupterm(): - interp_curses._curses_setupterm_null(1) + null = lltype.nullptr(rffi.CCHARP.TO) + fficurses.rpy_curses_setupterm(null, 1) fn = compile(runs_setupterm, []) fn() def test_ctgetstr(self): from rpython.translator.c.test.test_genc import compile - from pypy.module._minimal_curses import interp_curses + from rpython.rtyper.lltypesystem import lltype, rffi + from pypy.module._minimal_curses import fficurses + def runs_ctgetstr(): - interp_curses._curses_setupterm("xterm", 1) - return interp_curses._curses_tigetstr('cup') + with rffi.scoped_str2charp("xterm") as ll_term: + fficurses.rpy_curses_setupterm(ll_term, 1) + with rffi.scoped_str2charp("cup") as ll_capname: + ll = fficurses.rpy_curses_tigetstr(ll_capname) + return rffi.charp2str(ll) fn = compile(runs_ctgetstr, []) res = fn() @@ -96,11 +104,16 @@ def test_ctparm(self): from rpython.translator.c.test.test_genc import compile - from pypy.module._minimal_curses import interp_curses + from rpython.rtyper.lltypesystem import lltype, rffi + from pypy.module._minimal_curses import fficurses + def runs_tparm(): - interp_curses._curses_setupterm("xterm", 1) - cup = interp_curses._curses_tigetstr('cup') - return interp_curses._curses_tparm(cup, [5, 3]) + with rffi.scoped_str2charp("xterm") as ll_term: + fficurses.rpy_curses_setupterm(ll_term, 1) + with rffi.scoped_str2charp("cup") as ll_capname: + cup = fficurses.rpy_curses_tigetstr(ll_capname) + res = fficurses.rpy_curses_tparm(cup, 5, 3, 0, 0, 0, 0, 0, 0, 0) + return rffi.charp2str(res) fn = compile(runs_tparm, []) res = fn() diff --git a/pypy/module/cpyext/include/pymem.h b/pypy/module/cpyext/include/pymem.h --- a/pypy/module/cpyext/include/pymem.h +++ b/pypy/module/cpyext/include/pymem.h @@ -60,6 +60,25 @@ #define PyMem_Del PyMem_Free #define PyMem_DEL PyMem_FREE + +/* From CPython 3.6, with a different goal. _PyTraceMalloc_Track() + * is equivalent to __pypy__.add_memory_pressure(size); it works with + * or without the GIL. _PyTraceMalloc_Untrack() is an empty stub. + * You can check if these functions are available by using: + * + * #if defined(PYPY_TRACEMALLOC) || \ + * (PY_VERSION_HEX >= 0x03060000 && !defined(Py_LIMITED_API)) + */ +#define PYPY_TRACEMALLOC 1 + +typedef unsigned int _PyTraceMalloc_domain_t; + +PyAPI_FUNC(int) _PyTraceMalloc_Track(_PyTraceMalloc_domain_t domain, + uintptr_t ptr, size_t size); +PyAPI_FUNC(int) _PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, + uintptr_t ptr); + + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -457,3 +457,8 @@ with rffi.scoped_nonmovingbuffer(data) as buf: fwrite(buf, 1, count, fp) return 0 + + at cpython_api([lltype.Signed], lltype.Void) +def _PyPyGC_AddMemoryPressure(space, report): + from rpython.rlib import rgc + rgc.add_memory_pressure(report) diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c --- a/pypy/module/cpyext/src/pymem.c +++ b/pypy/module/cpyext/src/pymem.c @@ -84,3 +84,46 @@ { free(ptr); } + +int _PyTraceMalloc_Track(_PyTraceMalloc_domain_t domain, + uintptr_t ptr, size_t size) +{ + /* to avoid acquiring/releasing the GIL too often, only do it + if the total reported size exceeds 64KB. */ + static volatile long unreported_size = 0; + long prev, next, report; + + size += sizeof(long); + /* ^^^ to account for some alignment. Important, otherwise we'd + * collect sizes of, say, 1-bytes mallocs in 1-bytes increment */ + + retry: + report = 0; + prev = unreported_size; + next = prev + size; + if (next >= 65536) { + report = next; + next = 0; + } + if (prev != next) { +#ifdef _WIN32 + if (InterlockedCompareExchange(&unreported_size, next, prev) != prev) + goto retry; +#else + if (!__sync_bool_compare_and_swap(&unreported_size, prev, next)) + goto retry; +#endif + } + + if (report) { + PyGILState_STATE state = PyGILState_Ensure(); + _PyPyGC_AddMemoryPressure(report); + PyGILState_Release(state); + } +} + +int _PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, + uintptr_t ptr) +{ + /* nothing */ +} diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -191,10 +191,40 @@ class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): + from rpython.rlib import rgc + from pypy.interpreter import gateway + AppTestCpythonExtensionBase.setup_class.im_func(cls) tmpname = str(py.test.ensuretemp('out', dir=0)) cls.w_tmpname = cls.space.wrap(tmpname) + if not cls.runappdirect: + cls.total_mem = 0 + def add_memory_pressure(estimate): + assert estimate >= 0 + cls.total_mem += estimate + cls.orig_add_memory_pressure = [rgc.add_memory_pressure] + rgc.add_memory_pressure = add_memory_pressure + + def _reset_memory_pressure(space): + cls.total_mem = 0 + cls.w_reset_memory_pressure = cls.space.wrap( + gateway.interp2app(_reset_memory_pressure)) + + def _cur_memory_pressure(space): + return space.newint(cls.total_mem) + cls.w_cur_memory_pressure = cls.space.wrap( + gateway.interp2app(_cur_memory_pressure)) + else: + def _skip_test(*ignored): + pytest.skip("not for -A testing") + cls.w_reset_memory_pressure = _skip_test + + def teardown_class(cls): + from rpython.rlib import rgc + if hasattr(cls, 'orig_add_memory_pressure'): + [rgc.add_memory_pressure] = cls.orig_add_memory_pressure + def test_object_malloc(self): module = self.import_extension('foo', [ ("malloctest", "METH_NOARGS", @@ -310,6 +340,31 @@ assert type(module.asbytes(sub1(b''))) is bytes assert type(module.asbytes(sub2(b''))) is sub2 + def test_add_memory_pressure(self): + self.reset_memory_pressure() # for the potential skip + module = self.import_extension('foo', [ + ("foo", "METH_O", + """ + _PyTraceMalloc_Track(0, 0, PyInt_AsLong(args) - sizeof(long)); + Py_INCREF(Py_None); + return Py_None; + """)]) + self.reset_memory_pressure() + module.foo(42) + assert self.cur_memory_pressure() == 0 + module.foo(65000 - 42) + assert self.cur_memory_pressure() == 0 + module.foo(536) + assert self.cur_memory_pressure() == 65536 + module.foo(40000) + assert self.cur_memory_pressure() == 65536 + module.foo(40000) + assert self.cur_memory_pressure() == 65536 + 80000 + module.foo(35000) + assert self.cur_memory_pressure() == 65536 + 80000 + module.foo(35000) + assert self.cur_memory_pressure() == 65536 + 80000 + 70000 + class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase): """ PyBuffer_FillInfo populates the fields of a Py_buffer from its arguments. diff --git a/pypy/sandbox/test/test_pypy_interact.py b/pypy/sandbox/test/test_pypy_interact.py --- a/pypy/sandbox/test/test_pypy_interact.py +++ b/pypy/sandbox/test/test_pypy_interact.py @@ -74,7 +74,8 @@ def setup_module(mod): - t = Translation(mini_pypy_like_entry_point, backend='c', sandbox=True) + t = Translation(mini_pypy_like_entry_point, backend='c', sandbox=True, + lldebug=True) mod.executable = str(t.compile()) diff --git a/rpython/translator/c/src/stacklet/stacklet.c b/rpython/translator/c/src/stacklet/stacklet.c --- a/rpython/translator/c/src/stacklet/stacklet.c +++ b/rpython/translator/c/src/stacklet/stacklet.c @@ -16,6 +16,7 @@ * can redefine it to upwards growing, 1. */ #define STACK_DIRECTION 0 +#define STATIC_NOINLINE __attribute__((noinline)) static #include "src/stacklet/slp_platformselect.h" @@ -56,11 +57,6 @@ stacklet_thread_handle stack_thrd; /* the thread where the stacklet is */ }; -void *(*_stacklet_switchstack)(void*(*)(void*, void*), - void*(*)(void*, void*), void*) = NULL; -void (*_stacklet_initialstub)(struct stacklet_thread_s *, - stacklet_run_fn, void *) = NULL; - struct stacklet_thread_s { struct stacklet_s *g_stack_chain_head; /* NULL <=> running main */ char *g_current_stack_stop; @@ -252,8 +248,17 @@ return EMPTY_STACKLET_HANDLE; } -static void g_initialstub(struct stacklet_thread_s *thrd, - stacklet_run_fn run, void *run_arg) +STATIC_NOINLINE +void *_stacklet_switchstack(void *(*save_state)(void*, void*), + void *(*restore_state)(void*, void*), + void *extra) +{ + return slp_switch(save_state, restore_state, extra); +} + +STATIC_NOINLINE +void g_initialstub(struct stacklet_thread_s *thrd, + stacklet_run_fn run, void *run_arg) { struct stacklet_s *result; @@ -284,13 +289,6 @@ { struct stacklet_thread_s *thrd; - if (_stacklet_switchstack == NULL) { - /* set up the following global with an indirection, which is needed - to prevent any inlining */ - _stacklet_initialstub = g_initialstub; - _stacklet_switchstack = slp_switch; - } - thrd = malloc(sizeof(struct stacklet_thread_s)); if (thrd != NULL) memset(thrd, 0, sizeof(struct stacklet_thread_s)); @@ -311,7 +309,7 @@ thrd->g_current_stack_stop = ((char *)&stackmarker) + 1; thrd->g_current_stack_marker = (char *)&stackmarker; - _stacklet_initialstub(thrd, run, run_arg); + g_initialstub(thrd, run, run_arg); return thrd->g_source; } diff --git a/rpython/translator/c/src/stacklet/switch_x64_msvc.h b/rpython/translator/c/src/stacklet/switch_x64_msvc.h --- a/rpython/translator/c/src/stacklet/switch_x64_msvc.h +++ b/rpython/translator/c/src/stacklet/switch_x64_msvc.h @@ -5,3 +5,5 @@ void *(*restore_state)(void*, void*), void *extra); +#undef STATIC_NOINLINE +#define STATIC_NOINLINE static __declspec(noinline) diff --git a/rpython/translator/c/src/stacklet/switch_x86_msvc.h b/rpython/translator/c/src/stacklet/switch_x86_msvc.h --- a/rpython/translator/c/src/stacklet/switch_x86_msvc.h +++ b/rpython/translator/c/src/stacklet/switch_x86_msvc.h @@ -5,6 +5,9 @@ void *(*restore_state)(void*, void*), void *extra); +#undef STATIC_NOINLINE +#define STATIC_NOINLINE static __declspec(noinline) + #define WIN32_LEAN_AND_MEAN #include diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py --- a/rpython/translator/sandbox/test/test_sandbox.py +++ b/rpython/translator/sandbox/test/test_sandbox.py @@ -37,9 +37,9 @@ write_message(g, result, resulttype) g.flush() -def compile(f, gc='ref'): +def compile(f, gc='ref', **kwds): t = Translation(f, backend='c', sandbox=True, gc=gc, - check_str_without_nul=True) + check_str_without_nul=True, **kwds) return str(t.compile()) def run_in_subprocess(exe): @@ -198,7 +198,7 @@ l.append("x" * int(argv[2])) return int(len(l) > 1000) - exe = compile(entry_point, gc='hybrid') + exe = compile(entry_point, gc='hybrid', lldebug=True) pipe = subprocess.Popen([exe, '10', '10000'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) g = pipe.stdin diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py --- a/rpython/translator/tool/cbuild.py +++ b/rpython/translator/tool/cbuild.py @@ -334,3 +334,9 @@ d['separate_module_files'] = () d['separate_module_sources'] = () return ExternalCompilationInfo(**d) + + def copy_without(self, *names): + d = self._copy_attributes() + for name in names: + del d[name] + return ExternalCompilationInfo(**d) diff --git a/testrunner/app_level_tests.py b/testrunner/app_level_tests.py new file mode 100755 --- /dev/null +++ b/testrunner/app_level_tests.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +""" +This is what the buildbot runs to execute the app-level tests +on top of pypy-c. +""" + +import sys, os +import subprocess + +rootdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) +os.environ['PYTHONPATH'] = rootdir +os.environ['PYTEST_PLUGINS'] = '' + +popen = subprocess.Popen( + [sys.executable, "testrunner/runner.py", + "--logfile=pytest-A.log", + "--config=pypy/pytest-A.cfg", + "--config=pypy/pytest-A.py", + "--config=~/machine-A_cfg.py", + "--root=pypy", "--timeout=3600", + ] + sys.argv[1:], + cwd=rootdir) + +try: + ret = popen.wait() +except KeyboardInterrupt: + popen.kill() + print "\ninterrupted" + ret = 1 + +sys.exit(ret) diff --git a/testrunner/lib_python_tests.py b/testrunner/lib_python_tests.py new file mode 100755 --- /dev/null +++ b/testrunner/lib_python_tests.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +This is what the buildbot runs to execute the lib-python tests +on top of pypy-c. +""" + +import sys, os +import subprocess + +rootdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) +os.environ['PYTHONPATH'] = rootdir +os.environ['PYTEST_PLUGINS'] = '' + +popen = subprocess.Popen( + [sys.executable, "pypy/test_all.py", + "--pypy=pypy/goal/pypy3-c", + "--timeout=3600", + "--resultlog=cpython.log", "lib-python", + ] + sys.argv[1:], + cwd=rootdir) + +try: + ret = popen.wait() +except KeyboardInterrupt: + popen.kill() + print "\ninterrupted" + ret = 1 + +sys.exit(ret) diff --git a/testrunner/pypyjit_tests.py b/testrunner/pypyjit_tests.py new file mode 100755 --- /dev/null +++ b/testrunner/pypyjit_tests.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +This is what the buildbot runs to execute the pypyjit tests +on top of pypy-c. +""" + +import sys, os +import subprocess + +rootdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) +os.environ['PYTHONPATH'] = rootdir +os.environ['PYTEST_PLUGINS'] = '' + +popen = subprocess.Popen( + [sys.executable, "pypy/test_all.py", + "--pypy=pypy/goal/pypy3-c", + "--resultlog=pypyjit_new.log", + "pypy/module/pypyjit/test_pypy_c", + ] + sys.argv[1:], + cwd=rootdir) + +try: + ret = popen.wait() +except KeyboardInterrupt: + popen.kill() + print "\ninterrupted" + ret = 1 + +sys.exit(ret) From pypy.commits at gmail.com Mon Mar 13 03:50:32 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 00:50:32 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix Message-ID: <58c64f48.1a4b2e0a.42b6a.2673@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90645:3172a29c11fb Date: 2017-03-13 08:45 +0100 http://bitbucket.org/pypy/pypy/changeset/3172a29c11fb/ Log: fix diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg --- a/pypy/pytest-A.cfg +++ b/pypy/pytest-A.cfg @@ -1,5 +1,5 @@ cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] interp = ['python'] -test_driver = ['test_all.py', '-A', '--python=goal/pypy-c'] +test_driver = ['test_all.py', '-A', '--python=goal/pypy3-c'] From pypy.commits at gmail.com Mon Mar 13 03:50:35 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 00:50:35 -0700 (PDT) Subject: [pypy-commit] pypy default: Make these files executable Message-ID: <58c64f4b.1ce5190a.8bdb3.27e4@mx.google.com> Author: Armin Rigo Branch: Changeset: r90646:4f28537be39a Date: 2017-03-13 08:47 +0100 http://bitbucket.org/pypy/pypy/changeset/4f28537be39a/ Log: Make these files executable diff --git a/testrunner/lib_python_tests.py b/testrunner/lib_python_tests.py old mode 100644 new mode 100755 diff --git a/testrunner/pypyjit_tests.py b/testrunner/pypyjit_tests.py old mode 100644 new mode 100755 From pypy.commits at gmail.com Mon Mar 13 03:51:42 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 00:51:42 -0700 (PDT) Subject: [pypy-commit] buildbot default: run the new files testrunner/*_tests.py Message-ID: <58c64f8e.ca092e0a.dc54a.195a@mx.google.com> Author: Armin Rigo Branch: Changeset: r1020:b573abdcd2d7 Date: 2017-03-13 08:51 +0100 http://bitbucket.org/pypy/buildbot/changeset/b573abdcd2d7/ Log: run the new files testrunner/*_tests.py diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -437,63 +437,27 @@ app_tests = [] factory.addStep(PytestCmd( description="app-level (-A) test", - command=prefix + ["python", "testrunner/runner.py", - "--logfile=pytest-A.log", - "--config=pypy/pytest-A.cfg", - "--config=pypy/pytest-A.py", - "--config=~/machine-A_cfg.py", - "--root=pypy", "--timeout=3600" + command=prefix + ["python", "testrunner/app_level_tests.py", ] + ["--config=%s" % cfg for cfg in app_tests], logfiles={'pytestLog': 'pytest-A.log'}, timeout=4000, - env={"PYTHONPATH": ['.'], - "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) if lib_python: factory.addStep(PytestCmd( description="lib-python test", - command=prefix + ["python", "pypy/test_all.py", - "--pypy=pypy/goal/pypy-c", - "--timeout=3600", - "--resultlog=cpython.log", "lib-python"], + command=prefix + ["python", "testrunner/lib_python_tests.py"], timeout=4000, logfiles={'pytestLog': 'cpython.log'}, env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) if pypyjit: - # run test_pypy_c tests, which is a mess because it needs to be - # started differently on pypy2 or pypy3 factory.addStep(PytestCmd( description="pypyjit tests", - command=prefix + ["python", "-c", r""" -import sys, subprocess - -if sys.platform == 'win32': - cmd = r'pypy\goal\pypy-c' -else: - cmd = 'pypy/goal/pypy-c' - -g = subprocess.check_output([cmd, '-c', 'import sys;print(sys.version)']) -if g.startswith('2.7'): - # PyPy 2: must run 'pypy-c py.test' - cmdline = [cmd, "pypy/test_all.py"] -elif g.startswith('3'): - # PyPy 3: must run 'py.test --pypy=pypy-c' - cmdline = [sys.executable, "pypy/test_all.py", "--pypy=" + cmd] -else: - raise Exception("pypy-c output: %r" % (data,)) - -cmdline += ["--resultlog=pypyjit_new.log", - "pypy/module/pypyjit/test_pypy_c"] - -print("Running", cmdline) -res = subprocess.call(cmdline) -if res: - print("!!!!! error code ", res) - sys.exit(1) -"""], + command=prefix + ["python", "testrunner/pypyjit_tests.py"], + timeout=4000, logfiles={'pytestLog': 'pypyjit_new.log'}, env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) From pypy.commits at gmail.com Mon Mar 13 08:17:02 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 05:17:02 -0700 (PDT) Subject: [pypy-commit] cffi default: silence warnings Message-ID: <58c68dbe.a215190a.ff681.2fc3@mx.google.com> Author: Armin Rigo Branch: Changeset: r2907:5db068918b76 Date: 2017-03-13 13:17 +0100 http://bitbucket.org/cffi/cffi/changeset/5db068918b76/ Log: silence warnings diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -2039,7 +2039,7 @@ struct foo s = { 40, 200 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().x == 200 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2068,7 +2068,7 @@ s.b = 200; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().b == 200 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2094,7 +2094,7 @@ struct foo s = { 11 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().x == 11 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2118,7 +2118,7 @@ struct foo s = { 42 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().a == 42 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2142,7 +2142,7 @@ union foo s = { 42 }; return s; } - union foo g(int a, ...) { } + union foo g(int a, ...) { return f(); } """) assert lib.f().a == 42 e = py.test.raises(NotImplementedError, lib.g, 0) From pypy.commits at gmail.com Mon Mar 13 09:05:03 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 06:05:03 -0700 (PDT) Subject: [pypy-commit] cffi default: python 2.6 compat Message-ID: <58c698ff.039d190a.f8f14.3641@mx.google.com> Author: Armin Rigo Branch: Changeset: r2908:e4b182999315 Date: 2017-03-13 14:05 +0100 http://bitbucket.org/cffi/cffi/changeset/e4b182999315/ Log: python 2.6 compat diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py --- a/testing/cffi0/backend_tests.py +++ b/testing/cffi0/backend_tests.py @@ -1230,7 +1230,8 @@ def test_ffi_buffer_comparisons(self): ffi = FFI(backend=self.Backend()) ba = bytearray(range(100, 110)) - assert ba == memoryview(ba) # justification for the following + if sys.version_info >= (2, 7): + assert ba == memoryview(ba) # justification for the following a = ffi.new("uint8_t[]", list(ba)) c = ffi.new("uint8_t[]", [99] + list(ba)) try: From pypy.commits at gmail.com Mon Mar 13 09:07:42 2017 From: pypy.commits at gmail.com (tobweber) Date: Mon, 13 Mar 2017 06:07:42 -0700 (PDT) Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Correction to duration logging events Message-ID: <58c6999e.02a2190a.2625e.3339@mx.google.com> Author: Tobias Weber Branch: c8-overheads-instrumentation Changeset: r2028:b59eb6c25bca Date: 2017-03-13 11:54 +0100 http://bitbucket.org/pypy/stmgc/changeset/b59eb6c25bca/ Log: Correction to duration logging events diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -578,8 +578,7 @@ STM_DURATION_WRITE_GC_ONLY, STM_DURATION_WRITE_SLOWPATH, STM_DURATION_VALIDATION, - STM_DURATION_COMMIT_MINOR_GC, - STM_DURATION_COMMIT_ALL, + STM_DURATION_COMMIT_EXCEPT_GC, STM_DURATION_MINOR_GC, STM_DURATION_MAJOR_GC_LOG_ONLY, STM_DURATION_MAJOR_GC_FULL, @@ -587,26 +586,25 @@ _STM_EVENT_N }; -#define STM_EVENT_NAMES \ - "transaction start", \ - "transaction commit", \ - "transaction abort", \ - "contention write read", \ - "wait free segment", \ - "wait other inevitable", \ - "wait done", \ - "gc minor start", \ - "gc minor done", \ - "gc major start", \ - "gc major done", \ - /* names of duration events */ \ - "duration of minor gc due to write", \ - "duration of write slowpath", \ - "duration of validation", \ - "duration of minor gc due to commit", \ - "duration of commit except minor gc", \ - "duration of minor gc", \ - "duration of major gc doing log clean up only",\ +#define STM_EVENT_NAMES \ + "transaction start", \ + "transaction commit", \ + "transaction abort", \ + "contention write read", \ + "wait free segment", \ + "wait other inevitable", \ + "wait done", \ + "gc minor start", \ + "gc minor done", \ + "gc major start", \ + "gc major done", \ + /* names of duration events */ \ + "duration of gc due to write", \ + "duration of write slowpath", \ + "duration of validation", \ + "duration of commit except gc", \ + "duration of minor gc", \ + "duration of major gc doing log clean up only", \ "duration of full major gc" /* The markers pushed in the shadowstack are an odd number followed by a From pypy.commits at gmail.com Mon Mar 13 09:07:44 2017 From: pypy.commits at gmail.com (tobweber) Date: Mon, 13 Mar 2017 06:07:44 -0700 (PDT) Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Add pause function to timing utility Message-ID: <58c699a0.4395190a.8d951.3594@mx.google.com> Author: Tobias Weber Branch: c8-overheads-instrumentation Changeset: r2029:4bb06ec721cd Date: 2017-03-13 12:29 +0100 http://bitbucket.org/pypy/stmgc/changeset/4bb06ec721cd/ Log: Add pause function to timing utility diff --git a/c8/stm/timing.h b/c8/stm/timing.h --- a/c8/stm/timing.h +++ b/c8/stm/timing.h @@ -1,19 +1,21 @@ #include +#define continue_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &start); + /* Use raw monotonic time, i.e., solely based on local hardware (no NTP adjustments) as in prof.c to obtain values comparable with total program runtime. */ -#define start_timer() struct timespec start; \ - clock_gettime(CLOCK_MONOTONIC_RAW, &start); +#define start_timer() struct timespec start, stop, duration = { 0, 0 }; \ + continue_timer() -#define stop_timer() struct timespec stop; \ - clock_gettime(CLOCK_MONOTONIC_RAW, &stop); +/* Must use start_timer before using this macro. */ +#define get_duration() duration.tv_sec = \ + stop.tv_sec - start.tv_sec + duration.tv_sec; \ + duration.tv_nsec = \ + stop.tv_nsec - start.tv_nsec + duration.tv_nsec; -/* Must use start_timer and stop_timer before using this macro. */ -#define get_duration() struct timespec duration = { \ - stop.tv_sec - start.tv_sec, \ - stop.tv_nsec - start.tv_nsec \ - }; +#define pause_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &stop); \ + get_duration() #define stm_duration_payload(duration) \ stm_timing_event_payload_data_t stm_duration_data = \ @@ -23,10 +25,10 @@ #define publish_event(event) \ (timing_enabled() ? \ - stmcb_timing_event(STM_SEGMENT->running_thread, event, &stm_duration_payload) : \ + stmcb_timing_event( \ + STM_SEGMENT->running_thread, event, &stm_duration_payload) : \ (void)0); -#define stop_timer_and_publish(event) stop_timer() \ - get_duration() \ +#define stop_timer_and_publish(event) pause_timer() \ stm_duration_payload(duration) \ publish_event(event) From pypy.commits at gmail.com Mon Mar 13 09:07:47 2017 From: pypy.commits at gmail.com (tobweber) Date: Mon, 13 Mar 2017 06:07:47 -0700 (PDT) Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Instrument all contributors to transaction management overhead in an initial coarse grained manner Message-ID: <58c699a3.48572e0a.87d22.3d2f@mx.google.com> Author: Tobias Weber Branch: c8-overheads-instrumentation Changeset: r2030:8ef2322269a5 Date: 2017-03-13 14:02 +0100 http://bitbucket.org/pypy/stmgc/changeset/8ef2322269a5/ Log: Instrument all contributors to transaction management overhead in an initial coarse grained manner diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -156,6 +156,8 @@ static bool _stm_validate(void) { + start_timer(); + /* returns true if we reached a valid state, or false if we need to abort now */ dprintf(("_stm_validate() at cl=%p, rev=%lu\n", STM_PSEGMENT->last_commit_log_entry, @@ -338,6 +340,8 @@ release_privatization_lock(my_segnum); } + stop_timer_and_publish(STM_DURATION_VALIDATION); + return !needs_abort; } @@ -817,6 +821,8 @@ static void write_slowpath_common(object_t *obj, bool mark_card) { + start_timer(); + assert(_seems_to_be_running_transaction()); assert(!_is_in_nursery(obj)); assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); @@ -826,6 +832,8 @@ part again: */ assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); write_slowpath_overflow_obj(obj, mark_card); + + stop_timer_and_publish(STM_DURATION_WRITE_GC_ONLY); return; } @@ -893,6 +901,8 @@ } DEBUG_EXPECT_SEGFAULT(true); + + stop_timer_and_publish(STM_DURATION_WRITE_SLOWPATH); } @@ -959,9 +969,7 @@ __attribute__((flatten)) void _stm_write_slowpath(object_t *obj) { - start_timer() write_slowpath_common(obj, /* mark_card */ false); - stop_timer_and_publish(STM_DURATION_WRITE_SLOWPATH) } @@ -1208,6 +1216,8 @@ static void _core_commit_transaction(bool external) { + start_timer(); + exec_local_finalizers(); assert(!_has_mutex()); @@ -1226,11 +1236,17 @@ assert(STM_SEGMENT->running_thread->wait_event_emitted == 0); dprintf(("> stm_commit_transaction(external=%d)\n", (int)external)); + + pause_timer(); minor_collection(/*commit=*/ true, external); + continue_timer(); + if (!external && is_major_collection_requested()) { s_mutex_lock(); if (is_major_collection_requested()) { /* if still true */ + pause_timer(); major_collection_with_mutex(); + continue_timer(); } s_mutex_unlock(); } @@ -1246,7 +1262,10 @@ stm_validate() at the start of a new transaction is happy even if there is an inevitable tx running) */ bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE; + + pause_timer(); _validate_and_add_to_commit_log(); + continue_timer(); if (external) { /* from this point on, unlink the original 'stm_thread_local_t *' @@ -1295,6 +1314,8 @@ s_mutex_unlock(); + stop_timer_and_publish(STM_DURATION_COMMIT_EXCEPT_GC); + /* between transactions, call finalizers. this will execute a transaction itself */ if (tl != NULL) diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -750,6 +750,8 @@ /* first, force a minor collection in each of the other segments */ major_do_validation_and_minor_collections(); + start_timer(); + dprintf((" | used before collection: %ld\n", (long)pages_ctl.total_allocated)); dprintf((" | commit log entries before: %ld\n", @@ -782,6 +784,8 @@ if (must_abort()) abort_with_mutex(); + stop_timer_and_publish(STM_DURATION_MAJOR_GC_LOG_ONLY); + return; #endif } @@ -838,4 +842,6 @@ dprintf(("must abort?:%d\n", (int)must_abort())); if (must_abort()) abort_with_mutex(); + + stop_timer_and_publish(STM_DURATION_MAJOR_GC_FULL); } diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -534,6 +534,8 @@ static void _do_minor_collection(bool commit) { + start_timer(); + dprintf(("minor_collection commit=%d\n", (int)commit)); assert(!STM_SEGMENT->no_safe_point_here); @@ -578,6 +580,8 @@ throw_away_nursery(get_priv_segment(STM_SEGMENT->segment_num)); assert(MINOR_NOTHING_TO_DO(STM_PSEGMENT)); + + stop_timer_and_publish(STM_DURATION_MINOR_GC); } static void minor_collection(bool commit, bool external) From pypy.commits at gmail.com Mon Mar 13 09:38:13 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 06:38:13 -0700 (PDT) Subject: [pypy-commit] cffi default: skip on windows Message-ID: <58c6a0c5.4395190a.8d951.381c@mx.google.com> Author: Armin Rigo Branch: Changeset: r2909:3bea1a98c958 Date: 2017-03-13 14:37 +0100 http://bitbucket.org/cffi/cffi/changeset/3bea1a98c958/ Log: skip on windows diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -2106,6 +2106,8 @@ "set_source() and not taking a final '...' argument)") def test_call_with_zero_length_field(): + if sys.platform == 'win32': + py.test.skip("zero-length field not supported by MSVC") ffi = FFI() ffi.cdef(""" struct foo { int a; int x[0]; }; From pypy.commits at gmail.com Mon Mar 13 09:40:46 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 06:40:46 -0700 (PDT) Subject: [pypy-commit] pypy default: import cffi/3bea1a98c958 Message-ID: <58c6a15e.12582e0a.ae8e6.2dd9@mx.google.com> Author: Armin Rigo Branch: Changeset: r90648:c98863bd0296 Date: 2017-03-13 14:40 +0100 http://bitbucket.org/pypy/pypy/changeset/c98863bd0296/ Log: import cffi/3bea1a98c958 diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1231,7 +1231,8 @@ def test_ffi_buffer_comparisons(self): ffi = FFI(backend=self.Backend()) ba = bytearray(range(100, 110)) - assert ba == memoryview(ba) # justification for the following + if sys.version_info >= (2, 7): + assert ba == memoryview(ba) # justification for the following a = ffi.new("uint8_t[]", list(ba)) c = ffi.new("uint8_t[]", [99] + list(ba)) try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -2040,7 +2040,7 @@ struct foo s = { 40, 200 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().x == 200 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2069,7 +2069,7 @@ s.b = 200; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().b == 200 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2095,7 +2095,7 @@ struct foo s = { 11 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().x == 11 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2107,6 +2107,8 @@ "set_source() and not taking a final '...' argument)") def test_call_with_zero_length_field(): + if sys.platform == 'win32': + py.test.skip("zero-length field not supported by MSVC") ffi = FFI() ffi.cdef(""" struct foo { int a; int x[0]; }; @@ -2119,7 +2121,7 @@ struct foo s = { 42 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().a == 42 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2143,7 +2145,7 @@ union foo s = { 42 }; return s; } - union foo g(int a, ...) { } + union foo g(int a, ...) { return f(); } """) assert lib.f().a == 42 e = py.test.raises(NotImplementedError, lib.g, 0) From pypy.commits at gmail.com Mon Mar 13 09:47:08 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 06:47:08 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: OS/X translation fix (with jit) Message-ID: <58c6a2dc.124b2e0a.8f89d.3de6@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90649:45fabe7ba733 Date: 2017-03-13 14:46 +0100 http://bitbucket.org/pypy/pypy/changeset/45fabe7ba733/ Log: OS/X translation fix (with jit) diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -48,9 +48,10 @@ force_ignore=False)[0] elif _MACOSX: bytes = space.bytes_w(w_string) - uni = runicode.str_decode_utf_8( - bytes, len(bytes), 'surrogateescape', - errorhandler=state.decode_error_handler)[0] + uni = runicode.str_decode_utf_8_impl( + bytes, len(bytes), 'surrogateescape', final=True, + errorhandler=state.decode_error_handler, + allow_surrogates=False)[0] elif space.sys.filesystemencoding is None or state.codec_need_encodings: # bootstrap check: if the filesystemencoding isn't initialized # or the filesystem codec is implemented in Python we cannot @@ -77,9 +78,10 @@ force_replace=False) elif _MACOSX: uni = space.unicode_w(w_uni) - bytes = runicode.unicode_encode_utf_8( + bytes = runicode.unicode_encode_utf_8_impl( uni, len(uni), 'surrogateescape', - errorhandler=state.encode_error_handler) + errorhandler=state.encode_error_handler, + allow_surrogates=False) elif space.sys.filesystemencoding is None or state.codec_need_encodings: # bootstrap check: if the filesystemencoding isn't initialized # or the filesystem codec is implemented in Python we cannot From pypy.commits at gmail.com Mon Mar 13 12:04:49 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 09:04:49 -0700 (PDT) Subject: [pypy-commit] pypy default: include for InterlockedCompareExchange Message-ID: <58c6c321.1c10190a.8b50d.3c4b@mx.google.com> Author: Armin Rigo Branch: Changeset: r90650:b28294dd298f Date: 2017-03-13 16:56 +0100 http://bitbucket.org/pypy/pypy/changeset/b28294dd298f/ Log: include for InterlockedCompareExchange diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c --- a/pypy/module/cpyext/src/pymem.c +++ b/pypy/module/cpyext/src/pymem.c @@ -1,5 +1,10 @@ #include +#ifdef _WIN32 +# include +#endif + + void * PyMem_Malloc(size_t n) { return malloc((n) ? (n) : 1); From pypy.commits at gmail.com Mon Mar 13 12:30:29 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 09:30:29 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Merged in nanjekye/pypy/jumbo (pull request #528) Message-ID: <58c6c925.6718190a.eed30.3be8@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90661:f90ac97e3a04 Date: 2017-03-13 16:29 +0000 http://bitbucket.org/pypy/pypy/changeset/f90ac97e3a04/ Log: Merged in nanjekye/pypy/jumbo (pull request #528) Pread/pwrite attribute on py3.5 diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -217,6 +217,11 @@ if sys.platform.startswith('linux'): #hasattr(rposix, 'sendfile'): interpleveldefs['sendfile'] = 'interp_posix.sendfile' + if hasattr(rposix, 'pread'): + interpleveldefs['pread'] = 'interp_posix.pread' + if hasattr(rposix, 'pwrite'): + interpleveldefs['pwrite'] = 'interp_posix.pwrite' + for _name in ["O_CLOEXEC"]: if getattr(rposix, _name) is not None: interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name) diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -376,6 +376,31 @@ except OSError as e: wrap_oserror(space, e, eintr_retry=True) + at unwrap_spec(fd=c_int, length=int, offset=r_longlong) +def pread(space, fd, length, offset): + """Read a string to a file descriptor at a given offset. + """ + while True: + try: + s = rposix.pread(fd, length, offset) + except OSError as e: + wrap_oserror(space, e, eintr_retry=True) + else: + return space.newbytes(s) + + at unwrap_spec(fd=c_int, offset=r_longlong) +def pwrite(space, fd, w_data, offset): + """Write a string to a file descriptor at a given offset. + """ + data = space.getarg_w('y*', w_data) + while True: + try: + res = rposix.pwrite(fd, data.as_str(), offset) + except OSError as e: + wrap_oserror(space, e, eintr_retry=True) + else: + return space.newint(res) + # ____________________________________________________________ STAT_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS)) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -836,6 +836,30 @@ os.chdir(localdir) raises(ValueError, os.fchdir, -1) + if hasattr(rposix, 'pread'): + def test_os_pread(self): + os = self.posix + fd = os.open(self.path2 + 'test_os_pread', os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, 0) + assert os.pread(fd, 2, 1) == b'es' + assert os.read(fd, 2) == b'te' + finally: + os.close(fd) + + if hasattr(rposix, 'pwrite'): + def test_os_pwrite(self): + os = self.posix + fd = os.open(self.path2 + 'test_os_pwrite', os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, 0) + os.pwrite(fd, b'xx', 1) + assert os.read(fd, 4) == b'txxt' + finally: + os.close(fd) + def test_largefile(self): os = self.posix fd = os.open(self.path2 + 'test_largefile', diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -464,6 +464,29 @@ how = SEEK_END return handle_posix_error('lseek', c_lseek(fd, pos, how)) +c_pread = external('pread', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T , OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) +c_pwrite = external('pwrite', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) + + at enforceargs(int, int, None) +def pread(fd, count, offset): + if count < 0: + raise OSError(errno.EINVAL, None) + validate_fd(fd) + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) + + at enforceargs(int, None, None) +def pwrite(fd, data, offset): + count = len(data) + validate_fd(fd) + with rffi.scoped_nonmovingbuffer(data) as buf: + return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) + c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) c_fsync = external('fsync' if not _WIN32 else '_commit', [rffi.INT], rffi.INT, diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -676,7 +676,7 @@ prio = rposix.getpriority(rposix.PRIO_PROCESS, 0) rposix.setpriority(rposix.PRIO_PROCESS, 0, prio) py.test.raises(OSError, rposix.getpriority, rposix.PRIO_PGRP, 123456789) - + if sys.platform != 'win32': def test_sendfile(): from rpython.rlib import rsocket @@ -723,3 +723,30 @@ os.close(fd) s2.close() s1.close() + + at rposix_requires('pread') +def test_pread(): + fname = str(udir.join('os_test.txt')) + fd = os.open(fname, os.O_RDWR | os.O_CREAT) + try: + assert fd >= 0 + os.write(fd, b'Hello world') + os.lseek(fd, 0, 0) + assert rposix.pread(fd, 2, 1) == b'el' + finally: + os.close(fd) + py.test.raises(OSError, rposix.pread, fd, 2, 1) + + at rposix_requires('pwrite') +def test_pwrite(): + fname = str(udir.join('os_test.txt')) + fd = os.open(fname, os.O_RDWR | os.O_CREAT, 0777) + try: + assert fd >= 0 + os.write(fd, b'Hello world') + os.lseek(fd, 0, 0) + rposix.pwrite(fd, b'ea', 1) + assert os.read(fd, 4) == b'Heal' + finally: + os.close(fd) + py.test.raises(OSError, rposix.pwrite, fd, b'ea', 1) From pypy.commits at gmail.com Mon Mar 13 12:30:42 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:42 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: pwrite fix Message-ID: <58c6c932.4121190a.db54f.40a5@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90653:14dc42986418 Date: 2017-03-02 19:36 +0300 http://bitbucket.org/pypy/pypy/changeset/14dc42986418/ Log: pwrite fix diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -392,12 +392,13 @@ """Write a string to a file descriptor at a given offset. """ data = space.getarg_w('y*', w_data) - try: - res = os.write(fd, data.as_str()) - except OSError as e: - raise wrap_oserror(space, e, eintr_retry=True) - else: - return space.newint(res) + while True: + try: + res = rposix.pwrite(fd, data.as_str(), offset) + except OSError as e: + wrap_oserror(space, e, eintr_retry=True) + else: + return space.newint(res) # ____________________________________________________________ From pypy.commits at gmail.com Mon Mar 13 12:30:38 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:38 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: pread/pwrite Message-ID: <58c6c92e.5ed7190a.1d79f.45b5@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90651:9d0d9dcf4948 Date: 2017-03-02 17:37 +0300 http://bitbucket.org/pypy/pypy/changeset/9d0d9dcf4948/ Log: pread/pwrite diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -213,6 +213,11 @@ assert getattr(rposix, _name) is not None, "missing %r" % (_name,) interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name) + if hasattr(rposix, 'pread'): + interpleveldefs['pread'] = 'interp_posix.pread' + if hasattr(rposix, 'pwrite'): + interpleveldefs['pwrite'] = 'interp_posix.pwrite' + for _name in ["O_CLOEXEC"]: if getattr(rposix, _name) is not None: interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name) diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -376,6 +376,29 @@ except OSError as e: wrap_oserror(space, e, eintr_retry=True) + at unwrap_spec(fd=c_int, length=int, offset=int) +def pread(space, fd, length, offset): + """Read a string to a file descriptor at a given offset. + """ + try: + s = rposix.pread(fd, length, offset) + except OSError as e: + raise wrap_oserror(space, e, eintr_retry=True) + else: + return space.newbytes(s) + + at unwrap_spec(fd=c_int, offset=int) +def pwrite(space, fd, w_data, offset): + """Write a string to a file descriptor at a given offset. + """ + data = space.getarg_w('y*', w_data) + try: + res = os.write(fd, data.as_str()) + except OSError as e: + raise wrap_oserror(space, e, eintr_retry=True) + else: + return space.newint(res) + # ____________________________________________________________ STAT_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS)) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -836,6 +836,30 @@ os.chdir(localdir) raises(ValueError, os.fchdir, -1) + if hasattr(rposix, 'pread'): + def test_os_pread(self): + os = self.posix + fd = os.open(self.path2 + 'test_os_pread', os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, 0) + assert os.pread(fd, 2, 1) == b'es' + assert os.read(fd, 2) == b'te' + finally: + os.close(fd) + + if hasattr(rposix, 'pwrite'): + def test_os_pwrite(self): + os = self.posix + fd = os.open(self.path2 + 'test_os_pwrite', os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, 0) + os.pwrite(fd, b'xx', 1) + assert os.read(fd, 4) == b'txxt' + finally: + os.close(fd) + def test_largefile(self): os = self.posix fd = os.open(self.path2 + 'test_largefile', diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -254,6 +254,7 @@ PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP') PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER') O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK') + OFF_T = rffi_platform.SimpleType('off_t') OFF_T_SIZE = rffi_platform.SizeOf('off_t') HAVE_UTIMES = rffi_platform.Has('utimes') @@ -463,6 +464,23 @@ how = SEEK_END return handle_posix_error('lseek', c_lseek(fd, pos, how)) +c_pread = external('pread', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T , OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) +c_pwrite = external('pwrite', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) + + at replace_os_function('pread') + at enforceargs(int, int, None) +def pread(fd, count, offset): + if count < 0: + raise OSError(errno.EINVAL, None) + validate_fd(fd) + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) + c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) c_fsync = external('fsync' if not _WIN32 else '_commit', [rffi.INT], rffi.INT, From pypy.commits at gmail.com Mon Mar 13 12:30:45 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:45 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: removing extra space Message-ID: <58c6c935.cf4d2e0a.6f66f.3582@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90654:f2823ce45778 Date: 2017-03-02 19:45 +0300 http://bitbucket.org/pypy/pypy/changeset/f2823ce45778/ Log: removing extra space diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -489,7 +489,6 @@ with rffi.scoped_nonmovingbuffer(data) as buf: return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) - c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) c_fsync = external('fsync' if not _WIN32 else '_commit', [rffi.INT], rffi.INT, From pypy.commits at gmail.com Mon Mar 13 12:30:40 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:40 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: pwrite Message-ID: <58c6c930.124b2e0a.8f89d.4622@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90652:41b02003b58e Date: 2017-03-02 18:26 +0300 http://bitbucket.org/pypy/pypy/changeset/41b02003b58e/ Log: pwrite diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -480,6 +480,15 @@ with rffi.scoped_alloc_buffer(count) as buf: void_buf = rffi.cast(rffi.VOIDP, buf.raw) return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) + + at replace_os_function('pwrite') + at enforceargs(int, None, None) +def pwrite(fd, data, offset): + count = len(data) + validate_fd(fd) + with rffi.scoped_nonmovingbuffer(data) as buf: + return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) + c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) From pypy.commits at gmail.com Mon Mar 13 12:30:49 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:49 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: rposix tests for pread/pwrite Message-ID: <58c6c939.4b1b190a.ceb96.3e0d@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90656:459d7473aca8 Date: 2017-03-08 12:37 +0300 http://bitbucket.org/pypy/pypy/changeset/459d7473aca8/ Log: rposix tests for pread/pwrite diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -384,7 +384,7 @@ try: s = rposix.pread(fd, length, offset) except OSError as e: - raise wrap_oserror(space, e, eintr_retry=True) + raise wrap_oserror(space, e, eintr_retry=False) else: return space.newbytes(s) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -471,7 +471,6 @@ [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) - at replace_os_function('pread') @enforceargs(int, int, None) def pread(fd, count, offset): if count < 0: @@ -481,7 +480,6 @@ void_buf = rffi.cast(rffi.VOIDP, buf.raw) return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) - at replace_os_function('pwrite') @enforceargs(int, None, None) def pwrite(fd, data, offset): count = len(data) diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -676,3 +676,30 @@ prio = rposix.getpriority(rposix.PRIO_PROCESS, 0) rposix.setpriority(rposix.PRIO_PROCESS, 0, prio) py.test.raises(OSError, rposix.getpriority, rposix.PRIO_PGRP, 123456789) + + at rposix_requires('pread') +def test_pread(): + fname = str(udir.join('os_test.txt')) + fd = os.open(fname, os.O_RDWR | os.O_CREAT) + try: + assert fd >= 0 + os.write(fd, b'Hello world') + os.lseek(fd, 0, 0) + assert rposix.pread(fd, 2, 1) == b'el' + finally: + os.close(fd) + py.test.raises(OSError, rposix.pread, fd, 2, 1) + + at rposix_requires('pwrite') +def test_pwrite(): + fname = str(udir.join('os_test.txt')) + fd = os.open(fname, os.O_RDWR | os.O_CREAT, 0777) + try: + assert fd >= 0 + os.write(fd, b'Hello world') + os.lseek(fd, 0, 0) + rposix.pwrite(fd, b'ea', 1) + assert os.read(fd, 4) == b'Heal' + finally: + os.close(fd) + py.test.raises(OSError, rposix.pwrite, fd, b'ea', 1) From pypy.commits at gmail.com Mon Mar 13 12:30:47 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:47 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: while while retrying for pread Message-ID: <58c6c937.13542e0a.25fb6.3dfb@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90655:ec48d98c34a6 Date: 2017-03-03 16:07 +0300 http://bitbucket.org/pypy/pypy/changeset/ec48d98c34a6/ Log: while while retrying for pread diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -380,12 +380,13 @@ def pread(space, fd, length, offset): """Read a string to a file descriptor at a given offset. """ - try: - s = rposix.pread(fd, length, offset) - except OSError as e: - raise wrap_oserror(space, e, eintr_retry=True) - else: - return space.newbytes(s) + while True: + try: + s = rposix.pread(fd, length, offset) + except OSError as e: + raise wrap_oserror(space, e, eintr_retry=True) + else: + return space.newbytes(s) @unwrap_spec(fd=c_int, offset=int) def pwrite(space, fd, w_data, offset): From pypy.commits at gmail.com Mon Mar 13 12:30:53 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:53 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: merge conflict Message-ID: <58c6c93d.06d2190a.c9b49.392a@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90657:1e6aa8fc49eb Date: 2017-03-08 15:07 +0300 http://bitbucket.org/pypy/pypy/changeset/1e6aa8fc49eb/ Log: merge conflict diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py --- a/lib-python/3/datetime.py +++ b/lib-python/3/datetime.py @@ -1000,6 +1000,11 @@ else: return (self.__class__, args, state) + # PyPy: added for compatibility with the _datetime module + # issue #2489 + def __new__(cls, *args, **kwds): + return super(tzinfo, cls).__new__(cls) + _tzinfo_class = tzinfo class time: diff --git a/lib-python/3/test/test_functools.py b/lib-python/3/test/test_functools.py --- a/lib-python/3/test/test_functools.py +++ b/lib-python/3/test/test_functools.py @@ -233,21 +233,21 @@ f = self.partial(capture) f.__setstate__((f, (), {}, {})) try: - self.assertEqual(repr(f), '%s(%s(...))' % (name, name)) + self.assertEqual(repr(f), '%s(...)' % (name)) finally: f.__setstate__((capture, (), {}, {})) f = self.partial(capture) f.__setstate__((capture, (f,), {}, {})) try: - self.assertEqual(repr(f), '%s(%r, %s(...))' % (name, capture, name)) + self.assertEqual(repr(f), '%s(%r, ...)' % (name, capture)) finally: f.__setstate__((capture, (), {}, {})) f = self.partial(capture) f.__setstate__((capture, (), {'a': f}, {})) try: - self.assertEqual(repr(f), '%s(%r, a=%s(...))' % (name, capture, name)) + self.assertEqual(repr(f), '%s(%r, a=...)' % (name, capture)) finally: f.__setstate__((capture, (), {}, {})) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -176,3 +176,20 @@ .. branch: py3.5-text-utf8 space.text_w now encodes to utf-8 not preserving surrogates. + +.. branch: fix-cpyext-releasebuffer + +Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews +keeping objects alive forever (missing decref), and make sure that +bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. + +.. branch: fix-global + +Fix bug (bad reported info) when asked to translate SyntaxWarning to +SyntaxError. + +.. branch: optinfo-into-bridges-3 + +Improve the optimization of branchy Python code by retaining more +information across failing guards. This is done by appending some +carefully encoded extra information into the resume code. diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py --- a/pypy/interpreter/error.py +++ b/pypy/interpreter/error.py @@ -672,15 +672,17 @@ w_filename2=w_filename2, eintr_retry=eintr_retry) -def exception_from_saved_errno(space, w_type): - from rpython.rlib.rposix import get_saved_errno - - errno = get_saved_errno() +def exception_from_errno(space, w_type, errno): msg = strerror(errno) w_error = space.call_function(w_type, space.newint(errno), space.newunicode(msg)) return OperationError(w_type, w_error) +def exception_from_saved_errno(space, w_type): + from rpython.rlib.rposix import get_saved_errno + errno = get_saved_errno() + return exception_from_errno(space, w_type, errno) + def new_exception_class(space, name, w_bases=None, w_dict=None): """Create a new exception type. @param name: the name of the type. diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -5,6 +5,8 @@ TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w) from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from rpython.rlib.buffer import Buffer, SubBuffer +from rpython.rlib.rgc import ( + nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr) from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import r_longlong, intmask from rpython.rlib import rposix @@ -13,7 +15,6 @@ W_IOBase, DEFAULT_BUFFER_SIZE, convert_size, trap_eintr, check_readable_w, check_writable_w, check_seekable_w) from rpython.rlib import rthread -from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list STATE_ZERO, STATE_OK, STATE_DETACHED = range(3) @@ -159,7 +160,7 @@ def __init__(self, n): self.length = n - self.buf = ['\0'] * n + self.buf = resizable_list_supporting_raw_ptr(['\0'] * n) self.readonly = False def getlength(self): diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -2,10 +2,13 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import ( OperationError, oefmt, wrap_oserror, wrap_oserror2) +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_longlong +from rpython.rlib.rposix import c_read, get_saved_errno from rpython.rlib.rstring import StringBuilder from rpython.rlib import rposix from rpython.rlib.rposix_stat import STAT_FIELD_TYPES +from rpython.rtyper.lltypesystem import lltype, rffi from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL import sys, os, stat, errno from pypy.module._io.interp_iobase import ( @@ -126,6 +129,7 @@ return currentsize + BIGCHUNK return currentsize + SMALLCHUNK + class W_FileIO(W_RawIOBase): def __init__(self, space): W_RawIOBase.__init__(self, space) @@ -447,18 +451,43 @@ self._check_readable(space) rwbuffer = space.getarg_w('w*', w_buffer) length = rwbuffer.getlength() - while True: + + target_address = lltype.nullptr(rffi.CCHARP.TO) + if length > 64: try: - buf = os.read(self.fd, length) - break - except OSError as e: - if e.errno == errno.EAGAIN: - return space.w_None - wrap_oserror(space, e, - exception_name='w_IOError', - eintr_retry=True) - rwbuffer.setslice(0, buf) - return space.newint(len(buf)) + target_address = rwbuffer.get_raw_address() + except ValueError: + pass + + if not target_address: + # unoptimized case + while True: + try: + buf = os.read(self.fd, length) + break + except OSError as e: + if e.errno == errno.EAGAIN: + return space.w_None + wrap_oserror(space, e, exception_name='w_IOError', + eintr_retry=True) + rwbuffer.setslice(0, buf) + return space.newint(len(buf)) + else: + # optimized case: reading more than 64 bytes into a rwbuffer + # with a valid raw address + while True: + got = c_read(self.fd, target_address, length) + keepalive_until_here(rwbuffer) + got = rffi.cast(lltype.Signed, got) + if got >= 0: + return space.newint(got) + else: + err = get_saved_errno() + if err == errno.EAGAIN: + return space.w_None + e = OSError(err, "read failed") + wrap_oserror(space, e, exception_name='w_IOError', + eintr_retry=True) def readall_w(self, space): self._check_closed(space) diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -12,6 +12,9 @@ tmpfile = udir.join('tmpfile') tmpfile.write("a\nb\nc", mode='wb') cls.w_tmpfile = cls.space.wrap(str(tmpfile)) + bigtmpfile = udir.join('bigtmpfile') + bigtmpfile.write("a\nb\nc" * 20, mode='wb') + cls.w_bigtmpfile = cls.space.wrap(str(bigtmpfile)) def test_simple_read(self): import _io @@ -287,7 +290,21 @@ raises(_io.UnsupportedOperation, bufio.tell) class AppTestBufferedReaderWithThreads(AppTestBufferedReader): - spaceconfig = dict(usemodules=['_io', 'thread']) + spaceconfig = dict(usemodules=['_io', 'thread', 'time']) + + def test_readinto_small_parts(self): + import _io, os, _thread, time + read_fd, write_fd = os.pipe() + raw = _io.FileIO(read_fd) + f = _io.BufferedReader(raw) + a = bytearray(b'x' * 10) + os.write(write_fd, b"abcde") + def write_more(): + time.sleep(0.5) + os.write(write_fd, b"fghij") + _thread.start_new_thread(write_more, ()) + assert f.readinto(a) == 10 + assert a == b'abcdefghij' class AppTestBufferedWriter: diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -1,4 +1,5 @@ # encoding: utf-8 +from pypy.interpreter.gateway import interp2app from rpython.tool.udir import udir import os @@ -6,14 +7,18 @@ class AppTestFileIO: spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else [])) - def setup_class(cls): + def setup_method(self, meth): tmpfile = udir.join('tmpfile') tmpfile.write("a\nb\nc", mode='wb') - cls.w_tmpfile = cls.space.wrap(str(tmpfile)) - cls.w_tmpdir = cls.space.wrap(str(udir)) - cls.w_posix = cls.space.appexec([], """(): + self.w_tmpfile = self.space.wrap(str(tmpfile)) + self.w_tmpdir = self.space.wrap(str(udir)) + self.w_posix = self.space.appexec([], """(): import %s as m; return m""" % os.name) + if meth == self.test_readinto_optimized: + bigfile = udir.join('bigfile') + bigfile.write('a' * 1000, mode='wb') + self.w_bigfile = self.space.wrap(self.space.wrap(str(bigfile))) def test_constructor(self): import _io @@ -151,13 +156,10 @@ import _io a = bytearray(b'x' * 10) f = _io.FileIO(self.tmpfile, 'r+') - f.seek(5) - f.write(b'\x00' * 5) - f.seek(0) - assert f.readinto(a) == 10 + assert f.readinto(a) == 5 f.seek(0) m = memoryview(bytearray(b"helloworld")) - assert f.readinto(m) == 10 + assert f.readinto(m) == 5 # exc = raises(TypeError, f.readinto, u"hello") msg = str(exc.value) @@ -170,7 +172,7 @@ assert " read-write b" in msg and msg.endswith(", not memoryview") # f.close() - assert a == b'a\nb\nc\0\0\0\0\0' + assert a == b'a\nb\ncxxxxx' # a = bytearray(b'x' * 10) f = _io.FileIO(self.tmpfile, 'r+') @@ -179,6 +181,13 @@ f.close() assert a == b'a\nbxxxxxxx' + def test_readinto_optimized(self): + import _io + a = bytearray(b'x' * 1024) + f = _io.FileIO(self.bigfile, 'r+') + assert f.readinto(a) == 1000 + assert a == b'a' * 1000 + b'x' * 24 + def test_nonblocking_read(self): try: import os, fcntl @@ -194,6 +203,8 @@ assert f.read(10) is None a = bytearray(b'x' * 10) assert f.readinto(a) is None + a2 = bytearray(b'x' * 1024) + assert f.readinto(a2) is None def test_repr(self): import _io diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -565,6 +565,8 @@ raise oefmt(space.w_ValueError, "negative buffersize in recv_into") if nbytes == 0: nbytes = lgt + if lgt < nbytes: + raise oefmt(space.w_ValueError, "buffer too small for requested bytes") while True: try: nbytes_read = self.sock.recvinto(rwbuffer, nbytes, flags) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -296,7 +296,8 @@ class AppTestSocket: - spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct', 'select']) + spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct', 'select', + 'unicodedata']) def setup_class(cls): cls.space = space @@ -667,8 +668,6 @@ def test_hostname_unicode(self): import _socket domain = u"испытание.pythontest.net" - # XXX figure out why the idna encoding is sometimes missing in - # tests, notably if we run all tests instead of just this one _socket.gethostbyname(domain) _socket.gethostbyname_ex(domain) _socket.getaddrinfo(domain, 0, _socket.AF_UNSPEC, _socket.SOCK_STREAM) diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py --- a/pypy/module/_warnings/interp_warnings.py +++ b/pypy/module/_warnings/interp_warnings.py @@ -226,14 +226,16 @@ return False def normalize_module(space, w_filename): - filename = space.text_w(w_filename) + # XXX: could be more efficient (doesn't necessarily need + # fsencoding/redecoding) + filename = space.fsencode_w(w_filename) if len(filename) == 0: return space.newtext("") if filename.endswith(".py"): n = len(filename) - 3 assert n >= 0 filename = filename[:n] - return space.newtext(filename) + return space.newfilename(filename) return w_filename def show_warning(space, w_filename, lineno, w_text, w_category, diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -355,7 +355,7 @@ return rffi.cast(rffi.UCHAR, space.c_uint_w(call_capi(space, 'call_b', args))) def c_call_c(space, cppmethod, cppobject, nargs, cargs): args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)] - return rffi.cast(rffi.CHAR, space.text_w(call_capi(space, 'call_c', args))[0]) + return rffi.cast(rffi.CHAR, space.bytes_w(call_capi(space, 'call_c', args))[0]) def c_call_h(space, cppmethod, cppobject, nargs, cargs): args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)] return rffi.cast(rffi.SHORT, space.int_w(call_capi(space, 'call_h', args))) diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py --- a/pypy/module/cppyy/executor.py +++ b/pypy/module/cppyy/executor.py @@ -132,7 +132,11 @@ if ccpresult == rffi.cast(rffi.CCHARP, 0): return space.newbytes("") result = rffi.charp2str(ccpresult) # TODO: make it a choice to free - return space.newbytes(result) + # debatable whether this should be newtext or newbytes; they are bytes + # but will be more likely used as text after binding ... probably need + # to make this configurable on a per-function bases (same as the old + # char* v.s. byte* problem) + return space.newtext(result) class ConstructorExecutor(FunctionExecutor): diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py --- a/pypy/module/cppyy/ffitypes.py +++ b/pypy/module/cppyy/ffitypes.py @@ -81,13 +81,15 @@ raise oefmt(space.w_ValueError, "char arg not in range(256)") value = rffi.cast(rffi.CHAR, space.c_int_w(w_value)) + elif space.isinstance_w(w_value, space.w_text): + value = space.text_w(w_value) else: value = space.bytes_w(w_value) if len(value) != 1: raise oefmt(space.w_ValueError, "char expected, got string of size %d", len(value)) - return value[0] # turn it into a "char" to the annotator + return rffi.cast(rffi.CHAR, value[0]) # help the annotator def cffi_type(self, space): state = space.fromcache(State) diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py --- a/pypy/module/cppyy/test/test_datatypes.py +++ b/pypy/module/cppyy/test/test_datatypes.py @@ -38,9 +38,9 @@ assert not c.get_bool(); assert not c.get_bool_cr(); assert not c.get_bool_r() # reading char types - assert c.m_char == 'a' - assert c.m_schar == 'b' - assert c.m_uchar == 'c' + assert c.m_char == b'a' + assert c.m_schar == b'b' + assert c.m_uchar == b'c' # reading integer types assert c.m_short == -11; assert c.get_short_cr() == -11; assert c.get_short_r() == -11 @@ -133,18 +133,20 @@ raises(ValueError, 'c.set_bool(10)') # char types through functions - c.set_char('c'); assert c.get_char() == 'c' - c.set_uchar('e'); assert c.get_uchar() == 'e' + c.set_char('c'); assert c.get_char() == b'c' + c.set_char(b'b'); assert c.get_char() == b'b' + c.set_uchar('e'); assert c.get_uchar() == b'e' + c.set_uchar(b'd'); assert c.get_uchar() == b'd' # char types through data members - c.m_char = 'b'; assert c.get_char() == 'b' - c.m_char = 40; assert c.get_char() == chr(40) - c.set_char('c'); assert c.m_char == 'c' - c.set_char(41); assert c.m_char == chr(41) - c.m_uchar = 'd'; assert c.get_uchar() == 'd' - c.m_uchar = 42; assert c.get_uchar() == chr(42) - c.set_uchar('e'); assert c.m_uchar == 'e' - c.set_uchar(43); assert c.m_uchar == chr(43) + c.m_char = 'b'; assert c.get_char() == b'b' + c.m_char = 40; assert c.get_char() == str.encode(chr(40)) + c.set_char('c'); assert c.m_char == b'c' + c.set_char(41); assert c.m_char == str.encode(chr(41)) + c.m_uchar = 'd'; assert c.get_uchar() == b'd' + c.m_uchar = 42; assert c.get_uchar() == str.encode(chr(42)) + c.set_uchar('e'); assert c.m_uchar == b'e' + c.set_uchar(43); assert c.m_uchar == str.encode(chr(43)) raises(ValueError, 'c.set_char("string")') raises(ValueError, 'c.set_char(500)') @@ -246,10 +248,10 @@ assert isinstance(c, CppyyTestData) # char types - assert CppyyTestData.s_char == 'c' - assert c.s_char == 'c' - assert c.s_uchar == 'u' - assert CppyyTestData.s_uchar == 'u' + assert CppyyTestData.s_char == b'c' + assert c.s_char == b'c' + assert c.s_uchar == b'u' + assert CppyyTestData.s_uchar == b'u' # integer types assert CppyyTestData.s_short == -101 @@ -259,15 +261,15 @@ assert CppyyTestData.s_int == -202 assert c.s_int == -202 assert c.s_uint == 202 - assert CppyyTestData.s_uint == 202 - assert CppyyTestData.s_long == -303 + assert CppyyTestData.s_uint == 202 + assert CppyyTestData.s_long == -303 assert c.s_long == -303 assert c.s_ulong == 303 - assert CppyyTestData.s_ulong == 303 - assert CppyyTestData.s_llong == -404 + assert CppyyTestData.s_ulong == 303 + assert CppyyTestData.s_llong == -404 assert c.s_llong == -404 - assert c.s_ullong == 505 - assert CppyyTestData.s_ullong == 505 + assert c.s_ullong == 404 + assert CppyyTestData.s_ullong == 404 # floating point types assert round(CppyyTestData.s_float + 606., 5) == 0 @@ -287,57 +289,57 @@ assert isinstance(c, CppyyTestData) # char types - CppyyTestData.s_char = 'a' - assert c.s_char == 'a' - c.s_char = 'b' - assert CppyyTestData.s_char == 'b' - CppyyTestData.s_uchar = 'c' - assert c.s_uchar == 'c' - c.s_uchar = 'd' - assert CppyyTestData.s_uchar == 'd' + CppyyTestData.s_char = 'a' + assert c.s_char == b'a' + c.s_char = 'b' + assert CppyyTestData.s_char == b'b' + CppyyTestData.s_uchar = 'c' + assert c.s_uchar == b'c' + c.s_uchar = 'd' + assert CppyyTestData.s_uchar == b'd' raises(ValueError, setattr, CppyyTestData, 's_uchar', -1) - raises(ValueError, setattr, c, 's_uchar', -1) + raises(ValueError, setattr, c, 's_uchar', -1) # integer types c.s_short = -102 - assert CppyyTestData.s_short == -102 - CppyyTestData.s_short = -203 + assert CppyyTestData.s_short == -102 + CppyyTestData.s_short = -203 assert c.s_short == -203 c.s_ushort = 127 - assert CppyyTestData.s_ushort == 127 - CppyyTestData.s_ushort = 227 + assert CppyyTestData.s_ushort == 127 + CppyyTestData.s_ushort = 227 assert c.s_ushort == 227 - CppyyTestData.s_int = -234 + CppyyTestData.s_int = -234 assert c.s_int == -234 c.s_int = -321 - assert CppyyTestData.s_int == -321 - CppyyTestData.s_uint = 1234 + assert CppyyTestData.s_int == -321 + CppyyTestData.s_uint = 1234 assert c.s_uint == 1234 c.s_uint = 4321 - assert CppyyTestData.s_uint == 4321 + assert CppyyTestData.s_uint == 4321 raises(ValueError, setattr, c, 's_uint', -1) raises(ValueError, setattr, CppyyTestData, 's_uint', -1) - CppyyTestData.s_long = -87 + CppyyTestData.s_long = -87 assert c.s_long == -87 c.s_long = 876 - assert CppyyTestData.s_long == 876 - CppyyTestData.s_ulong = 876 + assert CppyyTestData.s_long == 876 + CppyyTestData.s_ulong = 876 assert c.s_ulong == 876 c.s_ulong = 678 - assert CppyyTestData.s_ulong == 678 + assert CppyyTestData.s_ulong == 678 raises(ValueError, setattr, CppyyTestData, 's_ulong', -1) - raises(ValueError, setattr, c, 's_ulong', -1) + raises(ValueError, setattr, c, 's_ulong', -1) # floating point types CppyyTestData.s_float = -3.1415 - assert round(c.s_float, 5 ) == -3.1415 - c.s_float = 3.1415 + assert round(c.s_float, 5 ) == -3.1415 + c.s_float = 3.1415 assert round(CppyyTestData.s_float, 5 ) == 3.1415 import math - c.s_double = -math.pi + c.s_double = -math.pi assert CppyyTestData.s_double == -math.pi CppyyTestData.s_double = math.pi - assert c.s_double == math.pi + assert c.s_double == math.pi c.destruct() diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -579,7 +579,7 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', - 'PyBuffer_Release', + 'PyObject_GetBuffer', 'PyBuffer_Release', 'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr', 'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr', @@ -597,6 +597,7 @@ 'PyImport_ImportModuleLevel', 'PyOS_getsig', 'PyOS_setsig', + '_Py_RestoreSignals', 'PyThread_get_thread_ident', 'PyThread_allocate_lock', 'PyThread_free_lock', 'PyThread_acquire_lock', 'PyThread_release_lock', 'PyThread_create_key', 'PyThread_delete_key', 'PyThread_set_key_value', diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -1,5 +1,8 @@ #ifndef Py_ABSTRACTOBJECT_H #define Py_ABSTRACTOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif /* new buffer API */ @@ -10,9 +13,22 @@ /* Return 1 if the getbuffer function is available, otherwise return 0 */ + PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, Py_buffer *view, + int flags); + + /* This is a C-API version of the getbuffer function call. It checks + to make sure object has the required function pointer and issues the + call. Returns -1 and raises an error on failure and returns 0 on + success + */ + PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. */ + +#ifdef __cplusplus +} +#endif #endif /* Py_ABSTRACTOBJECT_H */ diff --git a/pypy/module/cpyext/include/pylifecycle.h b/pypy/module/cpyext/include/pylifecycle.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/pylifecycle.h @@ -0,0 +1,17 @@ +#ifndef Py_PYLIFECYCLE_H +#define Py_PYLIFECYCLE_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _Py_RestoreSignals(void); +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_PYLIFECYCLE_H */ diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -1,9 +1,11 @@ +from rpython.rlib.objectmodel import keepalive_until_here +from pypy.interpreter.error import oefmt from pypy.module.cpyext.api import ( cpython_api, Py_buffer, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP, PyObjectFields, cpython_struct, - bootstrap_function, Py_bufferP, generic_cpy_call, slot_function) + bootstrap_function, Py_bufferP, slot_function, generic_cpy_call) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, as_pyobj, incref, decref, from_ref, make_typedescr, + PyObject, make_ref, as_pyobj, decref, from_ref, make_typedescr, get_typedescr, track_reference) from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import widen @@ -94,30 +96,6 @@ mem_obj.c_view.c_obj = rffi.cast(PyObject, 0) _dealloc(space, py_obj) - at cpython_api([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) -def PyObject_GetBuffer(space, exporter, view, flags): - """Send a request to exporter to fill in view as specified by flags. If the - exporter cannot provide a buffer of the exact type, it MUST raise - PyExc_BufferError, set view->obj to NULL and return -1. - - On success, fill in view, set view->obj to a new reference to exporter and - return 0. In the case of chained buffer providers that redirect requests - to a single object, view->obj MAY refer to this object instead of exporter. - - Successful calls to PyObject_GetBuffer() must be paired with calls to - PyBuffer_Release(), similar to malloc() and free(). Thus, after the - consumer is done with the buffer, PyBuffer_Release() must be called exactly - once. - """ - # XXX compare this implementation with PyPy2's XXX - pb = exporter.c_ob_type.c_tp_as_buffer - if not pb or not pb.c_bf_getbuffer: - w_exporter = from_ref(space, exporter) - raise oefmt(space.w_TypeError, - "a bytes-like object is required, not '%T'", w_exporter) - return generic_cpy_call(space, pb.c_bf_getbuffer, exporter, view, flags) - def fill_Py_buffer(space, buf, view): # c_buf, c_obj have been filled in ndim = buf.getndim() diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -7,14 +7,14 @@ from rpython.rlib import rgc # Force registration of gc.collect from pypy.module.cpyext.api import ( slot_function, generic_cpy_call, PyObject, Py_ssize_t, - pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr) + pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr, cts) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, ternaryfunc, binaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, getbufferproc, releasebufferproc, ssizessizeobjargproc) -from pypy.module.cpyext.pyobject import make_ref, decref +from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.memoryobject import fill_Py_buffer from pypy.module.cpyext.state import State @@ -318,13 +318,14 @@ _immutable_ = True def __init__(self, space, ptr, size, w_obj, format='B', shape=None, - strides=None, ndim=1, itemsize=1, readonly=True, - releasebufferproc=rffi.cast(rffi.VOIDP, 0)): + strides=None, ndim=1, itemsize=1, readonly=True, + needs_decref=False, + releasebufferproc=rffi.cast(rffi.VOIDP, 0)): self.space = space self.ptr = ptr self.size = size self.w_obj = w_obj # kept alive - self.pyobj = make_ref(space, w_obj) + self.pyobj = as_pyobj(space, w_obj) self.format = format self.ndim = ndim self.itemsize = itemsize @@ -348,30 +349,33 @@ # XXX: missing init_strides_from_shape self.strides = strides self.readonly = readonly + self.needs_decref = needs_decref self.releasebufferproc = releasebufferproc def releasebuffer(self): if self.pyobj: - decref(self.space, self.pyobj) + if self.needs_decref: + if self.releasebufferproc: + func_target = rffi.cast(releasebufferproc, self.releasebufferproc) + with lltype.scoped_alloc(Py_buffer) as pybuf: + pybuf.c_buf = self.ptr + pybuf.c_len = self.size + pybuf.c_ndim = cts.cast('int', self.ndim) + pybuf.c_shape = cts.cast('Py_ssize_t*', pybuf.c__shape) + pybuf.c_strides = cts.cast('Py_ssize_t*', pybuf.c__strides) + for i in range(self.ndim): + pybuf.c_shape[i] = self.shape[i] + pybuf.c_strides[i] = self.strides[i] + if self.format: + pybuf.c_format = rffi.str2charp(self.format) + else: + pybuf.c_format = rffi.str2charp("B") + generic_cpy_call(self.space, func_target, self.pyobj, pybuf) + decref(self.space, self.pyobj) self.pyobj = lltype.nullptr(PyObject.TO) else: #do not call twice return - if self.releasebufferproc: - func_target = rffi.cast(releasebufferproc, self.releasebufferproc) - with lltype.scoped_alloc(Py_buffer) as pybuf: - pybuf.c_buf = self.ptr - pybuf.c_len = self.size - pybuf.c_ndim = rffi.cast(rffi.INT_real, self.ndim) - for i in range(self.ndim): - pybuf.c_shape[i] = self.shape[i] - pybuf.c_strides[i] = self.strides[i] - if self.format: - pybuf.c_format = rffi.str2charp(self.format) - else: - pybuf.c_format = rffi.str2charp("B") - generic_cpy_call(self.space, func_target, self.pyobj, pybuf) - self.releasebufferproc = rffi.cast(rffi.VOIDP, 0) def getlength(self): return self.size @@ -467,22 +471,25 @@ ptr = pybuf.c_buf size = pybuf.c_len ndim = widen(pybuf.c_ndim) + shape = None if pybuf.c_shape: - shape = [pybuf.c_shape[i] for i in range(ndim)] - else: - shape = None + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = None if pybuf.c_strides: strides = [pybuf.c_strides[i] for i in range(ndim)] - else: - strides = [1] if pybuf.c_format: format = rffi.charp2str(pybuf.c_format) else: format = 'B' + # the CPython docs mandates that you do an incref whenever you call + # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't + # leak we release the buffer: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer buf = CPyBuffer(space, ptr, size, w_self, format=format, ndim=ndim, shape=shape, strides=strides, itemsize=pybuf.c_itemsize, readonly=widen(pybuf.c_readonly), + needs_decref=True, releasebufferproc = rbp) fq.register_finalizer(buf) return space.newbuffer(buf, itemsize=buf.itemsize) @@ -717,33 +724,12 @@ slot_func = slot_tp_new elif name == 'tp_as_buffer.c_bf_getbuffer': buff_fn = w_type.getdictvalue(space, '__buffer__') - if buff_fn is None: + if buff_fn is not None: + buff_w = slot_from___buffer__(space, typedef, buff_fn) + elif typedef.buffer: + buff_w = slot_from_buffer_w(space, typedef, buff_fn) + else: return - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def buff_w(space, w_self, view, flags): - args = Arguments(space, [space.newint(flags)]) - w_obj = space.call_args(space.get(buff_fn, w_self), args) - if view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - view.c_obj = make_ref(space, w_s) - view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, view) - return ret - return 0 - # XXX remove this when it no longer crashes a translated PyPy - return slot_func = buff_w else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce @@ -753,6 +739,60 @@ return slot_func + +def slot_from___buffer__(space, typedef, buff_fn): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, view, flags): + args = Arguments(space, [space.newint(flags)]) + w_obj = space.call_args(space.get(buff_fn, w_self), args) + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + return buff_w + +def slot_from_buffer_w(space, typedef, buff_fn): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, view, flags): + w_obj = w_self + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + return buff_w + PyWrapperFlag_KEYWORDS = 1 class TypeSlot: diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -98,6 +98,18 @@ /* Buffer C-API for Python 3.0 */ +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + void PyBuffer_Release(Py_buffer *view) { diff --git a/pypy/module/cpyext/src/pylifecycle.c b/pypy/module/cpyext/src/pylifecycle.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/src/pylifecycle.c @@ -0,0 +1,15 @@ +#include "Python.h" + +void +_Py_RestoreSignals(void) +{ +#ifdef SIGPIPE + PyOS_setsig(SIGPIPE, SIG_DFL); +#endif +#ifdef SIGXFZ + PyOS_setsig(SIGXFZ, SIG_DFL); +#endif +#ifdef SIGXFSZ + PyOS_setsig(SIGXFSZ, SIG_DFL); +#endif +} diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2351,10 +2351,10 @@ } else if(obj1->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); else if(obj2->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -2401,10 +2401,10 @@ } else if(obj1->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); + ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name); else if(obj2->ob_type == &Arraytype) fprintf(stderr, "\nCannot multiply array of type %c and %s\n", - ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); + ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name); Py_INCREF(Py_NotImplemented); return Py_NotImplemented; } @@ -2426,7 +2426,7 @@ static int -array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) +array_getbuffer(arrayobject *self, Py_buffer *view, int flags) { if (view==NULL) goto finish; @@ -2463,10 +2463,19 @@ return 0; } +static long releasebuffer_cnt = 0; + +static PyObject * +get_releasebuffer_cnt(void) +{ + return PyLong_FromLong(releasebuffer_cnt); +} + static void -array_buffer_relbuf(arrayobject *self, Py_buffer *view) +array_releasebuffer(arrayobject *self, Py_buffer *view) { self->ob_exports--; + releasebuffer_cnt++; } static PySequenceMethods array_as_sequence = { @@ -2483,8 +2492,8 @@ }; static PyBufferProcs array_as_buffer = { - (getbufferproc)array_buffer_getbuf, - (releasebufferproc)array_buffer_relbuf + (getbufferproc)array_getbuffer, + (releasebufferproc)array_releasebuffer }; static PyObject * @@ -2872,6 +2881,16 @@ } static PyObject * +create_and_release_buffer(PyObject *self, PyObject *obj) +{ + Py_buffer view; + int res = PyObject_GetBuffer(obj, &view, 0); + if (res < 0) + return NULL; + PyBuffer_Release(&view); + Py_RETURN_NONE; +} +static PyObject * write_buffer_len(PyObject * self, PyObject * obj) { void* buf; @@ -2890,6 +2909,8 @@ PyDoc_STR("Internal. Used for pickling support.")}, {"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL}, {"readbuffer_as_string", (PyCFunction)readbuffer_as_string, METH_VARARGS, NULL}, + {"get_releasebuffer_cnt", (PyCFunction)get_releasebuffer_cnt, METH_NOARGS, NULL}, + {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {"write_buffer_len", write_buffer_len, METH_O, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -69,6 +69,23 @@ b'\x03\0\0\0' b'\x04\0\0\0') + def test_releasebuffer(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert module.get_releasebuffer_cnt() == 0 + module.create_and_release_buffer(arr) + assert module.get_releasebuffer_cnt() == 1 + + def test_Py_buffer(self): + module = self.import_module(name='array') + arr = module.array('i', [1,2,3,4]) + assert module.get_releasebuffer_cnt() == 0 + m = memoryview(arr) + assert module.get_releasebuffer_cnt() == 0 + del m + self.debug_collect() + assert module.get_releasebuffer_cnt() == 1 + def test_0d_view(self): module = self.import_module(name='array') arr = module.array('B', b'\0\0\0\x01') diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -214,6 +214,15 @@ print(module.fmt(b'd:%d', 10)) assert module.fmt(b'd:%d', 10) == b'd:10' + def test_suboffsets(self): + module = self.import_extension('foo', [ + ("check_suboffsets", "METH_O", + """ + Py_buffer view; + PyObject_GetBuffer(args, &view, 0); + return PyLong_FromLong(view.suboffsets == NULL); + """)]) + assert module.check_suboffsets(b'1234') == 1 class TestBytes(BaseApiTest): def test_bytes_resize(self, space, api): @@ -304,9 +313,3 @@ assert api.PyBytes_FromObject(w_obj) is None api.PyErr_Clear() - def test_suboffsets(self, space, api): - w_bytes = space.newbytes('1234') - view = lltype.malloc(Py_buffer, flavor='raw', zero=True) - flags = rffi.cast(rffi.INT_real, 0) - api.PyObject_GetBuffer(w_bytes, view, flags) - assert not view.c_suboffsets diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -10,15 +10,6 @@ only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" class TestMemoryViewObject(BaseApiTest): - def test_fromobject(self, space, api): - w_hello = space.newbytes("hello") - #assert api.PyObject_CheckBuffer(w_hello) - w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello)) - w_byte = space.call_method(w_view, '__getitem__', space.wrap(0)) - assert space.eq_w(w_byte, space.wrap(ord('h'))) - w_bytes = space.call_method(w_view, "tobytes") - assert space.unwrap(w_bytes) == "hello" - def test_frombuffer(self, space, api): w_buf = space.newbuffer(StringBuffer("hello")) c_memoryview = rffi.cast( @@ -96,6 +87,19 @@ assert result.tobytes() == b'hello, world.' class AppTestBufferProtocol(AppTestCpythonExtensionBase): + def test_fromobject(self): + foo = self.import_extension('foo', [ + ("make_view", "METH_O", + """ + if (!PyObject_CheckBuffer(args)) + return Py_None; + return PyMemoryView_FromObject(args); + """)]) + hello = b'hello' + mview = foo.make_view(hello) + assert mview[0] == hello[0] + assert mview.tobytes() == hello + def test_buffer_protocol_app(self): module = self.import_module(name='buffer_test') arr = module.PyMyArray(10) @@ -183,3 +187,75 @@ " on too long format string" finally: warnings.resetwarnings() + + def test_releasebuffer(self): + if not self.runappdirect: + skip("Fails due to ll2ctypes nonsense") + module = self.import_extension('foo', [ + ("create_test", "METH_NOARGS", + """ + PyObject *obj; + obj = PyObject_New(PyObject, (PyTypeObject*)type); + return obj; + """), + ("get_cnt", "METH_NOARGS", + 'return PyLong_FromLong(cnt);'), + ("get_dealloc_cnt", "METH_NOARGS", + 'return PyLong_FromLong(dealloc_cnt);'), + ], + prologue=""" + static float test_data = 42.f; + static int cnt=0; + static int dealloc_cnt=0; + static PyHeapTypeObject * type=NULL; + + void dealloc(PyObject *self) { + dealloc_cnt++; + } + int getbuffer(PyObject *obj, Py_buffer *view, int flags) { + + cnt ++; + memset(view, 0, sizeof(Py_buffer)); + view->obj = obj; + /* see the CPython docs for why we need this incref: + https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer */ + Py_INCREF(obj); + view->ndim = 0; + view->buf = (void *) &test_data; + view->itemsize = sizeof(float); + view->len = 1; + view->strides = NULL; + view->shape = NULL; + view->format = "f"; + return 0; + } + + void releasebuffer(PyObject *obj, Py_buffer *view) { + cnt --; + } + """, more_init=""" + type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + + type->ht_type.tp_name = "Test"; + type->ht_type.tp_basicsize = sizeof(PyObject); + type->ht_name = PyUnicode_FromString("Test"); + type->ht_type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HEAPTYPE; + type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; + + type->ht_type.tp_dealloc = dealloc; + type->ht_type.tp_as_buffer = &type->as_buffer; + type->as_buffer.bf_getbuffer = getbuffer; + type->as_buffer.bf_releasebuffer = releasebuffer; + + if (PyType_Ready(&type->ht_type) < 0) INITERROR; + """, ) + import gc + assert module.get_cnt() == 0 + a = memoryview(module.create_test()) + assert module.get_cnt() == 1 + assert module.get_dealloc_cnt() == 0 + del a + self.debug_collect() + assert module.get_cnt() == 0 + assert module.get_dealloc_cnt() == 1 diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -235,6 +235,18 @@ """)]) assert module.test_macro_invocations() == u'' + def test_AsUTF8AndSize(self): + module = self.import_extension('foo', [ + ("utf8", "METH_O", + """ + Py_ssize_t size; + char *utf8 = PyUnicode_AsUTF8AndSize(args, &size); + return PyBytes_FromStringAndSize(utf8, size); + """)]) + assert module.utf8('xyz') == b'xyz' + assert module.utf8('café') == 'café'.encode('utf-8') + + class TestUnicode(BaseApiTest): def test_unicodeobject(self, space): encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, )) diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -103,17 +103,20 @@ return py_obj.c_state def get_kind(py_obj): - return get_state(py_obj).c_kind + return rffi.getintfield(get_state(py_obj), 'c_kind') def set_kind(py_obj, value): get_state(py_obj).c_kind = cts.cast('unsigned int', value) def get_ascii(py_obj): - return get_state(py_obj).c_ascii + return rffi.getintfield(get_state(py_obj), 'c_ascii') def set_ascii(py_obj, value): get_state(py_obj).c_ascii = cts.cast('unsigned int', value) +def get_ready(py_obj): + return rffi.getintfield(get_state(py_obj), 'c_ready') + def set_ready(py_obj, value): get_state(py_obj).c_ready = cts.cast('unsigned int', value) @@ -259,7 +262,7 @@ def _PyUnicode_Ready(space, w_obj): assert isinstance(w_obj, unicodeobject.W_UnicodeObject) py_obj = as_pyobj(space, w_obj) - assert widen(get_kind(py_obj)) == WCHAR_KIND + assert get_kind(py_obj) == WCHAR_KIND maxchar = 0 for c in w_obj._value: if ord(c) > maxchar: @@ -303,8 +306,8 @@ set_ready(py_obj, 1) return 0 - at cpython_api([PyObject], rffi.CWCHARP) -def PyUnicode_AsUnicode(space, ref): + at cts.decl("Py_UNICODE * PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size)") +def PyUnicode_AsUnicodeAndSize(space, ref, psize): """Return a read-only pointer to the Unicode object's internal Py_UNICODE buffer, NULL if unicode is not a Unicode object.""" # Don't use PyUnicode_Check, it will realize the object :-( @@ -316,10 +319,22 @@ w_unicode = from_ref(space, rffi.cast(PyObject, ref)) u = space.unicode_w(w_unicode) set_wbuffer(ref, rffi.unicode2wcharp(u)) + set_wsize(ref, len(u)) + if psize: + psize[0] = get_wsize(ref) return get_wbuffer(ref) - at cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)") -def PyUnicode_AsUTF8(space, ref): + at cts.decl("Py_UNICODE * PyUnicode_AsUnicode(PyObject *unicode)") +def PyUnicode_AsUnicode(space, ref): + return PyUnicode_AsUnicodeAndSize(space, ref, cts.cast('Py_ssize_t *', 0)) + + at cts.decl("char * PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize)") +def PyUnicode_AsUTF8AndSize(space, ref, psize): + if not PyUnicode_Check(space, ref): + PyErr_BadArgument(space) + if not get_ready(ref): + res = _PyUnicode_Ready(space, ref) + if not get_utf8(ref): # Copy unicode buffer w_unicode = from_ref(space, ref) @@ -327,8 +342,15 @@ "strict") s = space.bytes_w(w_encoded) set_utf8(ref, rffi.str2charp(s)) + set_utf8_len(ref, len(s)) + if psize: + psize[0] = get_utf8_len(ref) return get_utf8(ref) + at cts.decl("char * PyUnicode_AsUTF8(PyObject *unicode)") +def PyUnicode_AsUTF8(space, ref): + return PyUnicode_AsUTF8AndSize(space, ref, cts.cast('Py_ssize_t *', 0)) + @cpython_api([PyObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1) def PyUnicode_AsWideChar(space, ref, buf, size): """Copy the Unicode object contents into the wchar_t buffer w. At most diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -1513,7 +1513,7 @@ return res """, filename=__file__).interphook('ptp') -W_NDimArray.typedef = TypeDef("numpy.ndarray", +W_NDimArray.typedef = TypeDef("numpy.ndarray", None, None, 'read-write', __new__ = interp2app(descr_new_array), __len__ = interp2app(W_NDimArray.descr_len), diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -1,3 +1,4 @@ +import sys from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rposix from rpython.rlib import rdynload @@ -213,6 +214,9 @@ assert getattr(rposix, _name) is not None, "missing %r" % (_name,) interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name) + if sys.platform.startswith('linux'): #hasattr(rposix, 'sendfile'): + interpleveldefs['sendfile'] = 'interp_posix.sendfile' + if hasattr(rposix, 'pread'): interpleveldefs['pread'] = 'interp_posix.pread' if hasattr(rposix, 'pwrite'): diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -2371,3 +2371,28 @@ rposix.set_status_flags(fd, flags) except OSError as e: raise wrap_oserror(space, e, eintr_retry=False) + + at unwrap_spec(out=c_int, count=int) +def sendfile(space, out, w_in, w_offset, count): + # why is an argument called "in"??? that doesn't make sense (it is + # a reserved word), but that's what CPython does + in_ = space.c_int_w(w_in) + + # XXX only supports the common arguments for now (BSD takes more). + # Until that is fixed, we only expose sendfile() on linux. + if space.is_none(w_offset): # linux only + while True: + try: + res = rposix.sendfile_no_offset(out, in_, count) + break + except OSError as e: + wrap_oserror(space, e, eintr_retry=True) + else: + offset = space.gateway_r_longlong_w(w_offset) + while True: + try: + res = rposix.sendfile(out, in_, offset, count) + break + except OSError as e: + wrap_oserror(space, e, eintr_retry=True) + return space.newint(res) diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py --- a/pypy/module/posix/interp_scandir.py +++ b/pypy/module/posix/interp_scandir.py @@ -161,7 +161,7 @@ if (self.flags & FLAG_LSTAT) == 0: # Unlike CPython, try to use fstatat() if possible dirfd = self.scandir_iterator.dirfd - if dirfd != -1: + if dirfd != -1 and rposix.HAVE_FSTATAT: st = rposix_stat.fstatat(self.name, dirfd, follow_symlinks=False) else: @@ -206,7 +206,7 @@ if must_call_stat: # Must call stat(). Try to use fstatat() if possible dirfd = self.scandir_iterator.dirfd - if dirfd != -1: + if dirfd != -1 and rposix.HAVE_FSTATAT: st = rposix_stat.fstatat(self.name, dirfd, follow_symlinks=True) else: diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -14,7 +14,7 @@ USEMODULES = ['binascii', 'posix', 'signal', 'struct', 'time'] # py3k os.open uses subprocess, requiring the following per platform if os.name != 'nt': - USEMODULES += ['fcntl', 'select', '_posixsubprocess'] + USEMODULES += ['fcntl', 'select', '_posixsubprocess', '_socket'] else: USEMODULES += ['_rawffi', 'thread'] @@ -1213,6 +1213,36 @@ raises(OSError, posix.get_blocking, 1234567) raises(OSError, posix.set_blocking, 1234567, True) + if sys.platform != 'win32': + def test_sendfile(self): + import _socket, posix + s1, s2 = _socket.socketpair() + fd = posix.open(self.path, posix.O_RDONLY) + res = posix.sendfile(s1.fileno(), fd, 3, 5) + assert res == 5 + assert posix.lseek(fd, 0, 1) == 0 + data = s2.recv(10) + expected = b'this is a test'[3:8] + assert data == expected + posix.close(fd) + s2.close() + s1.close() + + if sys.platform.startswith('linux'): + def test_sendfile_no_offset(self): + import _socket, posix + s1, s2 = _socket.socketpair() + fd = posix.open(self.path, posix.O_RDONLY) + posix.lseek(fd, 3, 0) + res = posix.sendfile(s1.fileno(), fd, None, 5) + assert res == 5 + assert posix.lseek(fd, 0, 1) == 8 + data = s2.recv(10) + expected = b'this is a test'[3:8] + assert data == expected + posix.close(fd) + s2.close() + s1.close() def test_urandom(self): os = self.posix diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py --- a/pypy/module/select/interp_kqueue.py +++ b/pypy/module/select/interp_kqueue.py @@ -1,8 +1,10 @@ +import errno from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import oefmt from pypy.interpreter.error import exception_from_saved_errno, wrap_oserror from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.typedef import TypeDef, generic_new_descr, GetSetProperty +from pypy.interpreter import timeutils from rpython.rlib._rsocket_rffi import socketclose_no_errno from rpython.rlib.rarithmetic import r_uint from rpython.rlib import rposix @@ -50,6 +52,12 @@ ("tv_nsec", rffi.LONG), ]) +def fill_timespec(time_float, timespec_ptr): + sec = int(time_float) + nsec = int(1e9 * (time_float - sec)) + rffi.setintfield(timespec_ptr, 'c_tv_sec', sec) + rffi.setintfield(timespec_ptr, 'c_tv_nsec', nsec) + symbol_map = { "KQ_FILTER_READ": "EVFILT_READ", @@ -180,13 +188,11 @@ raise oefmt(space.w_ValueError, "Timeout must be None or >= 0, got %s", str(_timeout)) - XXX # fix test_select_signal.py first, for PEP475! - sec = int(_timeout) - nsec = int(1e9 * (_timeout - sec)) - rffi.setintfield(timeout, 'c_tv_sec', sec) - rffi.setintfield(timeout, 'c_tv_nsec', nsec) + fill_timespec(_timeout, timeout) + timeout_at = timeutils.monotonic(space) + _timeout ptimeout = timeout else: + timeout_at = 0.0 ptimeout = lltype.nullptr(timespec) if not space.is_w(w_changelist, space.w_None): @@ -204,29 +210,40 @@ else: pchangelist = lltype.nullptr(rffi.CArray(kevent)) - nfds = syscall_kevent(self.kqfd, - pchangelist, - changelist_len, - eventlist, - max_events, - ptimeout) - if nfds < 0: - raise exception_from_saved_errno(space, space.w_OSError) - else: - elist_w = [None] * nfds - for i in xrange(nfds): + while True: + nfds = syscall_kevent(self.kqfd, + pchangelist, + changelist_len, + eventlist, + max_events, + ptimeout) + if nfds >= 0: + break + if rposix.get_saved_errno() != errno.EINTR: + raise exception_from_saved_errno(space, + space.w_OSError) + space.getexecutioncontext().checksignals() + if ptimeout: + _timeout = (timeout_at - + timeutils.monotonic(space)) + if _timeout < 0.0: + _timeout = 0.0 + fill_timespec(_timeout, ptimeout) - evt = eventlist[i] + elist_w = [None] * nfds + for i in xrange(nfds): - w_event = W_Kevent(space) - w_event.ident = evt.c_ident - w_event.filter = evt.c_filter - w_event.flags = evt.c_flags - w_event.fflags = evt.c_fflags - w_event.data = evt.c_data - w_event.udata = evt.c_udata + evt = eventlist[i] - elist_w[i] = w_event + w_event = W_Kevent(space) + w_event.ident = evt.c_ident + w_event.filter = evt.c_filter + w_event.flags = evt.c_flags + w_event.fflags = evt.c_fflags + w_event.data = evt.c_data + w_event.udata = evt.c_udata + + elist_w[i] = w_event return space.newlist(elist_w) diff --git a/pypy/module/select/test/test_kqueue.py b/pypy/module/select/test/test_kqueue.py --- a/pypy/module/select/test/test_kqueue.py +++ b/pypy/module/select/test/test_kqueue.py @@ -102,7 +102,10 @@ except socket.error as e: assert e.args[0] == errno.EINPROGRESS else: - assert False, "EINPROGRESS not raised" + #assert False, "EINPROGRESS not raised" + pass # FreeBSD doesn't raise an exception here + # (the above commented-out code is just like CPython's + # test_kqueue) server, addr = server_socket.accept() if sys.platform.startswith("darwin"): diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py --- a/pypy/objspace/std/objectobject.py +++ b/pypy/objspace/std/objectobject.py @@ -27,6 +27,9 @@ # and raises a TypeError if the condition holds true, this is done # just before reduce_2 is called in pypy state = getattr(obj, "__dict__", None) + # CPython returns None if the dict is empty + if state is not None and len(state) == 0: + state = None names = slotnames(cls) # not checking for list if names is not None: slots = {} diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -89,6 +89,11 @@ assert '__getnewargs__' not in seen assert '__getnewargs_ex__' not in seen + def test_reduce_state_empty_dict(self): + class X(object): + pass + assert X().__reduce_ex__(2)[2] is None + def test_default_format(self): class x(object): def __str__(self): diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -35,7 +35,7 @@ w_uni = space.wrap(u'abcd') assert space.text_w(w_uni) == 'abcd' w_uni = space.wrap(unichr(0xd921) + unichr(0xdddd)) - raises(UnicodeEncodeError, space.text_w, w_uni) + space.raises_w(space.w_UnicodeEncodeError, space.text_w, w_uni) class AppTestUnicodeStringStdOnly: @@ -500,6 +500,7 @@ assert u'x\ty'.expandtabs(-42) == u'xy' def test_translate(self): + import sys assert 'bbbc' == 'abababc'.translate({ord('a'):None}) assert 'iiic' == 'abababc'.translate({ord('a'):None, ord('b'):ord('i')}) assert 'iiix' == 'abababc'.translate({ord('a'):None, ord('b'):ord('i'), ord('c'):'x'}) @@ -511,6 +512,7 @@ assert 'abababc'.translate({ord('a'): ''}) == 'bbbc' raises(TypeError, 'hello'.translate) + raises(ValueError, "\xff".translate, {0xff: sys.maxunicode+1}) def test_maketrans(self): assert 'abababc' == 'abababc'.translate({'b': ''}) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -429,7 +429,7 @@ elif space.isinstance_w(w_newval, space.w_int): newval = space.int_w(w_newval) if newval < 0 or newval > maxunicode: - raise oefmt(space.w_TypeError, + raise oefmt(space.w_ValueError, "character mapping must be in range(%s)", hex(maxunicode + 1)) result.append(unichr(newval)) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -948,11 +948,11 @@ # If this also contains args[0], this returns the current # location too. arglocs = [self.loc(args[i]) for i in range(2, len(args))] - gcmap = self.get_gcmap() if op.type == 'v': # a plain COND_CALL. Calls the function when args[0] is # true. Often used just after a comparison operation. + gcmap = self.get_gcmap() self.load_condition_into_cc(op.getarg(0)) resloc = None else: @@ -969,6 +969,22 @@ resloc = self.rm.force_result_in_reg(op, args[0], forbidden_vars=args[2:]) + # Get the gcmap here, possibly including the spilled + # location, and always excluding the 'resloc' register. + # Some more details: the only interesting case is the case + # where we're doing the call (if we are not, the gcmap is + # not used); and in this case, the gcmap must include the + # spilled location (it contains a valid GC pointer to fix + # during the call if a GC occurs), and never 'resloc' + # (it will be overwritten with the result of the call, which + # is not computed yet if a GC occurs). + # + # (Note that the spilled value is always NULL at the moment + # if the call really occurs, but it's not worth the effort to + # not list it in the gcmap and get crashes if we tweak + # COND_CALL_VALUE_R in the future) + gcmap = self.get_gcmap([resloc]) + # Test the register for the result. self.assembler.test_location(resloc) self.assembler.guard_success_cc = rx86.Conditions['Z'] diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -773,24 +773,33 @@ from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound,\ IntLowerBound - if mode is None: + length = self.getstrlen1(mode) + if length < 0: # XXX we can do better if we know it's an array return IntLowerBound(0) - else: - return ConstIntBound(self.getstrlen1(mode)) + return ConstIntBound(length) def getstrlen(self, op, string_optimizer, mode): - return ConstInt(self.getstrlen1(mode)) + length = self.getstrlen1(mode) + if length < 0: + return None + return ConstInt(length) def getstrlen1(self, mode): from rpython.jit.metainterp.optimizeopt import vstring - + if mode is vstring.mode_string: s = self._unpack_str(vstring.mode_string) + if s is None: + return -1 + return len(s) + elif mode is vstring.mode_unicode: + s = self._unpack_str(vstring.mode_unicode) + if s is None: + return -1 return len(s) else: - s = self._unpack_str(vstring.mode_unicode) - return len(s) + return -1 def getstrhash(self, op, mode): from rpython.jit.metainterp.optimizeopt import vstring diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -2124,7 +2124,8 @@ def _malloc_out_of_nursery(self, totalsize): """Allocate non-movable memory for an object of the given 'totalsize' that lives so far in the nursery.""" - if raw_malloc_usage(totalsize) <= self.small_request_threshold: + if (r_uint(raw_malloc_usage(totalsize)) <= + r_uint(self.small_request_threshold)): # most common path return self.ac.malloc(totalsize) else: @@ -2133,6 +2134,9 @@ _malloc_out_of_nursery._always_inline_ = True def _malloc_out_of_nursery_nonsmall(self, totalsize): + if r_uint(raw_malloc_usage(totalsize)) > r_uint(self.nursery_size): + out_of_memory("memory corruption: bad size for object in the " + "nursery") # 'totalsize' should be aligned. ll_assert(raw_malloc_usage(totalsize) & (WORD-1) == 0, "misaligned totalsize in _malloc_out_of_nursery_nonsmall") diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -524,7 +524,8 @@ ll_attach = rffi.llexternal("AttachToVS", [], lltype.Void, compilation_info=make_vs_attach_eci()) def impl_attach_gdb(): - ll_attach() + #ll_attach() + print "AttachToVS is disabled at the moment (compilation failure)" register_external(attach_gdb, [], result=None, export_name="impl_attach_gdb", llimpl=impl_attach_gdb) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -2451,3 +2451,21 @@ def set_status_flags(fd, flags): res = c_set_status_flags(fd, flags) handle_posix_error('set_status_flags', res) + +if not _WIN32: + sendfile_eci = ExternalCompilationInfo(includes=["sys/sendfile.h"]) + _OFF_PTR_T = rffi.CArrayPtr(OFF_T) + c_sendfile = rffi.llexternal('sendfile', + [rffi.INT, rffi.INT, _OFF_PTR_T, rffi.SIZE_T], + rffi.SSIZE_T, compilation_info=sendfile_eci) + + def sendfile(out_fd, in_fd, offset, count): + with lltype.scoped_alloc(_OFF_PTR_T.TO, 1) as p_offset: + p_offset[0] = rffi.cast(OFF_T, offset) + res = c_sendfile(out_fd, in_fd, p_offset, count) + return handle_posix_error('sendfile', res) + + def sendfile_no_offset(out_fd, in_fd, count): + """Passes offset==NULL; not support on all OSes""" + res = c_sendfile(out_fd, in_fd, lltype.nullptr(_OFF_PTR_T.TO), count) + return handle_posix_error('sendfile', res) diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py --- a/rpython/translator/platform/linux.py +++ b/rpython/translator/platform/linux.py @@ -14,7 +14,7 @@ extra_libs = ('-lrt',) cflags = tuple( ['-O3', '-pthread', '-fomit-frame-pointer', - '-Wall', '-Wno-unused'] + '-Wall', '-Wno-unused', '-Wno-address'] + os.environ.get('CFLAGS', '').split()) standalone_only = () shared_only = ('-fPIC',) From pypy.commits at gmail.com Mon Mar 13 12:30:55 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:55 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: test_rposix.py merge conflict Message-ID: <58c6c93f.43142e0a.13044.0260@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90658:c1fc19dc40c2 Date: 2017-03-08 13:34 +0000 http://bitbucket.org/pypy/pypy/changeset/c1fc19dc40c2/ Log: test_rposix.py merge conflict diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -676,7 +676,40 @@ prio = rposix.getpriority(rposix.PRIO_PROCESS, 0) rposix.setpriority(rposix.PRIO_PROCESS, 0, prio) py.test.raises(OSError, rposix.getpriority, rposix.PRIO_PGRP, 123456789) + +if sys.platform != 'win32': + def test_sendfile(): + from rpython.rlib import rsocket + s1, s2 = rsocket.socketpair() + relpath = 'test_sendfile' + filename = str(udir.join(relpath)) + fd = os.open(filename, os.O_RDWR|os.O_CREAT, 0777) + os.write(fd, 'abcdefghij') + res = rposix.sendfile(s1.fd, fd, 3, 5) + assert res == 5 + data = os.read(s2.fd, 10) + assert data == 'defgh' + os.close(fd) + s2.close() + s1.close() +if sys.platform.startswith('linux'): + def test_sendfile_no_offset(): + from rpython.rlib import rsocket + s1, s2 = rsocket.socketpair() + relpath = 'test_sendfile' + filename = str(udir.join(relpath)) + fd = os.open(filename, os.O_RDWR|os.O_CREAT, 0777) + os.write(fd, 'abcdefghij') + os.lseek(fd, 3, 0) + res = rposix.sendfile_no_offset(s1.fd, fd, 5) + assert res == 5 + data = os.read(s2.fd, 10) + assert data == 'defgh' + os.close(fd) + s2.close() + s1.close() + @rposix_requires('pread') def test_pread(): fname = str(udir.join('os_test.txt')) From pypy.commits at gmail.com Mon Mar 13 12:30:57 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:30:57 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: no raise on pread Message-ID: <58c6c941.0da5190a.3c072.3f03@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90659:1be1c0a8fbcc Date: 2017-03-09 13:07 +0000 http://bitbucket.org/pypy/pypy/changeset/1be1c0a8fbcc/ Log: no raise on pread diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -384,7 +384,7 @@ try: s = rposix.pread(fd, length, offset) except OSError as e: - raise wrap_oserror(space, e, eintr_retry=False) + wrap_oserror(space, e, eintr_retry=True) else: return space.newbytes(s) From pypy.commits at gmail.com Mon Mar 13 12:31:00 2017 From: pypy.commits at gmail.com (nanjekye) Date: Mon, 13 Mar 2017 09:31:00 -0700 (PDT) Subject: [pypy-commit] pypy jumbo: change offset to r_longlong Message-ID: <58c6c944.05152e0a.8a182.6c73@mx.google.com> Author: Joannah Nanjekye Branch: jumbo Changeset: r90660:72708c9ace32 Date: 2017-03-13 16:24 +0000 http://bitbucket.org/pypy/pypy/changeset/72708c9ace32/ Log: change offset to r_longlong diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -376,7 +376,7 @@ except OSError as e: wrap_oserror(space, e, eintr_retry=True) - at unwrap_spec(fd=c_int, length=int, offset=int) + at unwrap_spec(fd=c_int, length=int, offset=r_longlong) def pread(space, fd, length, offset): """Read a string to a file descriptor at a given offset. """ @@ -388,7 +388,7 @@ else: return space.newbytes(s) - at unwrap_spec(fd=c_int, offset=int) + at unwrap_spec(fd=c_int, offset=r_longlong) def pwrite(space, fd, w_data, offset): """Write a string to a file descriptor at a given offset. """ From pypy.commits at gmail.com Mon Mar 13 12:34:45 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 09:34:45 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2480: Test and fix for 'tp_bases' Message-ID: <58c6ca25.51a0190a.f743a.33d9@mx.google.com> Author: Armin Rigo Branch: Changeset: r90662:1c56df677261 Date: 2017-03-13 17:34 +0100 http://bitbucket.org/pypy/pypy/changeset/1c56df677261/ Log: Issue #2480: Test and fix for 'tp_bases' diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1200,4 +1200,21 @@ assert type(obj).__doc__ == "The Base12 type or object" assert obj.__doc__ == "The Base12 type or object" - + def test_multiple_inheritance_fetch_tp_bases(self): + module = self.import_extension('foo', [ + ("foo", "METH_O", + ''' + PyTypeObject *tp; + tp = (PyTypeObject*)args; + Py_INCREF(tp->tp_bases); + return tp->tp_bases; + ''' + )]) + class A(object): + pass + class B(object): + pass + class C(A, B): + pass + bases = module.foo(C) + assert bases == (A, B) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -760,7 +760,7 @@ if builder.cpyext_type_init is not None: builder.cpyext_type_init.append((pto, w_type)) else: - finish_type_1(space, pto) + finish_type_1(space, pto, w_type.bases_w) finish_type_2(space, pto, w_type) pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) @@ -903,7 +903,7 @@ return w_obj -def finish_type_1(space, pto): +def finish_type_1(space, pto, bases_w=None): """ Sets up tp_bases, necessary before creating the interpreter type. """ @@ -915,11 +915,12 @@ if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type if not pto.c_tp_bases: - if not base: - bases = space.newtuple([]) - else: - bases = space.newtuple([from_ref(space, base_pyo)]) - pto.c_tp_bases = make_ref(space, bases) + if bases_w is None: + if not base: + bases_w = [] + else: + bases_w = [from_ref(space, base_pyo)] + pto.c_tp_bases = make_ref(space, space.newtuple(bases_w)) def finish_type_2(space, pto, w_obj): """ From pypy.commits at gmail.com Mon Mar 13 12:36:16 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 09:36:16 -0700 (PDT) Subject: [pypy-commit] pypy default: backport from f90ac97e3a04 the changes that go to rpython/ Message-ID: <58c6ca80.cdd8190a.421eb.47d5@mx.google.com> Author: Armin Rigo Branch: Changeset: r90663:dd08f2c2fef2 Date: 2017-03-13 17:35 +0100 http://bitbucket.org/pypy/pypy/changeset/dd08f2c2fef2/ Log: backport from f90ac97e3a04 the changes that go to rpython/ diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -464,6 +464,29 @@ how = SEEK_END return handle_posix_error('lseek', c_lseek(fd, pos, how)) +c_pread = external('pread', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T , OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) +c_pwrite = external('pwrite', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) + + at enforceargs(int, int, None) +def pread(fd, count, offset): + if count < 0: + raise OSError(errno.EINVAL, None) + validate_fd(fd) + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) + + at enforceargs(int, None, None) +def pwrite(fd, data, offset): + count = len(data) + validate_fd(fd) + with rffi.scoped_nonmovingbuffer(data) as buf: + return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) + c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) c_fsync = external('fsync' if not _WIN32 else '_commit', [rffi.INT], rffi.INT, diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -676,7 +676,7 @@ prio = rposix.getpriority(rposix.PRIO_PROCESS, 0) rposix.setpriority(rposix.PRIO_PROCESS, 0, prio) py.test.raises(OSError, rposix.getpriority, rposix.PRIO_PGRP, 123456789) - + if sys.platform != 'win32': def test_sendfile(): from rpython.rlib import rsocket @@ -723,3 +723,30 @@ os.close(fd) s2.close() s1.close() + + at rposix_requires('pread') +def test_pread(): + fname = str(udir.join('os_test.txt')) + fd = os.open(fname, os.O_RDWR | os.O_CREAT) + try: + assert fd >= 0 + os.write(fd, b'Hello world') + os.lseek(fd, 0, 0) + assert rposix.pread(fd, 2, 1) == b'el' + finally: + os.close(fd) + py.test.raises(OSError, rposix.pread, fd, 2, 1) + + at rposix_requires('pwrite') +def test_pwrite(): + fname = str(udir.join('os_test.txt')) + fd = os.open(fname, os.O_RDWR | os.O_CREAT, 0777) + try: + assert fd >= 0 + os.write(fd, b'Hello world') + os.lseek(fd, 0, 0) + rposix.pwrite(fd, b'ea', 1) + assert os.read(fd, 4) == b'Heal' + finally: + os.close(fd) + py.test.raises(OSError, rposix.pwrite, fd, b'ea', 1) From pypy.commits at gmail.com Mon Mar 13 12:56:21 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 09:56:21 -0700 (PDT) Subject: [pypy-commit] pypy default: missing return values Message-ID: <58c6cf35.090d2e0a.2f22a.42a2@mx.google.com> Author: Armin Rigo Branch: Changeset: r90664:b594a6c265f8 Date: 2017-03-13 17:55 +0100 http://bitbucket.org/pypy/pypy/changeset/b594a6c265f8/ Log: missing return values diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c --- a/pypy/module/cpyext/src/pymem.c +++ b/pypy/module/cpyext/src/pymem.c @@ -45,10 +45,21 @@ _PyPyGC_AddMemoryPressure(report); PyGILState_Release(state); } + + /* Should we return -2 or 0? In theory it should be -2, because + we're not using the info to really track the allocations. + But I'm sure someone is too clever somewhere and stops calling + _PyTraceMalloc_Track() if it returns -2. On the other hand, + returning 0 might lead to expectations that importing + 'tracemalloc' works on Python 3. Oh well, in that case we'll + just crash with ImportError during 'import tracemalloc'. + */ + return 0; } int _PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, uintptr_t ptr) { - /* nothing */ + /* nothing to do */ + return 0; } From pypy.commits at gmail.com Mon Mar 13 15:23:33 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 12:23:33 -0700 (PDT) Subject: [pypy-commit] pypy default: No, __thread doesn't work as we need it to, on OS/X Message-ID: <58c6f1b5.46162e0a.5c7a4.8e79@mx.google.com> Author: Armin Rigo Branch: Changeset: r90665:059c3a8c5232 Date: 2017-03-13 20:23 +0100 http://bitbucket.org/pypy/pypy/changeset/059c3a8c5232/ Log: No, __thread doesn't work as we need it to, on OS/X diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -26,10 +26,15 @@ SUPPORT__THREAD = ( # whether the particular C compiler supports __thread sys.platform.startswith("linux") or # Linux works - sys.platform.startswith("darwin")) # OS/X >= 10.7 works + #sys.platform.startswith("darwin") or # OS/X >= 10.7 works (*) + False) # Windows doesn't work. Please # add other platforms here if it works on them. +# (*) NOTE: __thread on OS/X does not work together with +# pthread_key_create(): when the destructor is called, the __thread is +# already freed! + MAINDIR = os.path.dirname(os.path.dirname(__file__)) CACHE_DIR = os.path.realpath(os.path.join(MAINDIR, '_cache')) From pypy.commits at gmail.com Mon Mar 13 17:49:08 2017 From: pypy.commits at gmail.com (mattip) Date: Mon, 13 Mar 2017 14:49:08 -0700 (PDT) Subject: [pypy-commit] pypy default: try and fail to reproduce issue #2482 Message-ID: <58c713d4.0e572e0a.45f13.391d@mx.google.com> Author: Matti Picus Branch: Changeset: r90666:78cdbb114c7a Date: 2017-03-13 23:23 +0200 http://bitbucket.org/pypy/pypy/changeset/78cdbb114c7a/ Log: try and fail to reproduce issue #2482 diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -972,12 +972,15 @@ pass class bar(f1, f2): pass + class foo(f2, f1): + pass assert bar.__base__ is f2 # On cpython, the size changes. if '__pypy__' in sys.builtin_module_names: assert module.size_of_instances(bar) == size else: assert module.size_of_instances(bar) >= size + assert module.size_of_instances(foo) == module.size_of_instances(bar) def test_app_cant_subclass_two_types(self): import sys From pypy.commits at gmail.com Tue Mar 14 02:55:22 2017 From: pypy.commits at gmail.com (arigo) Date: Mon, 13 Mar 2017 23:55:22 -0700 (PDT) Subject: [pypy-commit] pypy default: pom pom pom still trying Message-ID: <58c793da.5591190a.f755.0495@mx.google.com> Author: Armin Rigo Branch: Changeset: r90667:24e72d1a2755 Date: 2017-03-14 07:55 +0100 http://bitbucket.org/pypy/pypy/changeset/24e72d1a2755/ Log: pom pom pom still trying diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c --- a/pypy/module/cpyext/src/pymem.c +++ b/pypy/module/cpyext/src/pymem.c @@ -1,3 +1,7 @@ +#ifdef _WIN32 +# define _WIN32_WINNT 0x0501 +#endif + #include #ifdef _WIN32 From pypy.commits at gmail.com Tue Mar 14 03:35:47 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 00:35:47 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix (test by running "py.test -A" on pypy enough times until it crashes, Message-ID: <58c79d53.1459190a.79f3f.5e4c@mx.google.com> Author: Armin Rigo Branch: Changeset: r90668:5fd8b95992bb Date: 2017-03-14 08:35 +0100 http://bitbucket.org/pypy/pypy/changeset/5fd8b95992bb/ Log: Fix (test by running "py.test -A" on pypy enough times until it crashes, with options like MALLOC_CHECK_=7 to detect writes out of bound) diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -154,5 +154,10 @@ def test_tuple_subclass(self): module = self.import_module(name='foo') - a = module.TupleLike([1, 2, 3]) + a = module.TupleLike(range(100, 400, 100)) assert module.is_TupleLike(a) == 1 + assert isinstance(a, tuple) + assert issubclass(type(a), tuple) + assert list(a) == range(100, 400, 100) + assert list(a) == range(100, 400, 100) + assert list(a) == range(100, 400, 100) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -418,8 +418,8 @@ def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way - # (minimally, if tp_basicsize is zero we copy it from the base) - if not pto.c_tp_basicsize: + # (minimally, if tp_basicsize is zero or too low, we copy it from the base) + if pto.c_tp_basicsize < base_pto.c_tp_basicsize: pto.c_tp_basicsize = base_pto.c_tp_basicsize if pto.c_tp_itemsize < base_pto.c_tp_itemsize: pto.c_tp_itemsize = base_pto.c_tp_itemsize From pypy.commits at gmail.com Tue Mar 14 03:42:07 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 00:42:07 -0700 (PDT) Subject: [pypy-commit] pypy default: fix skips Message-ID: <58c79ecf.04abdf0a.6a930.33e0@mx.google.com> Author: Armin Rigo Branch: Changeset: r90669:2558b09410d8 Date: 2017-03-14 08:41 +0100 http://bitbucket.org/pypy/pypy/changeset/2558b09410d8/ Log: fix skips diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -1866,7 +1866,7 @@ def test_call_with_nested_anonymous_struct(self): import sys if sys.platform == 'win32': - py.test.skip("needs a GCC extension") + skip("needs a GCC extension") ffi, lib = self.prepare(""" struct foo { int a; union { int b, c; }; }; struct foo f(void); @@ -1914,6 +1914,9 @@ "set_source() and not taking a final '...' argument)") def test_call_with_zero_length_field(self): + import sys + if sys.platform == 'win32': + skip("zero-length field not supported by MSVC") ffi, lib = self.prepare(""" struct foo { int a; int x[0]; }; struct foo f(void); @@ -1959,7 +1962,7 @@ def test_call_with_packed_struct(self): import sys if sys.platform == 'win32': - py.test.skip("needs a GCC extension") + skip("needs a GCC extension") ffi, lib = self.prepare(""" struct foo { char y; int x; }; struct foo f(void); From pypy.commits at gmail.com Tue Mar 14 03:48:21 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 00:48:21 -0700 (PDT) Subject: [pypy-commit] pypy default: windows fixes Message-ID: <58c7a045.0d50190a.7769f.6129@mx.google.com> Author: Armin Rigo Branch: Changeset: r90670:7593b2f6ba5f Date: 2017-03-14 08:47 +0100 http://bitbucket.org/pypy/pypy/changeset/7593b2f6ba5f/ Log: windows fixes diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py --- a/pypy/module/cpyext/test/test_pystate.py +++ b/pypy/module/cpyext/test/test_pystate.py @@ -73,6 +73,7 @@ ("dance", "METH_NOARGS", """ PyThreadState *old_tstate, *new_tstate; + PyObject *d; PyEval_InitThreads(); @@ -81,7 +82,7 @@ return PyLong_FromLong(0); } - PyObject* d = PyThreadState_GetDict(); /* fails on cpython */ + d = PyThreadState_GetDict(); /* fails on cpython */ if (d != NULL) { return PyLong_FromLong(1); } @@ -144,6 +145,9 @@ ("bounce", "METH_NOARGS", """ PyThreadState * tstate; + PyObject *dict; + PyGILState_STATE gilstate; + if (PyEval_ThreadsInitialized() == 0) { PyEval_InitThreads(); @@ -152,11 +156,11 @@ if (tstate == NULL) { return PyLong_FromLong(0); } - PyObject* dict = PyThreadState_GetDict(); + dict = PyThreadState_GetDict(); if (dict != NULL) { return PyLong_FromLong(1); } - PyGILState_STATE gilstate = PyGILState_Ensure(); + gilstate = PyGILState_Ensure(); dict = PyThreadState_GetDict(); if (dict == NULL) { return PyLong_FromLong(2); diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py --- a/pypy/module/cpyext/test/test_userslots.py +++ b/pypy/module/cpyext/test/test_userslots.py @@ -160,8 +160,9 @@ }; ''', more_init=''' PyObject * mod = PyImport_ImportModule("datetime"); + PyObject * dt; if (mod == NULL) INITERROR; - PyObject * dt = PyString_FromString("datetime"); + dt = PyString_FromString("datetime"); datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod, dt); if (datetime_cls == NULL) INITERROR; _Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -464,28 +464,29 @@ how = SEEK_END return handle_posix_error('lseek', c_lseek(fd, pos, how)) -c_pread = external('pread', - [rffi.INT, rffi.VOIDP, rffi.SIZE_T , OFF_T], rffi.SSIZE_T, - save_err=rffi.RFFI_SAVE_ERRNO) -c_pwrite = external('pwrite', - [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T, - save_err=rffi.RFFI_SAVE_ERRNO) +if not _WIN32: + c_pread = external('pread', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T , OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) + c_pwrite = external('pwrite', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) - at enforceargs(int, int, None) -def pread(fd, count, offset): - if count < 0: - raise OSError(errno.EINVAL, None) - validate_fd(fd) - with rffi.scoped_alloc_buffer(count) as buf: - void_buf = rffi.cast(rffi.VOIDP, buf.raw) - return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) - - at enforceargs(int, None, None) -def pwrite(fd, data, offset): - count = len(data) - validate_fd(fd) - with rffi.scoped_nonmovingbuffer(data) as buf: - return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) + @enforceargs(int, int, None) + def pread(fd, count, offset): + if count < 0: + raise OSError(errno.EINVAL, None) + validate_fd(fd) + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) + + @enforceargs(int, None, None) + def pwrite(fd, data, offset): + count = len(data) + validate_fd(fd) + with rffi.scoped_nonmovingbuffer(data) as buf: + return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) From pypy.commits at gmail.com Tue Mar 14 03:58:18 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 00:58:18 -0700 (PDT) Subject: [pypy-commit] pypy default: windows fix Message-ID: <58c7a29a.04abdf0a.6a930.3834@mx.google.com> Author: Armin Rigo Branch: Changeset: r90671:bd26fe53021e Date: 2017-03-14 08:56 +0100 http://bitbucket.org/pypy/pypy/changeset/bd26fe53021e/ Log: windows fix diff --git a/pypy/module/_rawffi/alt/test/test_funcptr.py b/pypy/module/_rawffi/alt/test/test_funcptr.py --- a/pypy/module/_rawffi/alt/test/test_funcptr.py +++ b/pypy/module/_rawffi/alt/test/test_funcptr.py @@ -32,7 +32,10 @@ # c_file.write(py.code.Source('\n'.join(snippets))) eci = ExternalCompilationInfo(include_dirs=[cdir]) - return str(platform.compile([c_file], eci, 'x', standalone=False)) + # Windows note: can't reuse the same file name 'x.dll', because + # the previous one is likely still opened + return str(platform.compile([c_file], eci, 'x' + cls.__name__, + standalone=False)) def setup_class(cls): space = cls.space From pypy.commits at gmail.com Tue Mar 14 04:03:22 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 01:03:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: mark this test as failing Message-ID: <58c7a3ca.1a0e2e0a.ccb2e.62ab@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90672:f224cca395b7 Date: 2017-03-14 09:03 +0100 http://bitbucket.org/pypy/pypy/changeset/f224cca395b7/ Log: mark this test as failing diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -681,6 +681,10 @@ assert isinstance(list({b'a': 1})[0], bytes) def test_interned_keywords(self): + py.test.skip("no longer works") + # At some point in the past, we had kwargsdict automatically + # intern every single key we get out of it. That's a big + # pointless waste of time. So the following test fails now. assert list(dict(abcdef=1))[0] is 'abcdef' From pypy.commits at gmail.com Tue Mar 14 04:12:01 2017 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 14 Mar 2017 01:12:01 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: copy source code from github repo (pypy/fast-utf8-methods, 0a7e7ba813), add rpython wrapper to access c api Message-ID: <58c7a5d1.50152e0a.10dc8.ee66@mx.google.com> Author: Richard Plangger Branch: unicode-utf8 Changeset: r90673:5ca17ebc466d Date: 2017-03-14 09:10 +0100 http://bitbucket.org/pypy/pypy/changeset/5ca17ebc466d/ Log: copy source code from github repo (pypy/fast-utf8-methods, 0a7e7ba813), add rpython wrapper to access c api diff --git a/rpython/rlib/rutf8/capi.py b/rpython/rlib/rutf8/capi.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rutf8/capi.py @@ -0,0 +1,56 @@ +import py +import sys +from rpython.tool.version import rpythonroot +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.tool import rffi_platform as platform + +ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rutf8') +SRC = ROOT.join('src') + +if sys.platform.startswith('linux'): + _libs = ['dl'] +else: + _libs = [] +eci_kwds = dict( + include_dirs = [SRC], + includes = ['utf8.h'], + libraries = _libs, + separate_module_files = [SRC.join('utf8.c')],) +global_eci = ExternalCompilationInfo(**eci_kwds) + +IDXTAB = lltype.ForwardReference() +IDXTAB.become(rffi.CStruct("fu8_idxtab", + ('character_step', rffi.INT), + ('byte_positions', lltype.Ptr(rffi.SIZE_T)), + ('bytepos_table_length', rffi.SIZE_T))) +IDXTABPP = lltype.Ptr(lltype.Ptr(IDXTAB)) + +def setup(): + compile_extra = ['-DRPYTHON_LL2CTYPES'] + platform.verify_eci(ExternalCompilationInfo( + compile_extra=compile_extra, + **eci_kwds)) + + eci = global_eci + count_utf8_code_points = rffi.llexternal("fu8_count_utf8_codepoints", + [rffi.CCHARP, rffi.SIZE_T], + rffi.SSIZE_T, compilation_info=eci, + _nowrapper=True) + index2byteposition = rffi.llexternal("fu8_idx2bytepos", + [rffi.SIZE_T, rffi.CCHARP, rffi.SIZE_T, IDXTABPP], + rffi.SSIZE_T, compilation_info=eci, + _nowrapper=True) + + return CInterface(locals()) + + +class CInterface(object): + def __init__(self, namespace): + for k, v in namespace.iteritems(): + setattr(self, k, v) + + def _freeze_(self): + return True + + diff --git a/rpython/rlib/rutf8/src/utf8-avx.c b/rpython/rlib/rutf8/src/utf8-avx.c new file mode 100644 --- /dev/null +++ b/rpython/rlib/rutf8/src/utf8-avx.c @@ -0,0 +1,254 @@ +#include "utf8.h" + +#include +#include +#include +#include + +#define BIT(B,P) ((B >> (P-1)) & 0x1) + +void _print_mmy(const char * msg, __m256i chunk) +{ + printf("%s:", msg); + // unpack the first 8 bytes, padding with zeros + uint64_t a = _mm256_extract_epi64(chunk, 0); + uint64_t b = _mm256_extract_epi64(chunk, 1); + uint64_t c = _mm256_extract_epi64(chunk, 2); + uint64_t d = _mm256_extract_epi64(chunk, 3); + printf("%.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x " + "%.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x", + (unsigned char)((a >> 0) & 0xff), + (unsigned char)((a >> 8) & 0xff), + (unsigned char)((a >> 16) & 0xff), + (unsigned char)((a >> 24) & 0xff), + + (unsigned char)((a >> 32) & 0xff), + (unsigned char)((a >> 40) & 0xff), + (unsigned char)((a >> 48) & 0xff), + (unsigned char)((a >> 56) & 0xff), + + (unsigned char)((b >> 0) & 0xff), + (unsigned char)((b >> 8) & 0xff), + (unsigned char)((b >> 16) & 0xff), + (unsigned char)((b >> 24) & 0xff), + + (unsigned char)((b >> 32) & 0xff), + (unsigned char)((b >> 40) & 0xff), + (unsigned char)((b >> 48) & 0xff), + (unsigned char)((b >> 56) & 0xff), + + (unsigned char)((c >> 0) & 0xff), + (unsigned char)((c >> 8) & 0xff), + (unsigned char)((c >> 16) & 0xff), + (unsigned char)((c >> 24) & 0xff), + + (unsigned char)((c >> 32) & 0xff), + (unsigned char)((c >> 40) & 0xff), + (unsigned char)((c >> 48) & 0xff), + (unsigned char)((c >> 56) & 0xff), + + (unsigned char)((d >> 0) & 0xff), + (unsigned char)((d >> 8) & 0xff), + (unsigned char)((d >> 16) & 0xff), + (unsigned char)((d >> 24) & 0xff), + + (unsigned char)((d >> 32) & 0xff), + (unsigned char)((d >> 40) & 0xff), + (unsigned char)((d >> 48) & 0xff), + (unsigned char)((d >> 56) & 0xff) + ); + + printf("\n"); +} + +ssize_t count_utf8_codepoints_avx(const uint8_t * encoded, size_t len) +{ + __builtin_prefetch(encoded, 0, 0); + size_t num_codepoints = 0; + __m256i chunk; + + if (len == 0) { + return 0; + } + __m256i zero = _mm256_set1_epi8(0x00); + while (len >= 32) { + chunk = _mm256_loadu_si256((__m256i*)encoded); + if (_mm256_movemask_epi8(chunk) == 0) { + // valid ascii chars! + len -= 32; + encoded += 32; + num_codepoints += 32; + continue; + } + __builtin_prefetch(encoded+32, 0, 0); + + __m256i count = _mm256_set1_epi8(0x1); + //_print_mm256x("chunk", chunk); + // fight against the fact that there is no comparison on unsigned values + __m256i chunk_signed = _mm256_add_epi8(chunk, _mm256_set1_epi8(0x80)); + //_print_mm256x("shunk", chunk_signed); + + // ERROR checking + // checking procedure works the following way: + // + // 1) mark all continuation bytes with either 0x1, 0x3, 0x7 (one, two or three bytes continuation) + // 2) then check that there is no byte that has an invalid continuation + __m256i twobytemarker = _mm256_cmpgt_epi8( chunk_signed, _mm256_set1_epi8(0xc0-1-0x80)); + __m256i threebytemarker = _mm256_cmpgt_epi8(chunk_signed, _mm256_set1_epi8(0xe0-1-0x80)); + __m256i fourbytemarker = _mm256_cmpgt_epi8( chunk_signed, _mm256_set1_epi8(0xf0-1-0x80)); + + // the general idea of the following code collects 0xff for each byte position + // in the variable contbytes. + // at the end check if each position in contbytes set to 0xff is a valid continuation byte + + // check that 0xc0 > 0xc2 + __m256i validtwobm = _mm256_cmpgt_epi8(chunk_signed, _mm256_set1_epi8(0xc2-1-0x80)); + if (_mm256_movemask_epi8(_mm256_xor_si256(validtwobm, twobytemarker)) != 0) { + // two byte marker should not be in range [0xc0-0xc2) + return -1; + } + + __m256i state2 = _mm256_andnot_si256(threebytemarker, twobytemarker); + __m256i contbytes = _mm256_slli_si256(_mm256_blendv_epi8(state2, _mm256_set1_epi8(0x1), twobytemarker), 1); + + if (_mm256_movemask_epi8(threebytemarker) != 0) { + // contains at least one 3 byte marker + __m256i istate3 = _mm256_andnot_si256(fourbytemarker, threebytemarker); + __m256i state3 = _mm256_slli_si256(_mm256_blendv_epi8(zero, _mm256_set1_epi8(0x3), istate3), 1); + state3 = _mm256_or_si256(state3, _mm256_slli_si256(state3, 1)); + + contbytes = _mm256_or_si256(contbytes, state3); + + // range check + __m256i equal_e0 = _mm256_cmpeq_epi8(_mm256_blendv_epi8(zero, chunk_signed, istate3), + _mm256_set1_epi8(0xe0-0x80)); + if (_mm256_movemask_epi8(equal_e0) != 0) { + __m256i mask = _mm256_blendv_epi8(_mm256_set1_epi8(0x7f), chunk_signed, _mm256_slli_si256(equal_e0, 1)); + __m256i check_surrogate = _mm256_cmpgt_epi8(_mm256_set1_epi8(0xa0-0x80), mask); // lt + if (_mm256_movemask_epi8(check_surrogate) != 0) { + // invalid surrograte character!!! + return -1; + } + } + + // verify that there are now surrogates + if (!ALLOW_SURROGATES) { + __m256i equal_ed = _mm256_cmpeq_epi8(_mm256_blendv_epi8(zero, chunk_signed, istate3), + _mm256_set1_epi8(0xed-0x80)); + if (_mm256_movemask_epi8(equal_ed) != 0) { + __m256i mask = _mm256_blendv_epi8(_mm256_set1_epi8(0x80), chunk_signed, _mm256_slli_si256(equal_ed, 1)); + __m256i check_surrogate = _mm256_cmpgt_epi8(mask, _mm256_set1_epi8(0xa0-1-0x80)); + if (_mm256_movemask_epi8(check_surrogate) != 0) { + // invalid surrograte character!!! + return -1; + } + } + } + } + + if (_mm256_movemask_epi8(fourbytemarker) != 0) { + // contain a 4 byte marker + __m256i istate4 = _mm256_slli_si256(_mm256_blendv_epi8(zero, _mm256_set1_epi8(0x7), fourbytemarker), 1); + __m256i state4 =_mm256_or_si256(istate4, _mm256_slli_si256(istate4, 1)); + state4 =_mm256_or_si256(state4, _mm256_slli_si256(istate4, 2)); + + contbytes = _mm256_or_si256(contbytes, state4); + + // range check, filter out f0 and + __m256i equal_f0 = _mm256_cmpeq_epi8(_mm256_blendv_epi8(zero, chunk_signed, fourbytemarker), + _mm256_set1_epi8(0xf0-0x80)); + if (_mm256_movemask_epi8(equal_f0) != 0) { + __m256i mask = _mm256_blendv_epi8(_mm256_set1_epi8(0x7f), chunk_signed, _mm256_slli_si256(equal_f0, 1)); + __m256i check_surrogate = _mm256_cmpgt_epi8(_mm256_set1_epi8(0x90-0x80), mask); + if (_mm256_movemask_epi8(check_surrogate) != 0) { + return -1; + } + } + + __m256i equal_f4 = _mm256_cmpeq_epi8(_mm256_blendv_epi8(zero, chunk_signed, fourbytemarker), + _mm256_set1_epi8(0xf4-0x80)); + if (_mm256_movemask_epi8(equal_f4) != 0) { + __m256i mask = _mm256_blendv_epi8(_mm256_set1_epi8(0x80), chunk_signed, _mm256_slli_si256(equal_f4, 1)); + __m256i check_surrogate = _mm256_cmpgt_epi8(mask, _mm256_set1_epi8(0x90-1-0x80)); + if (_mm256_movemask_epi8(check_surrogate) != 0) { + return -1; + } + } + + __m256i equal_f5_gt = _mm256_cmpgt_epi8(_mm256_blendv_epi8(zero, chunk_signed, fourbytemarker), + _mm256_set1_epi8(0xf4-0x80)); + if (_mm256_movemask_epi8(equal_f5_gt) != 0) { + return -1; + } + } + + // now check that contbytes and the actual byte values have a valid + // continuation at each position the marker indicates to have one + __m256i check_cont = _mm256_cmpgt_epi8(contbytes, zero); + __m256i contpos = _mm256_and_si256(_mm256_set1_epi8(0xc0), chunk); + contpos = _mm256_cmpeq_epi8(_mm256_set1_epi8(0x80), contpos); + __m256i validcont = _mm256_xor_si256(check_cont, contpos); + if (_mm256_movemask_epi8(validcont) != 0) { + // uff, nope, that is really not utf8 + return -1; + } + + // CORRECT, calculate the length + // copy 0x00 over to each place which is a continuation byte + count = _mm256_blendv_epi8(count, zero, contpos); + + // count the code points using 2x 32 bit hadd and one last 16 hadd + // the result will end up at the lowest position + count = _mm256_hadd_epi32(count, zero); + count = _mm256_hadd_epi32(count, zero); + count = _mm256_hadd_epi16(count, zero); + uint16_t c = _mm256_extract_epi16(count, 0); + uint16_t c2 = _mm256_extract_epi16(count, 8); + uint16_t points = (c & 0xff) + ((c >> 8) & 0xff) + (c2 & 0xff) + ((c2 >> 8) & 0xff); + + // these cases need to be handled: + // 16 byte boundary -> | <- 16 byte boundary + // -----------------------------------------+-------------------- + // 1) 2 byte code point. e.g. ... c2 | 80 ... + // 2) 3 byte code point. e.g. ... e6 | 80 80 ... + // 3) 3 byte code point. e.g. ... e6 80 | 80 ... + // 4) 4 byte code point. e.g. ... f2 | 80 80 80 ... + // 5) 4 byte code point. e.g. ... f2 80 | 80 80 ... + // 6) 4 byte code point. e.g. ... f2 80 80 | 80 ... + // + int mask_chunk = _mm256_movemask_epi8(chunk); + int mask_conti = _mm256_movemask_epi8(contpos); + + // little endian + int lenoff = 32; + int minus_codepoints = 0; + if (BIT(mask_chunk, 32) != 0 && BIT(mask_conti, 32) == 0) { // 1), 2), 4) + minus_codepoints = 1; + lenoff -= 1; + } else if (BIT(mask_chunk, 31) != 0 && BIT(mask_conti, 31) == 0 && + BIT(mask_conti, 32) == 1) { // 3), 5) + minus_codepoints = 1; + lenoff -= 2; + } else if (BIT(mask_chunk, 30) != 0 && BIT(mask_conti, 30) == 0 && + BIT(mask_conti, 31) == 1 && BIT(mask_conti, 32) == 1) { // 6) + minus_codepoints = 1; + lenoff -= 3; + } + + num_codepoints += points - minus_codepoints; + len -= lenoff; + encoded += lenoff; + } + + if (len == 0) { + return num_codepoints; + } + + ssize_t result = count_utf8_codepoints_seq(encoded, len); + if (result == -1) { + return -1; + } + + return num_codepoints + result; + return -1; +} diff --git a/rpython/rlib/rutf8/src/utf8-scalar.c b/rpython/rlib/rutf8/src/utf8-scalar.c new file mode 100644 --- /dev/null +++ b/rpython/rlib/rutf8/src/utf8-scalar.c @@ -0,0 +1,70 @@ +#include "utf8.h" + +int _check_continuation(const uint8_t ** encoded, const uint8_t * endptr, int count) { + ssize_t size = endptr - *encoded; + + if (size < count) { + // not enough bytes to be a valid 2 byte utf8 code point + return -1; + } + for (int i = 0; i < count; i++) { + uint8_t byte = *(*encoded)++; + if ((byte & 0xc0) != 0x80) { + // continuation byte does NOT match 0x10xxxxxx + return -1; + } + } + return 0; +} + +ssize_t count_utf8_codepoints_seq(const uint8_t * encoded, size_t len) { + size_t num_codepoints = 0; + uint8_t byte = 0; + const uint8_t * endptr = encoded + len; + + while (encoded < endptr) { + byte = *encoded++; + if (byte < 0x80) { + num_codepoints += 1; + continue; + } else { + //asm("int $3"); + if ((byte & 0xe0) == 0xc0) { + // one continuation byte + if (byte < 0xc2) { + return -1; + } + if (_check_continuation(&encoded, endptr, 1) != 0) { + return -1; + } + } else if ((byte & 0xf0) == 0xe0) { + // two continuation byte + if (_check_continuation(&encoded, endptr, 2) != 0) { + return -1; + } + uint8_t byte1 = encoded[-2]; + //surrogates shouldn't be valid UTF-8! + if ((byte == 0xe0 && byte1 < 0xa0) || + (byte == 0xed && byte1 > 0x9f && !ALLOW_SURROGATES)) { + return -1; + } + } else if ((byte & 0xf8) == 0xf0) { + // three continuation byte + if (_check_continuation(&encoded, endptr, 3) != 0) { + return -1; + } + uint8_t byte1 = encoded[-3]; + if ((byte == 0xf0 && byte1 < 0x90) || + (byte == 0xf4 && byte1 > 0x8f) || + (byte >= 0xf5)) { + return -1; + } + } else { + // TODO + return -1; + } + num_codepoints += 1; + } + } + return num_codepoints; +} diff --git a/rpython/rlib/rutf8/src/utf8-sse4.c b/rpython/rlib/rutf8/src/utf8-sse4.c new file mode 100644 --- /dev/null +++ b/rpython/rlib/rutf8/src/utf8-sse4.c @@ -0,0 +1,238 @@ +#include "utf8.h" + +#include +#include +#include +#include +#include + +#define BIT(B,P) ((B >> (P-1)) & 0x1) + +void _print_mmx(const char * msg, __m128i chunk) +{ + printf("%s:", msg); + // unpack the first 8 bytes, padding with zeros + uint64_t a = _mm_extract_epi64(chunk, 0); + uint64_t b = _mm_extract_epi64(chunk, 1); + printf("%.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x %.2x%.2x%.2x%.2x", + (unsigned char)((a >> 0) & 0xff), + (unsigned char)((a >> 8) & 0xff), + (unsigned char)((a >> 16) & 0xff), + (unsigned char)((a >> 24) & 0xff), + + (unsigned char)((a >> 32) & 0xff), + (unsigned char)((a >> 40) & 0xff), + (unsigned char)((a >> 48) & 0xff), + (unsigned char)((a >> 56) & 0xff), + + (unsigned char)((b >> 0) & 0xff), + (unsigned char)((b >> 8) & 0xff), + (unsigned char)((b >> 16) & 0xff), + (unsigned char)((b >> 24) & 0xff), + + (unsigned char)((b >> 32) & 0xff), + (unsigned char)((b >> 40) & 0xff), + (unsigned char)((b >> 48) & 0xff), + (unsigned char)((b >> 56) & 0xff) + ); + + printf("\n"); +} + + +ssize_t count_utf8_codepoints_sse4(const uint8_t * encoded, size_t len) +{ + __builtin_prefetch(encoded, 0, 0); + size_t num_codepoints = 0; + __m128i chunk; + + if (len == 0) { + return 0; + } + __m128i zero = _mm_set1_epi8(0x00); + + while (len >= 16) { + chunk = _mm_loadu_si128((__m128i*)encoded); + if (_mm_movemask_epi8(chunk) == 0) { + // valid ascii chars! + len -= 16; + encoded += 16; + num_codepoints += 16; + continue; + } + __builtin_prefetch(encoded+16, 0, 0); + + __m128i count = _mm_set1_epi8(0x1); + //_print_mmx("chunk", chunk); + // fight against the fact that there is no comparison on unsigned values + __m128i chunk_signed = _mm_add_epi8(chunk, _mm_set1_epi8(0x80)); + //_print_mmx("shunk", chunk_signed); + + // ERROR checking + // checking procedure works the following way: + // + // 1) mark all continuation bytes with either 0x1, 0x3, 0x7 (one, two or three bytes continuation) + // 2) then check that there is no byte that has an invalid continuation + __m128i twobytemarker = _mm_cmplt_epi8(_mm_set1_epi8(0xc0-1-0x80), chunk_signed); + __m128i threebytemarker = _mm_cmplt_epi8(_mm_set1_epi8(0xe0-1-0x80), chunk_signed); + __m128i fourbytemarker = _mm_cmplt_epi8(_mm_set1_epi8(0xf0-1-0x80), chunk_signed); + + // the general idea of the following code collects 0xff for each byte position + // in the variable contbytes. + // at the end check if each position in contbytes set to 0xff is a valid continuation byte + + // check that 0xc0 > 0xc2 + __m128i validtwobm = _mm_cmplt_epi8(_mm_set1_epi8(0xc2-1-0x80), chunk_signed); + if (_mm_movemask_epi8(_mm_xor_si128(validtwobm, twobytemarker)) != 0) { + // two byte marker should not be in range [0xc0-0xc2) + return -1; + } + + __m128i state2 = _mm_andnot_si128(threebytemarker, twobytemarker); + __m128i contbytes = _mm_slli_si128(_mm_blendv_epi8(state2, _mm_set1_epi8(0x1), twobytemarker), 1); + + if (_mm_movemask_epi8(threebytemarker) != 0) { + // contains at least one 3 byte marker + __m128i istate3 = _mm_andnot_si128(fourbytemarker, threebytemarker); + __m128i state3 = _mm_slli_si128(_mm_blendv_epi8(zero, _mm_set1_epi8(0x3), istate3), 1); + state3 = _mm_or_si128(state3, _mm_slli_si128(state3, 1)); + + contbytes = _mm_or_si128(contbytes, state3); + + // range check + __m128i equal_e0 = _mm_cmpeq_epi8(_mm_blendv_epi8(zero, chunk_signed, istate3), + _mm_set1_epi8(0xe0-0x80)); + if (_mm_movemask_epi8(equal_e0) != 0) { + __m128i mask = _mm_blendv_epi8(_mm_set1_epi8(0x7f), chunk_signed, _mm_slli_si128(equal_e0, 1)); + __m128i check_surrogate = _mm_cmplt_epi8(mask, _mm_set1_epi8(0xa0-0x80)); + if (_mm_movemask_epi8(check_surrogate) != 0) { + // invalid surrograte character!!! + return -1; + } + } + + // verify that there are now surrogates + if (!ALLOW_SURROGATES) { + __m128i equal_ed = _mm_cmpeq_epi8(_mm_blendv_epi8(zero, chunk_signed, istate3), + _mm_set1_epi8(0xed-0x80)); + if (_mm_movemask_epi8(equal_ed) != 0) { + __m128i mask = _mm_blendv_epi8(_mm_set1_epi8(0x80), chunk_signed, _mm_slli_si128(equal_ed, 1)); + __m128i check_surrogate = _mm_cmpgt_epi8(mask, _mm_set1_epi8(0xa0-1-0x80)); + if (_mm_movemask_epi8(check_surrogate) != 0) { + // invalid surrograte character!!! + return -1; + } + } + } + } + + if (_mm_movemask_epi8(fourbytemarker) != 0) { + // contain a 4 byte marker + __m128i istate4 = _mm_slli_si128(_mm_blendv_epi8(zero, _mm_set1_epi8(0x7), fourbytemarker), 1); + __m128i state4 =_mm_or_si128(istate4, _mm_slli_si128(istate4, 1)); + state4 =_mm_or_si128(state4, _mm_slli_si128(istate4, 2)); + + contbytes = _mm_or_si128(contbytes, state4); + + // range check, filter out f0 and + __m128i equal_f0 = _mm_cmpeq_epi8(_mm_blendv_epi8(zero, chunk_signed, fourbytemarker), + _mm_set1_epi8(0xf0-0x80)); + if (_mm_movemask_epi8(equal_f0) != 0) { + __m128i mask = _mm_blendv_epi8(_mm_set1_epi8(0x7f), chunk_signed, _mm_slli_si128(equal_f0, 1)); + __m128i check_surrogate = _mm_cmplt_epi8(mask, _mm_set1_epi8(0x90-0x80)); + if (_mm_movemask_epi8(check_surrogate) != 0) { + return -1; + } + } + + __m128i equal_f4 = _mm_cmpeq_epi8(_mm_blendv_epi8(zero, chunk_signed, fourbytemarker), + _mm_set1_epi8(0xf4-0x80)); + if (_mm_movemask_epi8(equal_f4) != 0) { + __m128i mask = _mm_blendv_epi8(_mm_set1_epi8(0x80), chunk_signed, _mm_slli_si128(equal_f4, 1)); + __m128i check_surrogate = _mm_cmpgt_epi8(mask, _mm_set1_epi8(0x90-1-0x80)); + if (_mm_movemask_epi8(check_surrogate) != 0) { + return -1; + } + } + + __m128i equal_f5_gt = _mm_cmpgt_epi8(_mm_blendv_epi8(zero, chunk_signed, fourbytemarker), + _mm_set1_epi8(0xf4-0x80)); + if (_mm_movemask_epi8(equal_f5_gt) != 0) { + return -1; + } + } + + // now check that contbytes and the actual byte values have a valid + // continuation at each position the marker indicates to have one + __m128i check_cont = _mm_cmpgt_epi8(contbytes, zero); + __m128i contpos = _mm_and_si128(_mm_set1_epi8(0xc0), chunk); + contpos = _mm_cmpeq_epi8(_mm_set1_epi8(0x80), contpos); + __m128i validcont = _mm_xor_si128(check_cont, contpos); + if (_mm_movemask_epi8(validcont) != 0) { + // uff, nope, that is really not utf8 + return -1; + } + + // CORRECT, calculate the length + // copy 0x00 over to each place which is a continuation byte + count = _mm_blendv_epi8(count, zero, contpos); + + // count the code points using 2x 32 bit hadd and one last 16 hadd + // the result will end up at the lowest position + count = _mm_hadd_epi32(count, count); + count = _mm_hadd_epi32(count, count); + count = _mm_hadd_epi16(count, count); + uint16_t c = _mm_extract_epi16(count, 0); + + // these cases need to be handled: + // 16 byte boundary -> | <- 16 byte boundary + // -----------------------------------------+-------------------- + // 1) 2 byte code point. e.g. ... c2 | 80 ... + // 2) 3 byte code point. e.g. ... e6 | 80 80 ... + // 3) 3 byte code point. e.g. ... e6 80 | 80 ... + // 4) 4 byte code point. e.g. ... f2 | 80 80 80 ... + // 5) 4 byte code point. e.g. ... f2 80 | 80 80 ... + // 6) 4 byte code point. e.g. ... f2 80 80 | 80 ... + // + int mask_chunk = _mm_movemask_epi8(chunk); + int mask_conti = _mm_movemask_epi8(contpos); + + // little endian + int lenoff = 16; + int minus_codepoints = 0; + if (BIT(mask_chunk, 16) != 0 && BIT(mask_conti, 16) == 0) { // 1), 2), 4) + minus_codepoints = 1; + lenoff -= 1; + } else if (BIT(mask_chunk, 15) != 0 && BIT(mask_conti, 15) == 0 && + BIT(mask_conti, 16) == 1) { // 3), 5) + minus_codepoints = 1; + lenoff -= 2; + } else if (BIT(mask_chunk, 14) != 0 && BIT(mask_conti, 14) == 0 && + BIT(mask_conti, 15) == 1 && BIT(mask_conti, 16) == 1) { // 6) + minus_codepoints = 1; + lenoff -= 3; + } + + num_codepoints += (c & 0xff) + ((c >> 8) & 0xff) - minus_codepoints; + len -= lenoff; + encoded += lenoff; + } + + if (len == 0) { + return num_codepoints; + } + + ssize_t result = count_utf8_codepoints_seq(encoded, len); + if (result == -1) { + return -1; + } + + return num_codepoints + result; +} + +ssize_t fu8_idx2bytepos_sse4(size_t index, + const uint8_t * utf8, size_t len, + struct fu8_idxtab * t) +{ + return 0; +} diff --git a/rpython/rlib/rutf8/src/utf8.c b/rpython/rlib/rutf8/src/utf8.c new file mode 100644 --- /dev/null +++ b/rpython/rlib/rutf8/src/utf8.c @@ -0,0 +1,240 @@ +#include "utf8.h" + +#include +#include + +#include "utf8-scalar.c" // copy code for scalar operations + + +int instruction_set = -1; +#define ISET_SSE4 0x1 +#define ISET_AVX 0x2 +#define ISET_AVX2 0x4 + +void detect_instructionset(void) +{ + long eax; + long ebx; + long ecx; + long edx; + long op = 1; + asm ("cpuid" + : "=a" (eax), + "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "a" (op)); + + instruction_set = 0; + if (ecx & (1<<19)) { // sse4.1 + instruction_set |= ISET_SSE4; + } + if(__builtin_cpu_supports("avx")) { + instruction_set |= ISET_AVX; + } + if(__builtin_cpu_supports("avx2")) { + instruction_set |= ISET_AVX2; + } +} + +ssize_t count_utf8_codepoints(const uint8_t * encoded, size_t len) +{ + if (instruction_set == -1) { + detect_instructionset(); + } + + if (len >= 32 && (instruction_set & ISET_AVX2) != 0) { + // to the MOON! + return count_utf8_codepoints_avx(encoded, len); + } + if (len >= 16 && (instruction_set == ISET_SSE4) != 0) { + // speed!! + return count_utf8_codepoints_sse4(encoded, len); + } + + // oh no, just do it sequentially! + return count_utf8_codepoints_seq(encoded, len); +} + +typedef struct fu8_idxtab { + int character_step; + size_t * byte_positions; + size_t bytepos_table_length; +} fu8_idxtab_t; + +#include + +fu8_idxtab_t * _fu8_alloc_idxtab(int cp_count, int character_step) +{ + if (cp_count <= character_step) { + return NULL; + } + long s = (cp_count/character_step) * sizeof(size_t); + char * c = calloc(1, sizeof(fu8_idxtab_t)+s); + fu8_idxtab_t * i = (fu8_idxtab_t*)c; + i->character_step = character_step; + i->byte_positions = (size_t*)(c + sizeof(fu8_idxtab_t)); + i->bytepos_table_length = cp_count/character_step; + return i; +} + +void fu8_free_idxtab(struct fu8_idxtab * t) +{ + // why manage this in C? + // it might at some point have a different data structure, + // then we can handle this easily here without modifying the API + free(t); t = NULL; +} + +void _fu8_itab_set_bucket(struct fu8_idxtab * tab, int bucket, size_t off, size_t cpidx) +{ + size_t oldval = tab->byte_positions[bucket]; + if (oldval != 0) { + assert(oldval != off && "table mismatch"); + } + assert(bucket >= 0 && bucket < tab->bytepos_table_length && "index out of bounds"); + tab->byte_positions[bucket] = off; +} + +ssize_t _fu8_build_idxtab(size_t cpidx, size_t cpidx_off, size_t cplen, + const uint8_t * utf8, size_t bytelen, size_t byteoff, + struct fu8_idxtab ** tab) { + size_t code_point_index = cpidx_off; + const uint8_t * utf8_start_position = utf8 + byteoff; + const uint8_t * utf8_end_position = utf8 + bytelen - byteoff; + + struct fu8_idxtab * itab = tab[0]; + if (itab == NULL) { + tab[0] = itab = _fu8_alloc_idxtab(cplen, 1000); + } + + int bucket_step = -1; + int bucket = -1; + if (itab) { + bucket_step = itab->character_step; + bucket = cpidx_off / bucket_step; + //printf("bucket %d step %d iindex_off %ld\n", bucket, bucket_step, cpidx_off); + } + + while (utf8 < utf8_end_position) { + //printf("%d %llx ok\n", code_point_index, utf8); + if (code_point_index == cpidx) { + //printf("return %llx %llx %llx\n", utf8_start_position, utf8, utf8_end_position); + return utf8 - utf8_start_position; + } + + if (bucket_step != -1 && code_point_index != 0 && (code_point_index % bucket_step) == 0) { + _fu8_itab_set_bucket(itab, bucket++, byteoff + utf8 - utf8_start_position, code_point_index); + } + + uint8_t c = *utf8++; + //printf("%x\n", c); + code_point_index += 1; + if ((c & 0xc0) == 0) { + continue; + } + if ((c & 0xe0) == 0xc0) { + utf8 += 1; + continue; + } + if ((c & 0xf0) == 0xe0) { + utf8 += 2; + continue; + } + if ((c & 0xf8) == 0xf0) { + utf8 += 3; + continue; + } + } + + return -1; // out of bounds!! +} + +size_t _fu8_idxtab_lookup_bytepos_i(struct fu8_idxtab * tab, size_t cpidx); + +ssize_t _fu8_idx2bytepos(size_t index, + const uint8_t * utf8, size_t bytelen, size_t cplen, + struct fu8_idxtab ** tab) +{ + + assert(index != 0 && "index must not be 0"); + // note that itab STILL can be NULL + +} + +size_t _fu8_idxtab_lookup_bytepos_i(struct fu8_idxtab * tab, size_t cpidx) +{ + if (cpidx == 0 || tab == NULL) { + return 0; + } + int step = tab->character_step; + int tidx = cpidx / step; + size_t val = tab->byte_positions[tidx]; + while (tidx > 0) { + if (val != 0) { + //printf("%llx at %d %d/%d\n", val, tidx, cpidx, step); + return val; + } + tidx--; + val = tab->byte_positions[tidx]; + } + // no clue, start at the beginning! + return 0; + + //int lp, rp; // left position, right position + //int mp; // middle position + //int count; + //lp = 0; + //rp = 16; + + //if (cpidx == 0) { + // return -1; + //} + + //size_t valid_left = -1; + + //do { + // count = (rp - lp); + // mp = lp + count / 2; + + // size_t lval = tab->codepoint_positions[lp]; + // size_t mval = tab->codepoint_positions[mp]; + // size_t rval = tab->codepoint_positions[rp]; + // printf("l %d m %d r %d\nlv %d mv %d rv %d\n", lp, mp, rp, lval, mval, rval); + // if (lval != 0 && lval <= cpidx) { + // valid_left = lp; + // } else if (lval == 0) { + // // nothing is known about the left most value + // break; + // } + + // if (mval == cpidx) { + // return mp; + // } + + // if (mval == 0 || mval < cpidx) { + // // nothing is known about the middle value, + // // or mval is smaller the searched code point index + // rp = mp; + // continue; + // } else { + // lp = mp; + // continue; + // } + + //} while (count > 1); + + //return valid_left; +} + +ssize_t fu8_idx2bytepos(size_t index, + const uint8_t * utf8, size_t bytelen, + size_t cplen, + struct fu8_idxtab ** tab) +{ + if (index == 0) { return 0; } + if (index >= cplen) { return -1; } + size_t off = _fu8_idxtab_lookup_bytepos_i(tab[0], index); + //printf("found %llx\n", off); + return _fu8_build_idxtab(index, 0, cplen, utf8, bytelen, 0, tab); +} diff --git a/rpython/rlib/rutf8/src/utf8.h b/rpython/rlib/rutf8/src/utf8.h new file mode 100644 --- /dev/null +++ b/rpython/rlib/rutf8/src/utf8.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include + +/** + * Returns -1 if the given string is not a valid utf8 encoded string. + * Otherwise returns the amount code point in the given string. + * len: length in bytes (8-bit) + * + * The above documentation also applies for several vectorized implementations + * found below. + * + * count_utf8_codepoints dispatches amongst several + * implementations (e.g. seq, SSE4, AVX) + */ +// TODO rename (fu8 prefix) +ssize_t fu8_count_utf8_codepoints(const uint8_t * encoded, size_t len); +ssize_t fu8_count_utf8_codepoints_seq(const uint8_t * encoded, size_t len); +ssize_t fu8_count_utf8_codepoints_sse4(const uint8_t * encoded, size_t len); +ssize_t fu8_count_utf8_codepoints_avx(const uint8_t * encoded, size_t len); + + +struct fu8_idxtab; + +/** + * Looks up the byte position of the utf8 code point at the index. + * Assumptions: + * + * * utf8 parameter is utf8 encoded, otherwise the result is undefined. + * * passing one struct fu8_idxtab instance to several different utf8 strings + * yields undefined behaviour + * + * Return values: + * + * -1, if the index is out of bounds of utf8 + * X, where X >= 0. X is the byte postion for the code point at index + * + * If table is not NULL, this routine builds up a lookup + * table to speed up indexing. + * + */ +ssize_t fu8_idx2bytepos(size_t index, + const uint8_t * utf8, size_t bytelen, + size_t cplen, + struct fu8_idxtab ** tab); +void fu8_free_idxtab(struct fu8_idxtab * t); +ssize_t fu8_idx2bytepso_sse4(size_t index, + const uint8_t * utf8, size_t len, + struct fu8_idxtab ** t); From pypy.commits at gmail.com Tue Mar 14 06:51:56 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 03:51:56 -0700 (PDT) Subject: [pypy-commit] pypy default: Factor out these three lines into a helper Message-ID: <58c7cb4c.c819190a.7baba.6163@mx.google.com> Author: Armin Rigo Branch: Changeset: r90674:44f31f6dd39f Date: 2017-03-14 11:51 +0100 http://bitbucket.org/pypy/pypy/changeset/44f31f6dd39f/ Log: Factor out these three lines into a helper diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -355,9 +355,7 @@ mc.RET() # # patch the JNZ above - offset = mc.get_relative_pos() - jnz_location - assert 0 < offset <= 127 - mc.overwrite(jnz_location-1, chr(offset)) + mc.patch_forward_jump(jnz_location) # From now on this function is basically "merged" with # its caller and so contains DEFAULT_FRAME_BYTES bytes # plus my own return address, which we'll ignore next @@ -841,9 +839,7 @@ self.push_gcmap(mc, gcmap, store=True) mc.CALL(imm(self._frame_realloc_slowpath)) # patch the JG above - offset = mc.get_relative_pos() - jg_location - assert 0 < offset <= 127 - mc.overwrite(jg_location-1, chr(offset)) + mc.patch_forward_jump(jg_location) self.frame_depth_to_patch.append(stack_check_cmp_ofs) self.frame_depth_to_patch.append(ofs2) @@ -864,9 +860,7 @@ ofs2 = mc.get_relative_pos() - 4 mc.CALL(imm(self.cpu.realloc_frame_crash)) # patch the JG above - offset = mc.get_relative_pos() - jg_location - assert 0 < offset <= 127 - mc.overwrite(jg_location-1, chr(offset)) + mc.patch_forward_jump(jg_location) self.frame_depth_to_patch.append(stack_check_cmp_ofs) self.frame_depth_to_patch.append(ofs2) @@ -1006,9 +1000,7 @@ jb_location = self.mc.get_relative_pos() self.mc.CALL(imm(self.stack_check_slowpath))# CALL slowpath # patch the JB above # .skip: - offset = self.mc.get_relative_pos() - jb_location - assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, chr(offset)) + self.mc.patch_forward_jump(jb_location) # def _call_footer(self): @@ -1248,9 +1240,7 @@ # always clear zero and carry self.mc.CMP_ri(ebp.value, 0) # patch the JNP above - offset = self.mc.get_relative_pos() - jnp_location - assert 0 < offset <= 127 - self.mc.overwrite(jnp_location-1, chr(offset)) + self.mc.patch_forward_jump(jnp_location) def _cmpop_float(cond, rev_cond): is_ne = cond == 'NE' @@ -1731,7 +1721,7 @@ jmp_adr = self.previous_cond_call_jcond offset = self.mc.get_relative_pos() - jmp_adr if offset <= 127: - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(jmp_adr) def genop_guard_guard_not_invalidated(self, guard_op, guard_token, locs, ign): @@ -1846,9 +1836,7 @@ jb_location = self.mc.get_relative_pos() self._cmp_guard_class(locs) # patch the JB above - offset = self.mc.get_relative_pos() - jb_location - assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, chr(offset)) + self.mc.patch_forward_jump(jb_location) # self.guard_success_cc = rx86.Conditions['E'] self.implement_guard(guard_token) @@ -2233,9 +2221,7 @@ self.mc.JMP_l8(0) # jump to done, patched later jmp_location = self.mc.get_relative_pos() # - offset = jmp_location - je_location - assert 0 < offset <= 127 - self.mc.overwrite(je_location - 1, chr(offset)) + self.mc.patch_forward_jump(je_location) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) # return jmp_location @@ -2255,9 +2241,7 @@ self.mc.MOV_rm(eax.value, (eax.value, ofs)) def _call_assembler_patch_jmp(self, jmp_location): - offset = self.mc.get_relative_pos() - jmp_location - assert 0 <= offset <= 127 - self.mc.overwrite(jmp_location - 1, chr(offset)) + self.mc.patch_forward_jump(jmp_location) # ------------------- END CALL ASSEMBLER ----------------------- @@ -2330,9 +2314,7 @@ jns_location = mc.get_relative_pos() # # patch the JS above - offset = mc.get_relative_pos() - js_location - assert 0 < offset <= 127 - mc.overwrite(js_location-1, chr(offset)) + mc.patch_forward_jump(js_location) # # case GCFLAG_CARDS_SET: emit a few instructions to do # directly the card flag setting @@ -2367,14 +2349,10 @@ raise AssertionError("index is neither RegLoc nor ImmedLoc") # # patch the JNS above - offset = mc.get_relative_pos() - jns_location - assert 0 < offset <= 127 - mc.overwrite(jns_location-1, chr(offset)) + mc.patch_forward_jump(jns_location) # patch the JZ above - offset = mc.get_relative_pos() - jz_location - assert 0 < offset <= 127 - mc.overwrite(jz_location-1, chr(offset)) + mc.patch_forward_jump(jz_location) def genop_discard_cond_call_gc_wb(self, op, arglocs): self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs) @@ -2459,9 +2437,7 @@ v = gpr_reg_mgr_cls.all_reg_indexes[eax.value] self.mc.MOV_rb(eax.value, v * WORD + base_ofs) # - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(jmp_adr) # might be overridden again to skip over the following # guard_no_exception too self.previous_cond_call_jcond = jmp_adr @@ -2476,9 +2452,7 @@ # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(jmp_adr) self.mc.MOV(heap(nursery_free_adr), edx) def malloc_cond_varsize_frame(self, nursery_free_adr, nursery_top_adr, @@ -2497,9 +2471,7 @@ # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(jmp_adr) self.mc.MOV(heap(nursery_free_adr), edx) def malloc_cond_varsize(self, kind, nursery_free_adr, nursery_top_adr, @@ -2545,9 +2517,7 @@ self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr1 = self.mc.get_relative_pos() # - offset = self.mc.get_relative_pos() - jmp_adr0 - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr0-1, chr(offset)) + self.mc.patch_forward_jump(jmp_adr0) # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) if kind == rewrite.FLAG_ARRAY: @@ -2566,18 +2536,14 @@ self.mc.JMP_l8(0) # jump to done, patched later jmp_location = self.mc.get_relative_pos() # - offset = self.mc.get_relative_pos() - jmp_adr1 - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr1-1, chr(offset)) + self.mc.patch_forward_jump(jmp_adr1) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) # write down the tid, but not if it's the result of the CALL self.mc.MOV(mem(ecx, 0), imm(arraydescr.tid)) # while we're at it, this line is not needed if we've done the CALL self.mc.MOV(heap(nursery_free_adr), edx) # - offset = self.mc.get_relative_pos() - jmp_location - assert 0 < offset <= 127 - self.mc.overwrite(jmp_location - 1, chr(offset)) + self.mc.patch_forward_jump(jmp_location) def store_force_descr(self, op, fail_locs, frame_depth): guard_token = self.implement_guard_recovery(op.opnum, diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -354,9 +354,7 @@ # general 'reacqgil_addr' below can acquire it again... mc.MOV(heap(fastgil), ecx) # patch the JNE above - offset = mc.get_relative_pos() - jne_location - assert 0 < offset <= 127 - mc.overwrite(jne_location-1, chr(offset)) + mc.patch_forward_jump(jne_location) else: mc.J_il8(rx86.Conditions['E'], 0) je_location = mc.get_relative_pos() @@ -374,9 +372,7 @@ self.restore_result_value(save_edx=False) # # patch the JE above - offset = mc.get_relative_pos() - je_location - assert 0 < offset <= 127 - mc.overwrite(je_location-1, chr(offset)) + mc.patch_forward_jump(je_location) # if restore_edx: mc.MOV_rs(edx.value, 12) # restore this diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py --- a/rpython/jit/backend/x86/codebuf.py +++ b/rpython/jit/backend/x86/codebuf.py @@ -53,3 +53,8 @@ adr[0] = rffi.cast(rffi.INT, intmask(adr[0]) - p) valgrind.discard_translations(addr, self.get_relative_pos()) self._dump(addr, "jit-backend-dump", backend_name) + + def patch_forward_jump(self, jcond_location): + offset = self.get_relative_pos() - jcond_location + assert 0 <= offset <= 127 + self.overwrite(jcond_location-1, chr(offset)) From pypy.commits at gmail.com Tue Mar 14 07:28:01 2017 From: pypy.commits at gmail.com (mattip) Date: Tue, 14 Mar 2017 04:28:01 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Merge release-pypy3.3-5.x into py3.5 Message-ID: <58c7d3c1.0be3190a.2ca08.686e@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r90675:a1ba0fb0c9fe Date: 2017-03-14 13:26 +0200 http://bitbucket.org/pypy/pypy/changeset/a1ba0fb0c9fe/ Log: Merge release-pypy3.3-5.x into py3.5 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -34,3 +34,4 @@ 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0 +ea9979b550eeae87924dc4bef06070e8f8d0e22f release-pypy3.3-5.5.0 diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -64,6 +64,7 @@ Michal Bendowski stian Jan de Mooij + Spenser Bauman Tyler Wade Vincent Legoll Michael Foord @@ -311,6 +312,7 @@ Asmo Soinio jiaaro Mads Kiilerich + JohnDoe Antony Lee Jason Madden Daniel Neuhäuser diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/release-pypy3.3-5.5.0.rst copy from pypy/doc/whatsnew-pypy3-head.rst copy to pypy/doc/release-pypy3.3-5.5.0.rst diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -1,37 +1,9 @@ ======================== -What's new in PyPy3 2.4+ +What's new in PyPy3 5.5+ ======================== -.. this is the revision after pypy3-release-2.4.x was branched -.. startrev: 3f967c2be00e - -.. branch: py3k-memoryview - -Implement new memoryview features. - -.. branch: py3.3 - -.. branch: py3.3-hashfix - -Use intobject hash function for specialisedtuple - -.. branch: follow_symlinks - -Add support for dir_fd and follow_symlinks in posix.stat() - -.. branch: stat_ns - -Implement the st_xtime_ns fields in stat_result() - -.. branch: 33_fix_itertools - -Add pickling support for the itertools classes - -.. branch: py3k-update - -.. branch: py3k-get_clock_info - -.. branch: py3k-update +.. this is the revision after release-pypy3.3-5.5.x was branched +.. startrev: c5fb5db3c8ee .. branch: py3.5-time @@ -39,4 +11,4 @@ .. branch: PEP393 -Implement some level of compatibility with PEP 393 APIs. +Implement some level of compatibility with PEP 393 APIs. \ No newline at end of file From pypy.commits at gmail.com Tue Mar 14 07:52:22 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 04:52:22 -0700 (PDT) Subject: [pypy-commit] pypy default: Factor out the logic to write a forward jump, too Message-ID: <58c7d976.44002e0a.2ee55.6950@mx.google.com> Author: Armin Rigo Branch: Changeset: r90680:43b8bd691613 Date: 2017-03-14 12:51 +0100 http://bitbucket.org/pypy/pypy/changeset/43b8bd691613/ Log: Factor out the logic to write a forward jump, too diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -349,8 +349,7 @@ # mc.MOV(eax, heap(self.cpu.pos_exception())) mc.TEST_rr(eax.value, eax.value) - mc.J_il8(rx86.Conditions['NZ'], 0) - jnz_location = mc.get_relative_pos() + jnz_location = mc.emit_forward_jump('NZ') # mc.RET() # @@ -832,8 +831,7 @@ ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) mc.CMP_bi(ofs, 0xffffff) # force writing 32 bit stack_check_cmp_ofs = mc.get_relative_pos() - 4 - mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = mc.get_relative_pos() + jg_location = mc.emit_forward_jump('GE') mc.MOV_si(WORD, 0xffffff) # force writing 32 bit ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, store=True) @@ -853,8 +851,7 @@ ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) mc.CMP_bi(ofs, 0xffffff) stack_check_cmp_ofs = mc.get_relative_pos() - 4 - mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = mc.get_relative_pos() + jg_location = mc.emit_forward_jump('GE') mc.MOV_rr(edi.value, ebp.value) mc.MOV_ri(esi.value, 0xffffff) ofs2 = mc.get_relative_pos() - 4 @@ -996,8 +993,7 @@ self.mc.MOV(eax, heap(endaddr)) # MOV eax, [start] self.mc.SUB(eax, esp) # SUB eax, current self.mc.CMP(eax, heap(lengthaddr)) # CMP eax, [length] - self.mc.J_il8(rx86.Conditions['BE'], 0) # JBE .skip - jb_location = self.mc.get_relative_pos() + jb_location = self.mc.emit_forward_jump('BE')#JBE .skip self.mc.CALL(imm(self.stack_check_slowpath))# CALL slowpath # patch the JB above # .skip: self.mc.patch_forward_jump(jb_location) @@ -1234,8 +1230,7 @@ return genop_cmp def _if_parity_clear_zero_and_carry(self): - self.mc.J_il8(rx86.Conditions['NP'], 0) - jnp_location = self.mc.get_relative_pos() + jnp_location = self.mc.emit_forward_jump('NP') # CMP EBP, 0: as EBP cannot be null here, that operation should # always clear zero and carry self.mc.CMP_ri(ebp.value, 0) @@ -1718,10 +1713,11 @@ # jump to jump over this GUARD_NO_EXCEPTION as well, if we can if self._find_nearby_operation(-1).getopnum() in ( rop.COND_CALL, rop.COND_CALL_VALUE_I, rop.COND_CALL_VALUE_R): - jmp_adr = self.previous_cond_call_jcond - offset = self.mc.get_relative_pos() - jmp_adr - if offset <= 127: - self.mc.patch_forward_jump(jmp_adr) + j_location = self.previous_cond_call_jcond + try: + self.mc.patch_forward_jump(j_location) + except codebuf.ShortJumpTooFar: + pass # ignore this case def genop_guard_guard_not_invalidated(self, guard_op, guard_token, locs, ign): @@ -1832,8 +1828,7 @@ def genop_guard_guard_nonnull_class(self, guard_op, guard_token, locs, ign): self.mc.CMP(locs[0], imm1) # Patched below - self.mc.J_il8(rx86.Conditions['B'], 0) - jb_location = self.mc.get_relative_pos() + jb_location = self.mc.emit_forward_jump('B') self._cmp_guard_class(locs) # patch the JB above self.mc.patch_forward_jump(jb_location) @@ -2211,15 +2206,13 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.CMP(mem(eax, ofs), imm(value)) # patched later - self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' - return self.mc.get_relative_pos() + return self.mc.emit_forward_jump('E') # goto B if we get 'done_with_this_frame' def _call_assembler_patch_je(self, result_loc, je_location): if (IS_X86_32 and isinstance(result_loc, FrameLoc) and result_loc.type == FLOAT): self.mc.FSTPL_b(result_loc.value) - self.mc.JMP_l8(0) # jump to done, patched later - jmp_location = self.mc.get_relative_pos() + jmp_location = self.mc.emit_forward_jump_uncond() # jump to the end # self.mc.patch_forward_jump(je_location) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) @@ -2272,16 +2265,14 @@ else: loc = addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs) mc.TEST8(loc, imm(mask)) - mc.J_il8(rx86.Conditions['Z'], 0) # patched later - jz_location = mc.get_relative_pos() + jz_location = mc.emit_forward_jump('Z') # patched later # for cond_call_gc_wb_array, also add another fast path: # if GCFLAG_CARDS_SET, then we can just set one bit and be done if card_marking: # GCFLAG_CARDS_SET is in this byte at 0x80, so this fact can # been checked by the status flags of the previous TEST8 - mc.J_il8(rx86.Conditions['S'], 0) # patched later - js_location = mc.get_relative_pos() + js_location = mc.emit_forward_jump('S') # patched later else: js_location = 0 @@ -2310,8 +2301,7 @@ # The helper ends again with a check of the flag in the object. # So here, we can simply write again a 'JNS', which will be # taken if GCFLAG_CARDS_SET is still not set. - mc.J_il8(rx86.Conditions['NS'], 0) # patched later - jns_location = mc.get_relative_pos() + jns_location = mc.emit_forward_jump('NS') # patched later # # patch the JS above mc.patch_forward_jump(js_location) @@ -2385,9 +2375,8 @@ def cond_call(self, gcmap, imm_func, arglocs, resloc=None): assert self.guard_success_cc >= 0 - self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0) - # patched later - jmp_adr = self.mc.get_relative_pos() + j_location = self.mc.emit_forward_jump_cond( + rx86.invert_condition(self.guard_success_cc)) self.guard_success_cc = rx86.cond_none # self.push_gcmap(self.mc, gcmap, store=True) @@ -2437,22 +2426,21 @@ v = gpr_reg_mgr_cls.all_reg_indexes[eax.value] self.mc.MOV_rb(eax.value, v * WORD + base_ofs) # - self.mc.patch_forward_jump(jmp_adr) + self.mc.patch_forward_jump(j_location) # might be overridden again to skip over the following # guard_no_exception too - self.previous_cond_call_jcond = jmp_adr + self.previous_cond_call_jcond = j_location def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): assert size & (WORD-1) == 0 # must be correctly aligned self.mc.MOV(ecx, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (ecx.value, size)) self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) - self.mc.patch_forward_jump(jmp_adr) + self.mc.patch_forward_jump(jna_location) self.mc.MOV(heap(nursery_free_adr), edx) def malloc_cond_varsize_frame(self, nursery_free_adr, nursery_top_adr, @@ -2466,12 +2454,11 @@ else: self.mc.LEA_ra(edx.value, (ecx.value, sizeloc.value, 0, 0)) self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) - self.mc.patch_forward_jump(jmp_adr) + self.mc.patch_forward_jump(jna_location) self.mc.MOV(heap(nursery_free_adr), edx) def malloc_cond_varsize(self, kind, nursery_free_adr, nursery_top_adr, @@ -2489,8 +2476,7 @@ varsizeloc = edx self.mc.CMP(varsizeloc, imm(maxlength)) - self.mc.J_il8(rx86.Conditions['A'], 0) # patched later - jmp_adr0 = self.mc.get_relative_pos() + ja_location = self.mc.emit_forward_jump('A') # patched later self.mc.MOV(ecx, heap(nursery_free_adr)) if valid_addressing_size(itemsize): @@ -2514,10 +2500,9 @@ # now edx contains the total size in bytes, rounded up to a multiple # of WORD, plus nursery_free_adr self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr1 = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # - self.mc.patch_forward_jump(jmp_adr0) + self.mc.patch_forward_jump(ja_location) # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) if kind == rewrite.FLAG_ARRAY: @@ -2533,10 +2518,9 @@ addr = self.malloc_slowpath_unicode self.mc.MOV(edx, lengthloc) self.mc.CALL(imm(follow_jump(addr))) - self.mc.JMP_l8(0) # jump to done, patched later - jmp_location = self.mc.get_relative_pos() + jmp_location = self.mc.emit_forward_jump_uncond() # jump to later # - self.mc.patch_forward_jump(jmp_adr1) + self.mc.patch_forward_jump(jna_location) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) # write down the tid, but not if it's the result of the CALL self.mc.MOV(mem(ecx, 0), imm(arraydescr.tid)) diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -341,23 +341,20 @@ # thread. So here we check if the shadowstack pointer # is still the same as before we released the GIL (saved # in 'ebx'), and if not, we fall back to 'reacqgil_addr'. - mc.J_il8(rx86.Conditions['NE'], 0) - jne_location = mc.get_relative_pos() + jne_location = mc.emit_forward_jump('NE') # here, ecx (=old_value) is zero (so rpy_fastgil was in 'released' # state before the XCHG, but the XCHG acquired it by writing 1) rst = gcrootmap.get_root_stack_top_addr() mc = self.mc mc.CMP(ebx, heap(rst)) - mc.J_il8(rx86.Conditions['E'], 0) - je_location = mc.get_relative_pos() + je_location = mc.emit_forward_jump('E') # revert the rpy_fastgil acquired above, so that the # general 'reacqgil_addr' below can acquire it again... mc.MOV(heap(fastgil), ecx) # patch the JNE above mc.patch_forward_jump(jne_location) else: - mc.J_il8(rx86.Conditions['E'], 0) - je_location = mc.get_relative_pos() + je_location = mc.emit_forward_jump('E') # # Yes, we need to call the reacqgil() function if not self.result_value_saved_early: diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py --- a/rpython/jit/backend/x86/codebuf.py +++ b/rpython/jit/backend/x86/codebuf.py @@ -1,12 +1,13 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import intmask +from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.debug import have_debug_prints from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from rpython.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder from rpython.jit.backend.x86.regloc import LocationCodeBuilder from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64, WORD -from rpython.jit.backend.x86 import valgrind +from rpython.jit.backend.x86 import rx86, valgrind # XXX: Seems nasty to change the superclass of MachineCodeBlockWrapper # like this @@ -17,6 +18,9 @@ codebuilder_cls = X86_64_CodeBuilder backend_name = 'x86_64' +class ShortJumpTooFar(Exception): + pass + class MachineCodeBlockWrapper(BlockBuilderMixin, LocationCodeBuilder, @@ -54,7 +58,21 @@ valgrind.discard_translations(addr, self.get_relative_pos()) self._dump(addr, "jit-backend-dump", backend_name) + @specialize.arg(1) + def emit_forward_jump(self, condition_string): + return self.emit_forward_jump_cond(rx86.Conditions[condition_string]) + + def emit_forward_jump_cond(self, cond): + self.J_il8(cond, 0) + return self.get_relative_pos() + + def emit_forward_jump_uncond(self): + self.JMP_l8(0) + return self.get_relative_pos() + def patch_forward_jump(self, jcond_location): offset = self.get_relative_pos() - jcond_location - assert 0 <= offset <= 127 + assert offset >= 0 + if offset > 127: + raise ShortJumpTooFar self.overwrite(jcond_location-1, chr(offset)) From pypy.commits at gmail.com Tue Mar 14 07:55:29 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 04:55:29 -0700 (PDT) Subject: [pypy-commit] pypy default: S is the Sign flag Message-ID: <58c7da31.06d2190a.c9b49.628e@mx.google.com> Author: Armin Rigo Branch: Changeset: r90681:8492cccac927 Date: 2017-03-14 12:55 +0100 http://bitbucket.org/pypy/pypy/changeset/8492cccac927/ Log: S is the Sign flag diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2271,7 +2271,7 @@ # if GCFLAG_CARDS_SET, then we can just set one bit and be done if card_marking: # GCFLAG_CARDS_SET is in this byte at 0x80, so this fact can - # been checked by the status flags of the previous TEST8 + # been checked by the sign flags of the previous TEST8 js_location = mc.emit_forward_jump('S') # patched later else: js_location = 0 From pypy.commits at gmail.com Tue Mar 14 08:07:26 2017 From: pypy.commits at gmail.com (plan_rich) Date: Tue, 14 Mar 2017 05:07:26 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: add test, avx+sse4 version are compiled in and only used when the platform at runtime supports it Message-ID: <58c7dcfe.43502e0a.a97a2.5521@mx.google.com> Author: Richard Plangger Branch: unicode-utf8 Changeset: r90682:b2dd71846ca0 Date: 2017-03-14 13:06 +0100 http://bitbucket.org/pypy/pypy/changeset/b2dd71846ca0/ Log: add test, avx+sse4 version are compiled in and only used when the platform at runtime supports it diff --git a/rpython/rlib/rutf8/capi.py b/rpython/rlib/rutf8/capi.py --- a/rpython/rlib/rutf8/capi.py +++ b/rpython/rlib/rutf8/capi.py @@ -4,6 +4,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.tool import rffi_platform as platform +from rpython.translator.platform import platform as trans_plaform ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rutf8') SRC = ROOT.join('src') @@ -12,33 +13,42 @@ _libs = ['dl'] else: _libs = [] -eci_kwds = dict( - include_dirs = [SRC], - includes = ['utf8.h'], - libraries = _libs, - separate_module_files = [SRC.join('utf8.c')],) -global_eci = ExternalCompilationInfo(**eci_kwds) + IDXTAB = lltype.ForwardReference() IDXTAB.become(rffi.CStruct("fu8_idxtab", ('character_step', rffi.INT), - ('byte_positions', lltype.Ptr(rffi.SIZE_T)), + ('byte_positions', rffi.SIZE_TP), ('bytepos_table_length', rffi.SIZE_T))) -IDXTABPP = lltype.Ptr(lltype.Ptr(IDXTAB)) +IDXTABP = lltype.Ptr(IDXTAB) def setup(): - compile_extra = ['-DRPYTHON_LL2CTYPES'] - platform.verify_eci(ExternalCompilationInfo( - compile_extra=compile_extra, - **eci_kwds)) + compile_extra = ['-DRPYTHON_LL2CTYPES', '-DALLOW_SURROGATES=0', '-fPIC'] + eci_kwds = dict( + include_dirs = [SRC], + includes = ['utf8.h'], + libraries = _libs, + compile_extra = compile_extra) + # compile the SSE4.1 and AVX version + compile_extra.append('-msse4.1') + ofile_eci = ExternalCompilationInfo(**eci_kwds) + sse4_o, = trans_plaform._compile_o_files([SRC.join('utf8-sse4.c')], ofile_eci) + compile_extra.pop() + compile_extra.append('-mavx2') + ofile_eci = ExternalCompilationInfo(**eci_kwds) + avx_o, = trans_plaform._compile_o_files([SRC.join('utf8-avx.c')], ofile_eci) + del ofile_eci - eci = global_eci - count_utf8_code_points = rffi.llexternal("fu8_count_utf8_codepoints", - [rffi.CCHARP, rffi.SIZE_T], + eci_kwds['separate_module_files'] = [SRC.join('utf8.c')] + eci_kwds['link_files'] = [sse4_o.strpath, avx_o.strpath] + eci = ExternalCompilationInfo(**eci_kwds) + platform.verify_eci(eci) + count_utf8_codepoints = rffi.llexternal("fu8_count_utf8_codepoints", + [rffi.CCHARP, rffi.SSIZE_T], rffi.SSIZE_T, compilation_info=eci, _nowrapper=True) index2byteposition = rffi.llexternal("fu8_idx2bytepos", - [rffi.SIZE_T, rffi.CCHARP, rffi.SIZE_T, IDXTABPP], + [rffi.SIZE_T, rffi.CCHARP, rffi.SIZE_T, IDXTABP], rffi.SSIZE_T, compilation_info=eci, _nowrapper=True) diff --git a/rpython/rlib/rutf8/src/utf8-avx.c b/rpython/rlib/rutf8/src/utf8-avx.c --- a/rpython/rlib/rutf8/src/utf8-avx.c +++ b/rpython/rlib/rutf8/src/utf8-avx.c @@ -61,8 +61,9 @@ printf("\n"); } -ssize_t count_utf8_codepoints_avx(const uint8_t * encoded, size_t len) +ssize_t fu8_count_utf8_codepoints_avx(const char * utf8, size_t len) { + const uint8_t * encoded = (const uint8_t*)utf8; __builtin_prefetch(encoded, 0, 0); size_t num_codepoints = 0; __m256i chunk; @@ -244,7 +245,7 @@ return num_codepoints; } - ssize_t result = count_utf8_codepoints_seq(encoded, len); + ssize_t result = fu8_count_utf8_codepoints_seq(encoded, len); if (result == -1) { return -1; } diff --git a/rpython/rlib/rutf8/src/utf8-scalar.c b/rpython/rlib/rutf8/src/utf8-scalar.c --- a/rpython/rlib/rutf8/src/utf8-scalar.c +++ b/rpython/rlib/rutf8/src/utf8-scalar.c @@ -17,9 +17,10 @@ return 0; } -ssize_t count_utf8_codepoints_seq(const uint8_t * encoded, size_t len) { +ssize_t fu8_count_utf8_codepoints_seq(const char * utf8, size_t len) { size_t num_codepoints = 0; uint8_t byte = 0; + const uint8_t * encoded = (const uint8_t*)utf8; const uint8_t * endptr = encoded + len; while (encoded < endptr) { diff --git a/rpython/rlib/rutf8/src/utf8-sse4.c b/rpython/rlib/rutf8/src/utf8-sse4.c --- a/rpython/rlib/rutf8/src/utf8-sse4.c +++ b/rpython/rlib/rutf8/src/utf8-sse4.c @@ -40,8 +40,9 @@ } -ssize_t count_utf8_codepoints_sse4(const uint8_t * encoded, size_t len) +ssize_t fu8_count_utf8_codepoints_sse4(const char * utf8, size_t len) { + const uint8_t * encoded = (const uint8_t*)utf8; __builtin_prefetch(encoded, 0, 0); size_t num_codepoints = 0; __m128i chunk; @@ -222,7 +223,7 @@ return num_codepoints; } - ssize_t result = count_utf8_codepoints_seq(encoded, len); + ssize_t result = fu8_count_utf8_codepoints_seq(encoded, len); if (result == -1) { return -1; } diff --git a/rpython/rlib/rutf8/src/utf8.c b/rpython/rlib/rutf8/src/utf8.c --- a/rpython/rlib/rutf8/src/utf8.c +++ b/rpython/rlib/rutf8/src/utf8.c @@ -37,7 +37,7 @@ } } -ssize_t count_utf8_codepoints(const uint8_t * encoded, size_t len) +ssize_t fu8_count_utf8_codepoints(const char * utf8, size_t len) { if (instruction_set == -1) { detect_instructionset(); @@ -45,15 +45,15 @@ if (len >= 32 && (instruction_set & ISET_AVX2) != 0) { // to the MOON! - return count_utf8_codepoints_avx(encoded, len); + return fu8_count_utf8_codepoints_avx(utf8, len); } if (len >= 16 && (instruction_set == ISET_SSE4) != 0) { // speed!! - return count_utf8_codepoints_sse4(encoded, len); + return fu8_count_utf8_codepoints_sse4(utf8, len); } // oh no, just do it sequentially! - return count_utf8_codepoints_seq(encoded, len); + return fu8_count_utf8_codepoints_seq(utf8, len); } typedef struct fu8_idxtab { diff --git a/rpython/rlib/rutf8/src/utf8.h b/rpython/rlib/rutf8/src/utf8.h --- a/rpython/rlib/rutf8/src/utf8.h +++ b/rpython/rlib/rutf8/src/utf8.h @@ -4,6 +4,20 @@ #include #include +#ifdef RPYTHON_LL2CTYPES + /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */ +#ifndef RPY_EXTERN +# define RPY_EXTERN RPY_EXPORTED +#endif + +#ifdef _WIN32 +# define RPY_EXPORTED __declspec(dllexport) +#else +# define RPY_EXPORTED extern __attribute__((visibility("default"))) +#endif + +#endif + /** * Returns -1 if the given string is not a valid utf8 encoded string. * Otherwise returns the amount code point in the given string. @@ -12,14 +26,14 @@ * The above documentation also applies for several vectorized implementations * found below. * - * count_utf8_codepoints dispatches amongst several + * fu8_count_utf8_codepoints dispatches amongst several * implementations (e.g. seq, SSE4, AVX) */ // TODO rename (fu8 prefix) -ssize_t fu8_count_utf8_codepoints(const uint8_t * encoded, size_t len); -ssize_t fu8_count_utf8_codepoints_seq(const uint8_t * encoded, size_t len); -ssize_t fu8_count_utf8_codepoints_sse4(const uint8_t * encoded, size_t len); -ssize_t fu8_count_utf8_codepoints_avx(const uint8_t * encoded, size_t len); +RPY_EXTERN ssize_t fu8_count_utf8_codepoints(const char * utf8, size_t len); +RPY_EXTERN ssize_t fu8_count_utf8_codepoints_seq(const char * utf8, size_t len); +RPY_EXTERN ssize_t fu8_count_utf8_codepoints_sse4(const char * utf8, size_t len); +RPY_EXTERN ssize_t fu8_count_utf8_codepoints_avx(const char * utf8, size_t len); struct fu8_idxtab; @@ -41,11 +55,11 @@ * table to speed up indexing. * */ -ssize_t fu8_idx2bytepos(size_t index, +RPY_EXTERN ssize_t fu8_idx2bytepos(size_t index, const uint8_t * utf8, size_t bytelen, size_t cplen, struct fu8_idxtab ** tab); -void fu8_free_idxtab(struct fu8_idxtab * t); -ssize_t fu8_idx2bytepso_sse4(size_t index, +RPY_EXTERN void fu8_free_idxtab(struct fu8_idxtab * t); +RPY_EXTERN ssize_t fu8_idx2bytepso_sse4(size_t index, const uint8_t * utf8, size_t len, struct fu8_idxtab ** t); From pypy.commits at gmail.com Tue Mar 14 10:40:27 2017 From: pypy.commits at gmail.com (rlamy) Date: Tue, 14 Mar 2017 07:40:27 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58c800db.8f472e0a.8c6a7.6873@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90683:afbf09453369 Date: 2017-03-14 14:26 +0000 http://bitbucket.org/pypy/pypy/changeset/afbf09453369/ Log: hg merge default diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v5.7.0.rst release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -53,6 +54,12 @@ release-0.7.0.rst release-0.6 +CPython 3.5 compatible versions +------------------------------- + +.. toctree:: + + release-v5.7.0.rst CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.7.0.rst whatsnew-pypy2-5.6.0.rst whatsnew-pypy2-5.4.0.rst whatsnew-pypy2-5.3.1.rst diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,204 +1,8 @@ ========================== -What's new in PyPy2.7 5.6+ +What's new in PyPy2.7 5.8+ ========================== -.. this is a revision shortly after release-pypy2.7-v5.6 -.. startrev: 7e9787939641 +.. this is a revision shortly after release-pypy2.7-v5.7 +.. startrev: 44f31f6dd39f -Since a while now, PyPy preserves the order of dictionaries and sets. -However, the set literal syntax ``{x, y, z}`` would by mistake build a -set with the opposite order: ``set([z, y, x])``. This has been fixed. -Note that CPython is inconsistent too: in 2.7.12, ``{5, 5.0}`` would be -``set([5.0])``, but in 2.7.trunk it is ``set([5])``. PyPy's behavior -changed in exactly the same way because of this fix. - - -.. branch: mappingproxy -.. branch: py3k-finish_time -.. branch: py3k-kwonly-builtin -.. branch: py3k_add_terminal_size -.. branch: testing-cleanup-py3k - -.. branch: rpython-resync -Backport rpython changes made directly on the py3k and py3.5 branches. - -.. branch: rpython-error-to-systemerror - -Any uncaught RPython exception (from a PyPy bug) is turned into an -app-level SystemError. This should improve the lot of users hitting an -uncaught RPython error. - -.. branch: union-side-effects-2 - -Try to improve the consistency of RPython annotation unions. - -.. branch: pytest-2.9.2 - -.. branch: clean-exported-state - -Clean-ups in the jit optimizeopt - -.. branch: conditional_call_value_4 - -Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally -call this function" returning a result. - -.. branch: desc-specialize - -Refactor FunctionDesc.specialize() and related code (RPython annotator). - -.. branch: raw-calloc - -.. branch: issue2446 - -Assign ``tp_doc`` to the new TypeObject's type dictionary ``__doc__`` key -so it will be picked up by app-level objects of that type - -.. branch: cling-support - -Module cppyy now uses cling as its backend (Reflex has been removed). The -user-facing interface and main developer tools (genreflex, selection files, -class loader, etc.) remain the same. A libcppyy_backend.so library is still -needed but is now available through PyPI with pip: PyPy-cppyy-backend. - -The Cling-backend brings support for modern C++ (11, 14, etc.), dynamic -template instantations, and improved integration with CFFI for better -performance. It also provides interactive C++ (and bindings to that). - -.. branch: better-PyDict_Next - -Improve the performance of ``PyDict_Next``. When trying ``PyDict_Next`` on a -typedef dict, the test exposed a problem converting a ``GetSetProperty`` to a -``PyGetSetDescrObject``. The other direction seem to be fully implemented. -This branch made a minimal effort to convert the basic fields to avoid -segfaults, but trying to use the ``PyGetSetDescrObject`` will probably fail. - -.. branch: stdlib-2.7.13 - -Updated the implementation to match CPython 2.7.13 instead of 2.7.13. - -.. branch: issue2444 - -Fix ``PyObject_GetBuffer`` and ``PyMemoryView_GET_BUFFER``, which leaked -memory and held references. Add a finalizer to CPyBuffer, add a -PyMemoryViewObject with a PyBuffer attached so that the call to -``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory. -Properly call ``bf_releasebuffer`` when not ``NULL``. - -.. branch: boehm-rawrefcount - -Support translations of cpyext with the Boehm GC (for special cases like -revdb). - -.. branch: strbuf-as-buffer - -Implement StringBuffer.get_raw_address (missing feature for the buffer protocol). -More generally it is now possible to obtain the address of any object (if it -is readonly) without pinning it. - -.. branch: cpyext-cleanup -.. branch: api_func-refactor - -Refactor cpyext initialisation. - -.. branch: cpyext-from2 - -Fix a test failure introduced by strbuf-as-buffer - -.. branch: cpyext-FromBuffer - -Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to -the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak. - -.. branch: issue2464 - -Give (almost?) all GetSetProperties a valid __objclass__. - -.. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419 -.. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033 - -.. branch: missing-tp_new - -Improve mixing app-level classes in c-extensions, especially if the app-level -class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect -all the method slots to be filled with a function pointer, where app-level will -search up the mro for an appropriate function at runtime. With this branch we -now fill many more slots in the c-extenion type objects. -Also fix for c-extension type that calls ``tp_hash`` during initialization -(str, unicode types), and fix instantiating c-extension types from built-in -classes by enforcing an order of instaniation. - -.. branch: rffi-parser-2 - -rffi structures in cpyext can now be created by parsing simple C headers. -Additionally, the cts object that holds the parsed information can act like -cffi's ffi objects, with the methods cts.cast() and cts.gettype(). - -.. branch: rpython-hash - -Don't freeze hashes in the translated pypy. In practice, that means -that we can now translate PyPy with the option --hash=siphash24 and get -the same hashes as CPython 3.5, which can be randomized (in a -crypographically good way). It is the default in PyPy3. The default of -PyPy2 remains unchanged: there are user programs out there that depend -on constant hashes (or even sometimes on specific hash results). - -.. branch: dict-move-to-end - -Our dicts, which are always ordered, now have an extra "method" for -Python 3.x which moves an item to first or last position. In PyPy 3.5 -it is the standard ``OrderedDict.move_to_end()`` method, but the -behavior is also available on Python 2.x or for the ``dict`` type by -calling ``__pypy__.move_to_end(dict, key, last=True)``. - - -.. branch optinfo-into-bridges-3 - -Improve the optimization of branchy Python code by retaining more information -across failing guards. - - -.. branch: space-newtext - -Internal refactoring of ``space.wrap()``, which is now replaced with -explicitly-typed methods. Notably, there are now ``space.newbytes()`` -and ``space.newtext()``: these two methods are identical on PyPy 2.7 but -not on PyPy 3.x. The latter is used to get an app-level unicode string -by decoding the RPython string, assumed to be utf-8. - -.. branch: space-wrap - -.. branch: fix_bool_restype - -Fix for ``ctypes.c_bool``-returning ctypes functions - -.. branch: py3.5-text-utf8 - -space.text_w now encodes to utf-8 not preserving surrogates. - -.. branch: fix-cpyext-releasebuffer - -Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews -keeping objects alive forever (missing decref), and make sure that -bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. - -.. branch: fix-global - -Fix bug (bad reported info) when asked to translate SyntaxWarning to -SyntaxError. - -.. branch: optinfo-into-bridges-3 - -Improve the optimization of branchy Python code by retaining more -information across failing guards. This is done by appending some -carefully encoded extra information into the resume code. - -.. branch: shadowstack-perf-2 - -Two changes that together bring the performance of shadowstack close to -asmgcc---close enough that we can now make shadowstack the default even -on Linux. This should remove a whole class of rare bugs introduced by -asmgcc. - -.. branch: fniephaus/fix-typo-1488123166752 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.7.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-5.7.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-5.7.0.rst @@ -1,6 +1,6 @@ -========================== -What's new in PyPy2.7 5.6+ -========================== +========================= +What's new in PyPy2.7 5.7 +========================= .. this is a revision shortly after release-pypy2.7-v5.6 .. startrev: 7e9787939641 diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -1866,7 +1866,7 @@ def test_call_with_nested_anonymous_struct(self): import sys if sys.platform == 'win32': - py.test.skip("needs a GCC extension") + skip("needs a GCC extension") ffi, lib = self.prepare(""" struct foo { int a; union { int b, c; }; }; struct foo f(void); @@ -1914,6 +1914,9 @@ "set_source() and not taking a final '...' argument)") def test_call_with_zero_length_field(self): + import sys + if sys.platform == 'win32': + skip("zero-length field not supported by MSVC") ffi, lib = self.prepare(""" struct foo { int a; int x[0]; }; struct foo f(void); @@ -1959,7 +1962,7 @@ def test_call_with_packed_struct(self): import sys if sys.platform == 'win32': - py.test.skip("needs a GCC extension") + skip("needs a GCC extension") ffi, lib = self.prepare(""" struct foo { char y; int x; }; struct foo f(void); diff --git a/pypy/module/_rawffi/alt/test/test_funcptr.py b/pypy/module/_rawffi/alt/test/test_funcptr.py --- a/pypy/module/_rawffi/alt/test/test_funcptr.py +++ b/pypy/module/_rawffi/alt/test/test_funcptr.py @@ -32,7 +32,10 @@ # c_file.write(py.code.Source('\n'.join(snippets))) eci = ExternalCompilationInfo(include_dirs=[cdir]) - return str(platform.compile([c_file], eci, 'x', standalone=False)) + # Windows note: can't reuse the same file name 'x.dll', because + # the previous one is likely still opened + return str(platform.compile([c_file], eci, 'x' + cls.__name__, + standalone=False)) def setup_class(cls): space = cls.space diff --git a/pypy/module/cpyext/src/pymem.c b/pypy/module/cpyext/src/pymem.c --- a/pypy/module/cpyext/src/pymem.c +++ b/pypy/module/cpyext/src/pymem.c @@ -1,5 +1,14 @@ +#ifdef _WIN32 +# define _WIN32_WINNT 0x0501 +#endif + #include +#ifdef _WIN32 +# include +#endif + + void * PyMem_RawMalloc(size_t size) { @@ -120,10 +129,21 @@ _PyPyGC_AddMemoryPressure(report); PyGILState_Release(state); } + + /* Should we return -2 or 0? In theory it should be -2, because + we're not using the info to really track the allocations. + But I'm sure someone is too clever somewhere and stops calling + _PyTraceMalloc_Track() if it returns -2. On the other hand, + returning 0 might lead to expectations that importing + 'tracemalloc' works on Python 3. Oh well, in that case we'll + just crash with ImportError during 'import tracemalloc'. + */ + return 0; } int _PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, uintptr_t ptr) { - /* nothing */ + /* nothing to do */ + return 0; } diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py --- a/pypy/module/cpyext/test/test_pystate.py +++ b/pypy/module/cpyext/test/test_pystate.py @@ -71,6 +71,7 @@ ("dance", "METH_NOARGS", """ PyThreadState *old_tstate, *new_tstate; + PyObject *d; PyEval_InitThreads(); @@ -79,7 +80,7 @@ return PyLong_FromLong(0); } - PyObject* d = PyThreadState_GetDict(); /* fails on cpython */ + d = PyThreadState_GetDict(); /* fails on cpython */ if (d != NULL) { return PyLong_FromLong(1); } @@ -142,6 +143,9 @@ ("bounce", "METH_NOARGS", """ PyThreadState * tstate; + PyObject *dict; + PyGILState_STATE gilstate; + if (PyEval_ThreadsInitialized() == 0) { PyEval_InitThreads(); @@ -150,11 +154,11 @@ if (tstate == NULL) { return PyLong_FromLong(0); } - PyObject* dict = PyThreadState_GetDict(); + dict = PyThreadState_GetDict(); if (dict != NULL) { return PyLong_FromLong(1); } - PyGILState_STATE gilstate = PyGILState_Ensure(); + gilstate = PyGILState_Ensure(); dict = PyThreadState_GetDict(); if (dict == NULL) { return PyLong_FromLong(2); diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -154,5 +154,10 @@ def test_tuple_subclass(self): module = self.import_module(name='foo') - a = module.TupleLike([1, 2, 3]) + a = module.TupleLike(range(100, 400, 100)) assert module.is_TupleLike(a) == 1 + assert isinstance(a, tuple) + assert issubclass(type(a), tuple) + assert list(a) == range(100, 400, 100) + assert list(a) == range(100, 400, 100) + assert list(a) == range(100, 400, 100) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -947,12 +947,15 @@ pass class bar(f1, f2): pass + class foo(f2, f1): + pass assert bar.__base__ is f2 # On cpython, the size changes. if '__pypy__' in sys.builtin_module_names: assert module.size_of_instances(bar) == size else: assert module.size_of_instances(bar) >= size + assert module.size_of_instances(foo) == module.size_of_instances(bar) def test_app_cant_subclass_two_types(self): import sys @@ -1176,4 +1179,21 @@ assert type(obj).__doc__ == "The Base12 type or object" assert obj.__doc__ == "The Base12 type or object" - + def test_multiple_inheritance_fetch_tp_bases(self): + module = self.import_extension('foo', [ + ("foo", "METH_O", + ''' + PyTypeObject *tp; + tp = (PyTypeObject*)args; + Py_INCREF(tp->tp_bases); + return tp->tp_bases; + ''' + )]) + class A(object): + pass + class B(object): + pass + class C(A, B): + pass + bases = module.foo(C) + assert bases == (A, B) diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py --- a/pypy/module/cpyext/test/test_userslots.py +++ b/pypy/module/cpyext/test/test_userslots.py @@ -160,8 +160,9 @@ }; ''', more_init=''' PyObject * mod1 = PyImport_ImportModule("datetime"); + PyObject * dt; if (mod1 == NULL) INITERROR; - PyObject * dt = PyUnicode_FromString("datetime"); + dt = PyUnicode_FromString("datetime"); datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod1, dt); if (datetime_cls == NULL) INITERROR; _Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -417,8 +417,8 @@ def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way - # (minimally, if tp_basicsize is zero we copy it from the base) - if not pto.c_tp_basicsize: + # (minimally, if tp_basicsize is zero or too low, we copy it from the base) + if pto.c_tp_basicsize < base_pto.c_tp_basicsize: pto.c_tp_basicsize = base_pto.c_tp_basicsize if pto.c_tp_itemsize < base_pto.c_tp_itemsize: pto.c_tp_itemsize = base_pto.c_tp_itemsize @@ -672,7 +672,7 @@ if builder.cpyext_type_init is not None: builder.cpyext_type_init.append((pto, w_type)) else: - finish_type_1(space, pto) + finish_type_1(space, pto, w_type.bases_w) finish_type_2(space, pto, w_type) pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) @@ -804,7 +804,7 @@ return w_obj -def finish_type_1(space, pto): +def finish_type_1(space, pto, bases_w=None): """ Sets up tp_bases, necessary before creating the interpreter type. """ @@ -816,11 +816,12 @@ if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type if not pto.c_tp_bases: - if not base: - bases = space.newtuple([]) - else: - bases = space.newtuple([from_ref(space, base_pyo)]) - pto.c_tp_bases = make_ref(space, bases) + if bases_w is None: + if not base: + bases_w = [] + else: + bases_w = [from_ref(space, base_pyo)] + pto.c_tp_bases = make_ref(space, space.newtuple(bases_w)) def finish_type_2(space, pto, w_obj): """ diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1231,7 +1231,8 @@ def test_ffi_buffer_comparisons(self): ffi = FFI(backend=self.Backend()) ba = bytearray(range(100, 110)) - assert ba == memoryview(ba) # justification for the following + if sys.version_info >= (2, 7): + assert ba == memoryview(ba) # justification for the following a = ffi.new("uint8_t[]", list(ba)) c = ffi.new("uint8_t[]", [99] + list(ba)) try: diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -2040,7 +2040,7 @@ struct foo s = { 40, 200 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().x == 200 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2069,7 +2069,7 @@ s.b = 200; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().b == 200 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2095,7 +2095,7 @@ struct foo s = { 11 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().x == 11 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2107,6 +2107,8 @@ "set_source() and not taking a final '...' argument)") def test_call_with_zero_length_field(): + if sys.platform == 'win32': + py.test.skip("zero-length field not supported by MSVC") ffi = FFI() ffi.cdef(""" struct foo { int a; int x[0]; }; @@ -2119,7 +2121,7 @@ struct foo s = { 42 }; return s; } - struct foo g(int a, ...) { } + struct foo g(int a, ...) { return f(); } """) assert lib.f().a == 42 e = py.test.raises(NotImplementedError, lib.g, 0) @@ -2143,7 +2145,7 @@ union foo s = { 42 }; return s; } - union foo g(int a, ...) { } + union foo g(int a, ...) { return f(); } """) assert lib.f().a == 42 e = py.test.raises(NotImplementedError, lib.g, 0) diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -26,10 +26,15 @@ SUPPORT__THREAD = ( # whether the particular C compiler supports __thread sys.platform.startswith("linux") or # Linux works - sys.platform.startswith("darwin")) # OS/X >= 10.7 works + #sys.platform.startswith("darwin") or # OS/X >= 10.7 works (*) + False) # Windows doesn't work. Please # add other platforms here if it works on them. +# (*) NOTE: __thread on OS/X does not work together with +# pthread_key_create(): when the destructor is called, the __thread is +# already freed! + MAINDIR = os.path.dirname(os.path.dirname(__file__)) CACHE_DIR = os.path.realpath(os.path.join(MAINDIR, '_cache')) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -349,15 +349,12 @@ # mc.MOV(eax, heap(self.cpu.pos_exception())) mc.TEST_rr(eax.value, eax.value) - mc.J_il8(rx86.Conditions['NZ'], 0) - jnz_location = mc.get_relative_pos() + jnz_location = mc.emit_forward_jump('NZ') # mc.RET() # # patch the JNZ above - offset = mc.get_relative_pos() - jnz_location - assert 0 < offset <= 127 - mc.overwrite(jnz_location-1, chr(offset)) + mc.patch_forward_jump(jnz_location) # From now on this function is basically "merged" with # its caller and so contains DEFAULT_FRAME_BYTES bytes # plus my own return address, which we'll ignore next @@ -834,16 +831,13 @@ ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) mc.CMP_bi(ofs, 0xffffff) # force writing 32 bit stack_check_cmp_ofs = mc.get_relative_pos() - 4 - mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = mc.get_relative_pos() + jg_location = mc.emit_forward_jump('GE') mc.MOV_si(WORD, 0xffffff) # force writing 32 bit ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, store=True) mc.CALL(imm(self._frame_realloc_slowpath)) # patch the JG above - offset = mc.get_relative_pos() - jg_location - assert 0 < offset <= 127 - mc.overwrite(jg_location-1, chr(offset)) + mc.patch_forward_jump(jg_location) self.frame_depth_to_patch.append(stack_check_cmp_ofs) self.frame_depth_to_patch.append(ofs2) @@ -857,16 +851,13 @@ ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) mc.CMP_bi(ofs, 0xffffff) stack_check_cmp_ofs = mc.get_relative_pos() - 4 - mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = mc.get_relative_pos() + jg_location = mc.emit_forward_jump('GE') mc.MOV_rr(edi.value, ebp.value) mc.MOV_ri(esi.value, 0xffffff) ofs2 = mc.get_relative_pos() - 4 mc.CALL(imm(self.cpu.realloc_frame_crash)) # patch the JG above - offset = mc.get_relative_pos() - jg_location - assert 0 < offset <= 127 - mc.overwrite(jg_location-1, chr(offset)) + mc.patch_forward_jump(jg_location) self.frame_depth_to_patch.append(stack_check_cmp_ofs) self.frame_depth_to_patch.append(ofs2) @@ -1002,13 +993,10 @@ self.mc.MOV(eax, heap(endaddr)) # MOV eax, [start] self.mc.SUB(eax, esp) # SUB eax, current self.mc.CMP(eax, heap(lengthaddr)) # CMP eax, [length] - self.mc.J_il8(rx86.Conditions['BE'], 0) # JBE .skip - jb_location = self.mc.get_relative_pos() + jb_location = self.mc.emit_forward_jump('BE')#JBE .skip self.mc.CALL(imm(self.stack_check_slowpath))# CALL slowpath # patch the JB above # .skip: - offset = self.mc.get_relative_pos() - jb_location - assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, chr(offset)) + self.mc.patch_forward_jump(jb_location) # def _call_footer(self): @@ -1242,15 +1230,12 @@ return genop_cmp def _if_parity_clear_zero_and_carry(self): - self.mc.J_il8(rx86.Conditions['NP'], 0) - jnp_location = self.mc.get_relative_pos() + jnp_location = self.mc.emit_forward_jump('NP') # CMP EBP, 0: as EBP cannot be null here, that operation should # always clear zero and carry self.mc.CMP_ri(ebp.value, 0) # patch the JNP above - offset = self.mc.get_relative_pos() - jnp_location - assert 0 < offset <= 127 - self.mc.overwrite(jnp_location-1, chr(offset)) + self.mc.patch_forward_jump(jnp_location) def _cmpop_float(cond, rev_cond): is_ne = cond == 'NE' @@ -1728,10 +1713,11 @@ # jump to jump over this GUARD_NO_EXCEPTION as well, if we can if self._find_nearby_operation(-1).getopnum() in ( rop.COND_CALL, rop.COND_CALL_VALUE_I, rop.COND_CALL_VALUE_R): - jmp_adr = self.previous_cond_call_jcond - offset = self.mc.get_relative_pos() - jmp_adr - if offset <= 127: - self.mc.overwrite(jmp_adr-1, chr(offset)) + j_location = self.previous_cond_call_jcond + try: + self.mc.patch_forward_jump(j_location) + except codebuf.ShortJumpTooFar: + pass # ignore this case def genop_guard_guard_not_invalidated(self, guard_op, guard_token, locs, ign): @@ -1842,13 +1828,10 @@ def genop_guard_guard_nonnull_class(self, guard_op, guard_token, locs, ign): self.mc.CMP(locs[0], imm1) # Patched below - self.mc.J_il8(rx86.Conditions['B'], 0) - jb_location = self.mc.get_relative_pos() + jb_location = self.mc.emit_forward_jump('B') self._cmp_guard_class(locs) # patch the JB above - offset = self.mc.get_relative_pos() - jb_location - assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, chr(offset)) + self.mc.patch_forward_jump(jb_location) # self.guard_success_cc = rx86.Conditions['E'] self.implement_guard(guard_token) @@ -2223,19 +2206,15 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.CMP(mem(eax, ofs), imm(value)) # patched later - self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' - return self.mc.get_relative_pos() + return self.mc.emit_forward_jump('E') # goto B if we get 'done_with_this_frame' def _call_assembler_patch_je(self, result_loc, je_location): if (IS_X86_32 and isinstance(result_loc, FrameLoc) and result_loc.type == FLOAT): self.mc.FSTPL_b(result_loc.value) - self.mc.JMP_l8(0) # jump to done, patched later - jmp_location = self.mc.get_relative_pos() + jmp_location = self.mc.emit_forward_jump_uncond() # jump to the end # - offset = jmp_location - je_location - assert 0 < offset <= 127 - self.mc.overwrite(je_location - 1, chr(offset)) + self.mc.patch_forward_jump(je_location) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) # return jmp_location @@ -2255,9 +2234,7 @@ self.mc.MOV_rm(eax.value, (eax.value, ofs)) def _call_assembler_patch_jmp(self, jmp_location): - offset = self.mc.get_relative_pos() - jmp_location - assert 0 <= offset <= 127 - self.mc.overwrite(jmp_location - 1, chr(offset)) + self.mc.patch_forward_jump(jmp_location) # ------------------- END CALL ASSEMBLER ----------------------- @@ -2288,16 +2265,14 @@ else: loc = addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs) mc.TEST8(loc, imm(mask)) - mc.J_il8(rx86.Conditions['Z'], 0) # patched later - jz_location = mc.get_relative_pos() + jz_location = mc.emit_forward_jump('Z') # patched later # for cond_call_gc_wb_array, also add another fast path: # if GCFLAG_CARDS_SET, then we can just set one bit and be done if card_marking: # GCFLAG_CARDS_SET is in this byte at 0x80, so this fact can - # been checked by the status flags of the previous TEST8 - mc.J_il8(rx86.Conditions['S'], 0) # patched later - js_location = mc.get_relative_pos() + # been checked by the sign flags of the previous TEST8 + js_location = mc.emit_forward_jump('S') # patched later else: js_location = 0 @@ -2326,13 +2301,10 @@ # The helper ends again with a check of the flag in the object. # So here, we can simply write again a 'JNS', which will be # taken if GCFLAG_CARDS_SET is still not set. - mc.J_il8(rx86.Conditions['NS'], 0) # patched later - jns_location = mc.get_relative_pos() + jns_location = mc.emit_forward_jump('NS') # patched later # # patch the JS above - offset = mc.get_relative_pos() - js_location - assert 0 < offset <= 127 - mc.overwrite(js_location-1, chr(offset)) + mc.patch_forward_jump(js_location) # # case GCFLAG_CARDS_SET: emit a few instructions to do # directly the card flag setting @@ -2367,14 +2339,10 @@ raise AssertionError("index is neither RegLoc nor ImmedLoc") # # patch the JNS above - offset = mc.get_relative_pos() - jns_location - assert 0 < offset <= 127 - mc.overwrite(jns_location-1, chr(offset)) + mc.patch_forward_jump(jns_location) # patch the JZ above - offset = mc.get_relative_pos() - jz_location - assert 0 < offset <= 127 - mc.overwrite(jz_location-1, chr(offset)) + mc.patch_forward_jump(jz_location) def genop_discard_cond_call_gc_wb(self, op, arglocs): self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs) @@ -2407,9 +2375,8 @@ def cond_call(self, gcmap, imm_func, arglocs, resloc=None): assert self.guard_success_cc >= 0 - self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0) - # patched later - jmp_adr = self.mc.get_relative_pos() + j_location = self.mc.emit_forward_jump_cond( + rx86.invert_condition(self.guard_success_cc)) self.guard_success_cc = rx86.cond_none # self.push_gcmap(self.mc, gcmap, store=True) @@ -2459,26 +2426,21 @@ v = gpr_reg_mgr_cls.all_reg_indexes[eax.value] self.mc.MOV_rb(eax.value, v * WORD + base_ofs) # - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(j_location) # might be overridden again to skip over the following # guard_no_exception too - self.previous_cond_call_jcond = jmp_adr + self.previous_cond_call_jcond = j_location def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): assert size & (WORD-1) == 0 # must be correctly aligned self.mc.MOV(ecx, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (ecx.value, size)) self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(jna_location) self.mc.MOV(heap(nursery_free_adr), edx) def malloc_cond_varsize_frame(self, nursery_free_adr, nursery_top_adr, @@ -2492,14 +2454,11 @@ else: self.mc.LEA_ra(edx.value, (ecx.value, sizeloc.value, 0, 0)) self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(jna_location) self.mc.MOV(heap(nursery_free_adr), edx) def malloc_cond_varsize(self, kind, nursery_free_adr, nursery_top_adr, @@ -2517,8 +2476,7 @@ varsizeloc = edx self.mc.CMP(varsizeloc, imm(maxlength)) - self.mc.J_il8(rx86.Conditions['A'], 0) # patched later - jmp_adr0 = self.mc.get_relative_pos() + ja_location = self.mc.emit_forward_jump('A') # patched later self.mc.MOV(ecx, heap(nursery_free_adr)) if valid_addressing_size(itemsize): @@ -2542,12 +2500,9 @@ # now edx contains the total size in bytes, rounded up to a multiple # of WORD, plus nursery_free_adr self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr1 = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # - offset = self.mc.get_relative_pos() - jmp_adr0 - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr0-1, chr(offset)) + self.mc.patch_forward_jump(ja_location) # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) if kind == rewrite.FLAG_ARRAY: @@ -2563,21 +2518,16 @@ addr = self.malloc_slowpath_unicode self.mc.MOV(edx, lengthloc) self.mc.CALL(imm(follow_jump(addr))) - self.mc.JMP_l8(0) # jump to done, patched later - jmp_location = self.mc.get_relative_pos() + jmp_location = self.mc.emit_forward_jump_uncond() # jump to later # - offset = self.mc.get_relative_pos() - jmp_adr1 - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr1-1, chr(offset)) + self.mc.patch_forward_jump(jna_location) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) # write down the tid, but not if it's the result of the CALL self.mc.MOV(mem(ecx, 0), imm(arraydescr.tid)) # while we're at it, this line is not needed if we've done the CALL self.mc.MOV(heap(nursery_free_adr), edx) # - offset = self.mc.get_relative_pos() - jmp_location - assert 0 < offset <= 127 - self.mc.overwrite(jmp_location - 1, chr(offset)) + self.mc.patch_forward_jump(jmp_location) def store_force_descr(self, op, fail_locs, frame_depth): guard_token = self.implement_guard_recovery(op.opnum, diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -341,25 +341,20 @@ # thread. So here we check if the shadowstack pointer # is still the same as before we released the GIL (saved # in 'ebx'), and if not, we fall back to 'reacqgil_addr'. - mc.J_il8(rx86.Conditions['NE'], 0) - jne_location = mc.get_relative_pos() + jne_location = mc.emit_forward_jump('NE') # here, ecx (=old_value) is zero (so rpy_fastgil was in 'released' # state before the XCHG, but the XCHG acquired it by writing 1) rst = gcrootmap.get_root_stack_top_addr() mc = self.mc mc.CMP(ebx, heap(rst)) - mc.J_il8(rx86.Conditions['E'], 0) - je_location = mc.get_relative_pos() + je_location = mc.emit_forward_jump('E') # revert the rpy_fastgil acquired above, so that the # general 'reacqgil_addr' below can acquire it again... mc.MOV(heap(fastgil), ecx) # patch the JNE above - offset = mc.get_relative_pos() - jne_location - assert 0 < offset <= 127 - mc.overwrite(jne_location-1, chr(offset)) + mc.patch_forward_jump(jne_location) else: - mc.J_il8(rx86.Conditions['E'], 0) - je_location = mc.get_relative_pos() + je_location = mc.emit_forward_jump('E') # # Yes, we need to call the reacqgil() function if not self.result_value_saved_early: @@ -374,9 +369,7 @@ self.restore_result_value(save_edx=False) # # patch the JE above - offset = mc.get_relative_pos() - je_location - assert 0 < offset <= 127 - mc.overwrite(je_location-1, chr(offset)) + mc.patch_forward_jump(je_location) # if restore_edx: mc.MOV_rs(edx.value, 12) # restore this diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py --- a/rpython/jit/backend/x86/codebuf.py +++ b/rpython/jit/backend/x86/codebuf.py @@ -1,12 +1,13 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import intmask +from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.debug import have_debug_prints from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from rpython.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder from rpython.jit.backend.x86.regloc import LocationCodeBuilder from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64, WORD -from rpython.jit.backend.x86 import valgrind +from rpython.jit.backend.x86 import rx86, valgrind # XXX: Seems nasty to change the superclass of MachineCodeBlockWrapper # like this @@ -17,6 +18,9 @@ codebuilder_cls = X86_64_CodeBuilder backend_name = 'x86_64' +class ShortJumpTooFar(Exception): + pass + class MachineCodeBlockWrapper(BlockBuilderMixin, LocationCodeBuilder, @@ -53,3 +57,22 @@ adr[0] = rffi.cast(rffi.INT, intmask(adr[0]) - p) valgrind.discard_translations(addr, self.get_relative_pos()) self._dump(addr, "jit-backend-dump", backend_name) + + @specialize.arg(1) + def emit_forward_jump(self, condition_string): + return self.emit_forward_jump_cond(rx86.Conditions[condition_string]) + + def emit_forward_jump_cond(self, cond): + self.J_il8(cond, 0) + return self.get_relative_pos() + + def emit_forward_jump_uncond(self): + self.JMP_l8(0) + return self.get_relative_pos() + + def patch_forward_jump(self, jcond_location): + offset = self.get_relative_pos() - jcond_location + assert offset >= 0 + if offset > 127: + raise ShortJumpTooFar + self.overwrite(jcond_location-1, chr(offset)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -464,28 +464,29 @@ how = SEEK_END return handle_posix_error('lseek', c_lseek(fd, pos, how)) -c_pread = external('pread', - [rffi.INT, rffi.VOIDP, rffi.SIZE_T , OFF_T], rffi.SSIZE_T, - save_err=rffi.RFFI_SAVE_ERRNO) -c_pwrite = external('pwrite', - [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T, - save_err=rffi.RFFI_SAVE_ERRNO) +if not _WIN32: + c_pread = external('pread', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T , OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) + c_pwrite = external('pwrite', + [rffi.INT, rffi.VOIDP, rffi.SIZE_T, OFF_T], rffi.SSIZE_T, + save_err=rffi.RFFI_SAVE_ERRNO) - at enforceargs(int, int, None) -def pread(fd, count, offset): - if count < 0: - raise OSError(errno.EINVAL, None) - validate_fd(fd) - with rffi.scoped_alloc_buffer(count) as buf: - void_buf = rffi.cast(rffi.VOIDP, buf.raw) - return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) - - at enforceargs(int, None, None) -def pwrite(fd, data, offset): - count = len(data) - validate_fd(fd) - with rffi.scoped_nonmovingbuffer(data) as buf: - return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) + @enforceargs(int, int, None) + def pread(fd, count, offset): + if count < 0: + raise OSError(errno.EINVAL, None) + validate_fd(fd) + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) + + @enforceargs(int, None, None) + def pwrite(fd, data, offset): + count = len(data) + validate_fd(fd) + with rffi.scoped_nonmovingbuffer(data) as buf: + return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO) From pypy.commits at gmail.com Tue Mar 14 10:42:42 2017 From: pypy.commits at gmail.com (rlamy) Date: Tue, 14 Mar 2017 07:42:42 -0700 (PDT) Subject: [pypy-commit] pypy stricter-encode: Raise error when attempting to encode surrogates to UTF16 or UTF32 Message-ID: <58c80162.4121190a.db54f.6ffe@mx.google.com> Author: Ronan Lamy Branch: stricter-encode Changeset: r90684:2ba11ddd00dc Date: 2017-03-14 14:39 +0000 http://bitbucket.org/pypy/pypy/changeset/2ba11ddd00dc/ Log: Raise error when attempting to encode surrogates to UTF16 or UTF32 diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -609,6 +609,9 @@ ch = ord(s[i]) i += 1 ch2 = 0 + if 0xD800 <= ch < 0xDFFF: + errorhandler( + errors, 'utf16', 'surrogates not allowed', s, i - 1, i) if ch >= 0x10000: ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) ch = 0xD800 | ((ch-0x10000) >> 10) @@ -774,6 +777,9 @@ ch = ord(s[i]) i += 1 ch2 = 0 + if 0xD800 <= ch < 0xDFFF: + errorhandler( + errors, 'utf32', 'surrogates not allowed', s, i - 1, i) if MAXUNICODE < 65536 and 0xD800 <= ch <= 0xDBFF and i < size: ch2 = ord(s[i]) if 0xDC00 <= ch2 <= 0xDFFF: From pypy.commits at gmail.com Tue Mar 14 17:02:04 2017 From: pypy.commits at gmail.com (mattip) Date: Tue, 14 Mar 2017 14:02:04 -0700 (PDT) Subject: [pypy-commit] pypy default: add draft of combined release doc Message-ID: <58c85a4c.0f472e0a.ca66.726e@mx.google.com> Author: Matti Picus Branch: Changeset: r90685:f0ba73fcc87c Date: 2017-03-14 22:59 +0200 http://bitbucket.org/pypy/pypy/changeset/f0ba73fcc87c/ Log: add draft of combined release doc diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.7.0.rst @@ -0,0 +1,182 @@ +============================================= +PyPy2.7 and PyPy3.5 v5.7 - two in one release +============================================= + +We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7. +This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and +PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version +3.5.3. + +We continue to make incremental improvements to our C-API +compatibility layer (cpyext). PyPy2 can now import and run many c-extension +packages, among the most notable are numpy, cython, and pandas. Performance may +be slower than CPython, especially for frequently-called short C functions. +Please let us know if your use case is slow, we have ideas how to make things +faster but need real-world examples (not micro-benchmarks) of problematic code. + +Work proceeds at a good pace on the PyPy3.5 +version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta +release. Thanks Mozilla !!! While we do not pass all tests, asyncio works +as `these benchmarks show`_ it already gives a nice speed bump. +We also backported the ``f""`` formatting from 3.6. + +CFFI_ has been updated to 1.10, improving an already great package for +interfacing with C. + +As always, this release fixed many issues and bugs raised by the +growing community of PyPy users. We strongly recommend updating. + +You can download the v5.7 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html +.. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) +============================================================================================= + +See also issues that were resolved_ + +* New features and cleanups + + * update the format of the PYPYLOG file and improvements to vmprof + * improve the consistency of RPython annotation unions + * emit more sysconfig values for downstream cextension packages + * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*, + _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork, + * add translation option --keepgoing to continue after the first AnnotationError + * detect and raise on recreation of a PyPy object from a PyObject during + tp_dealloc + * refactor and clean up poor handling of unicode exposed in work on py3.5 + * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) + * add translation time --disable_entrypoints option for embedding PyPy together + with another RPython VM + * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14 + * support translations with cpyext and the Boehm GC (for special cases like + revdb + * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is + now possible to obtain the address of any readonly object without pinning it + * refactor the initialization code in translating cpyext + * fix ``"".replace("", "x", num)`` to give the same result as CPython + * use a cffi-style C parser to create rffi objects in cpyext, now the + translating python must have cffi available + * add a rpython implementation of siphash24, allow choosing hash algorithm + randomizing the seed + * make ``attach_gdb`` work on Windows (with Visual Studio Debugger) + * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make + available as ``__pypy__.move_to_end`` and, on py3.5, + ``OrderedDict.move_to_end()`` + * remove completely RPython ``space.wrap`` in a major cleanup, differentiate + between ``space.newtext`` and ``space.newbytes`` on py3.5 + * improve shadowstack to where it is now the default in place of asmgcc + +* Bug Fixes + + * any uncaught RPython exception in the interpreter is turned into a + SystemError (rather than a segfault) + * create log files without the executable bit + * disable clock_gettime() on OS/X, since we support 10.11 and it was only + added in 10.12 + * support HAVE_FSTATVFS which was unintentionally always false + * fix user-created C-API heaptype, issue #2434 + * fix PyDict_Update is not actually the same as dict.update + * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute + issue #2446 + * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``, + ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` + * improve support for creating c-extension objects from app-level classes, + filling more slots especially ``tp_new`` and ``tp_dealloc`` + * add rstack.stack_almost_full() and use it to avoid stack overflow due to + the JIT where possible + * fix for ctypes.c_bool returning bool restype issue #2475 + * fix in corner cases with the GIL and C-API functions + +* Performance improvements: + + * clean-ups in the jit optimizeopt + * optimize ``if x is not None: return x`` or ``if x != 0: return x`` + * add ``jit.conditional_call_elidable()``, a way to tell the JIT + "conditonally call this function" returning a result + * try harder to propagate ``can_be_None=False`` information + * add ``rarithmetic.ovfcheck_int32_add/sub/mul`` + * add and use ``rgc.may_ignore_finalizer()``: an optimization hint that makes + the GC stop tracking the object + * replace malloc+memset with a single calloc, useful for large allocations? + * linux: try to implement os.urandom() as the syscall getrandom() if available + * propagate ``debug.ll_assert_not_none()`` through the JIT to reduce number of + guards + * improve the performance of ``PyDict_Next`` + * improve ``dict.pop()`` + * improve the optimization of branchy Python code by retaining more + information across failing guards + * add optimized "zero-copy" path for ``io.FileIO.readinto`` + +Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) +========================================================= + +Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo + +* New features + + * this first PyPy3.5 release implements much, but not all, of Python 3.5.3 + * PEP 456 allowing secure and interchangable hash algorithms + * use cryptography_'s cffi backend for SSL + +* Bug Fixes + + * implement fixes for some CPython issues that arose since the last release + * solve deadlocks in thread locking mechanism + +* Performance improvements: + + * do not create a list whenever descr_new of a bytesobject is called + * + * + * + +.. _resolved: whatsnew-pypy2-5.7.0.html +.. _cryptography: https://cryptography.io +.. _cppyy: cppyy.html + +Please update, and continue to help us make PyPy better. + +Cheers From pypy.commits at gmail.com Tue Mar 14 17:11:52 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 14:11:52 -0700 (PDT) Subject: [pypy-commit] pypy default: test and fix Message-ID: <58c85c98.58092e0a.ea952.7983@mx.google.com> Author: Armin Rigo Branch: Changeset: r90686:9cc33cbcd4a7 Date: 2017-03-14 22:11 +0100 http://bitbucket.org/pypy/pypy/changeset/9cc33cbcd4a7/ Log: test and fix diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1221,3 +1221,25 @@ pass bases = module.foo(C) assert bases == (A, B) + + def test_multiple_inheritance_old_style_base(self): + module = self.import_extension('foo', [ + ("foo", "METH_O", + ''' + PyTypeObject *tp; + tp = (PyTypeObject*)args; + Py_INCREF(tp->tp_bases); + return tp->tp_bases; + ''' + )]) + # used to segfault after some iterations + for i in range(11): + print i + class A(object): + pass + class B: + pass + class C(A, B): + pass + bases = module.foo(C) + assert bases == (A, B) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -931,7 +931,10 @@ if base: inherit_special(space, pto, base) for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): - inherit_slots(space, pto, w_base) + if isinstance(w_base, W_TypeObject): + inherit_slots(space, pto, w_base) + #else: + # w_base is a W_ClassObject, ignore it if not pto.c_tp_setattro: from pypy.module.cpyext.object import PyObject_GenericSetAttr From pypy.commits at gmail.com Tue Mar 14 17:24:42 2017 From: pypy.commits at gmail.com (mattip) Date: Tue, 14 Mar 2017 14:24:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: increment working version to 5.8, restart pypy3-head Message-ID: <58c85f9a.8d63190a.ea185.8bac@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r90687:92b0d855f603 Date: 2017-03-14 23:16 +0200 http://bitbucket.org/pypy/pypy/changeset/92b0d855f603/ Log: increment working version to 5.8, restart pypy3-head diff --git a/pypy/doc/whatsnew-pypy3-5.7.0.rst b/pypy/doc/whatsnew-pypy3-5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.7.0.rst @@ -0,0 +1,14 @@ +========================= +What's new in PyPy3 5.7.0 +========================= + +.. this is the revision after release-pypy3.3-5.5.x was branched +.. startrev: c5fb5db3c8ee + +.. branch: py3.5-time + +.. branch: py3.5-ssl + +.. branch: PEP393 + +Implement some level of compatibility with PEP 393 APIs. diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst deleted file mode 100644 --- a/pypy/doc/whatsnew-pypy3-head.rst +++ /dev/null @@ -1,14 +0,0 @@ -======================== -What's new in PyPy3 5.5+ -======================== - -.. this is the revision after release-pypy3.3-5.5.x was branched -.. startrev: c5fb5db3c8ee - -.. branch: py3.5-time - -.. branch: py3.5-ssl - -.. branch: PEP393 - -Implement some level of compatibility with PEP 393 APIs. \ No newline at end of file diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "3.5.2" /* PyPy version as a string */ -#define PYPY_VERSION "5.6.0-alpha0" -#define PYPY_VERSION_NUM 0x05060000 +#define PYPY_VERSION "5.8.0-alpha0" +#define PYPY_VERSION_NUM 0x05080000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 6, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 8, 0, "alpha", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Tue Mar 14 17:24:44 2017 From: pypy.commits at gmail.com (mattip) Date: Tue, 14 Mar 2017 14:24:44 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: restart pypy3-head Message-ID: <58c85f9c.02a2190a.2625e.8058@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r90688:27474b4d14e4 Date: 2017-03-14 23:16 +0200 http://bitbucket.org/pypy/pypy/changeset/27474b4d14e4/ Log: restart pypy3-head diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -0,0 +1,7 @@ +========================= +What's new in PyPy3 5.7+ +========================= + +.. this is the revision after release-pypy3.3-5.7.x was branched +.. startrev: afbf09453369 + From pypy.commits at gmail.com Tue Mar 14 17:24:49 2017 From: pypy.commits at gmail.com (mattip) Date: Tue, 14 Mar 2017 14:24:49 -0700 (PDT) Subject: [pypy-commit] pypy default: copy file to default Message-ID: <58c85fa1.548d190a.113a8.813c@mx.google.com> Author: Matti Picus Branch: Changeset: r90690:617809ac4251 Date: 2017-03-14 23:20 +0200 http://bitbucket.org/pypy/pypy/changeset/617809ac4251/ Log: copy file to default diff --git a/pypy/doc/whatsnew-pypy3-5.7.0.rst b/pypy/doc/whatsnew-pypy3-5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.7.0.rst @@ -0,0 +1,14 @@ +========================= +What's new in PyPy3 5.7.0 +========================= + +.. this is the revision after release-pypy3.3-5.5.x was branched +.. startrev: c5fb5db3c8ee + +.. branch: py3.5-time + +.. branch: py3.5-ssl + +.. branch: PEP393 + +Implement some level of compatibility with PEP 393 APIs. From pypy.commits at gmail.com Tue Mar 14 17:24:47 2017 From: pypy.commits at gmail.com (mattip) Date: Tue, 14 Mar 2017 14:24:47 -0700 (PDT) Subject: [pypy-commit] pypy default: increment default version to 5.8.0-alpha Message-ID: <58c85f9f.0f472e0a.ca66.7344@mx.google.com> Author: Matti Picus Branch: Changeset: r90689:a5dc5f7a99f6 Date: 2017-03-14 23:04 +0200 http://bitbucket.org/pypy/pypy/changeset/a5dc5f7a99f6/ Log: increment default version to 5.8.0-alpha diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.13" /* PyPy version as a string */ -#define PYPY_VERSION "5.7.0-alpha0" -#define PYPY_VERSION_NUM 0x05070000 +#define PYPY_VERSION "5.8.0-alpha0" +#define PYPY_VERSION_NUM 0x05080000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 7, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 8, 0, "alpha", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Tue Mar 14 17:24:51 2017 From: pypy.commits at gmail.com (mattip) Date: Tue, 14 Mar 2017 14:24:51 -0700 (PDT) Subject: [pypy-commit] pypy default: restart pypy3-head Message-ID: <58c85fa3.1a0e2e0a.ccb2e.8996@mx.google.com> Author: Matti Picus Branch: Changeset: r90691:8e5a1094ca6c Date: 2017-03-14 23:20 +0200 http://bitbucket.org/pypy/pypy/changeset/8e5a1094ca6c/ Log: restart pypy3-head diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -0,0 +1,7 @@ +========================= +What's new in PyPy3 5.7+ +========================= + +.. this is the revision after release-pypy3.3-5.7.x was branched +.. startrev: afbf09453369 + From pypy.commits at gmail.com Tue Mar 14 17:40:05 2017 From: pypy.commits at gmail.com (arigo) Date: Tue, 14 Mar 2017 14:40:05 -0700 (PDT) Subject: [pypy-commit] pypy default: tweaks in this paragraph Message-ID: <58c86335.065c2e0a.f1ceb.8154@mx.google.com> Author: Armin Rigo Branch: Changeset: r90692:424e6359b48e Date: 2017-03-14 22:39 +0100 http://bitbucket.org/pypy/pypy/changeset/424e6359b48e/ Log: tweaks in this paragraph diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -16,9 +16,10 @@ Work proceeds at a good pace on the PyPy3.5 version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta -release. Thanks Mozilla !!! While we do not pass all tests, asyncio works +release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and as `these benchmarks show`_ it already gives a nice speed bump. -We also backported the ``f""`` formatting from 3.6. +We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise +"PyPy3.5" supports the Python 3.5 language). CFFI_ has been updated to 1.10, improving an already great package for interfacing with C. From pypy.commits at gmail.com Tue Mar 14 17:58:35 2017 From: pypy.commits at gmail.com (mattip) Date: Tue, 14 Mar 2017 14:58:35 -0700 (PDT) Subject: [pypy-commit] pypy default: update contributors Message-ID: <58c8678b.06d2190a.c9b49.7db0@mx.google.com> Author: Matti Picus Branch: Changeset: r90693:699382943bd7 Date: 2017-03-14 23:57 +0200 http://bitbucket.org/pypy/pypy/changeset/699382943bd7/ Log: update contributors diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,5 @@ +#encoding utf-8 + License ======= @@ -42,9 +44,9 @@ Antonio Cuni Samuele Pedroni Matti Picus + Ronan Lamy Alex Gaynor Philip Jenvey - Ronan Lamy Brian Kearns Richard Plangger Michael Hudson @@ -55,12 +57,12 @@ Hakan Ardo Benjamin Peterson Anders Chrigstrom + Wim Lavrijsen Eric van Riet Paap - Wim Lavrijsen Richard Emslie Alexander Schremmer + Remi Meier Dan Villiom Podlaski Christiansen - Remi Meier Lukas Diekmann Sven Hager Anders Lehmann @@ -83,8 +85,8 @@ Lawrence Oluyede Bartosz Skowron Daniel Roberts + Adrien Di Mascio Niko Matsakis - Adrien Di Mascio Alexander Hesse Ludovic Aubry Jacob Hallen @@ -100,8 +102,8 @@ Michael Foord Stephan Diehl Stefan Schwarzer + Tomek Meka Valentino Volonghi - Tomek Meka Stefano Rivera Patrick Maupin Devin Jeanpierre @@ -109,268 +111,273 @@ Bruno Gola David Malcolm Jean-Paul Calderone - Timo Paulssen Edd Barrett Squeaky + Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton + Nicolas Truessel Martin Matusiak - Nicolas Truessel + Wenzhu Man Konstantin Lopuhin - Wenzhu Man John Witulski Laurence Tratt + Greg Price Ivan Sichmann Freitas - Greg Price Dario Bertini + Jeremy Thurgood Mark Pearse Simon Cross - Jeremy Thurgood + Tobias Pape Andreas Stührk - Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski + William Leslie + marky1991 + Ilya Osadchiy + Tobias Oberstein Paul deGrandis - Ilya Osadchiy - marky1991 - Tobias Oberstein + Boris Feigin + Taavi Burns Adrian Kuhn - Boris Feigin tav - Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp + Mike Blume Gerald Klix - Mike Blume Oscar Nierstrasz + Rami Chowdhury Stefan H. Muller - Rami Chowdhury + Joannah Nanjekye Eugene Oden + Tim Felgentreff + Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton - Jeff Terrace - Tim Felgentreff Dusty Phillips Lukas Renggli Guenter Jantzen - William Leslie Ned Batchelder + Amit Regmi Anton Gulenko - Amit Regmi - Ben Young + Sergey Matyunin Jasper Schulz + Andrew Chambers Nicolas Chauvat Andrew Durdin - Andrew Chambers - Sergey Matyunin + Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu + Michael Twomey Rocco Moretti Gintautas Miliauskas - Michael Twomey Lucian Branescu Mihaila anatoly techtonik + Karl Bartel Gabriel Lavoie + Jared Grubb Olivier Dormond - Jared Grubb - Karl Bartel Wouter van Heyst + Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina - Sebastian Pawluś - Stuart Williams - Daniel Patrick Aaron Iles Toby Watson + Daniel Patrick + Stuart Williams Antoine Pitrou Christian Hudon + Justas Sadzevicius + Neil Shepperd Michael Cheng - Justas Sadzevicius + Mikael Schönenberg + Stanislaw Halik + Berkin Ilbeyi Gasper Zejn - Neil Shepperd - Stanislaw Halik - Mikael Schönenberg - Berkin Ilbeyi Faye Zhao Elmo Mäntynen - Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani + Jonathan David Riehl Beatrice During Alex Perry + p_zieschang at yahoo.de + Robert Zaremba + Alan McIntyre + Alexander Sedov Vaibhav Sood - Alan McIntyre Reuben Cummings - Alexander Sedov - p_zieschang at yahoo.de Attila Gobi Christopher Pope - Aaron Gallagher + Tristan Arthur + Christian Tismer + Dan Stromberg + Carl Meyer Florin Papa - Christian Tismer - Marc Abramowitz - Dan Stromberg - Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Marc Abramowitz + Arjun Naik + Aaron Gallagher Alexis Daboville - Jens-Uwe Mager - Carl Meyer + Pieter Zieschang Karl Ramm - Pieter Zieschang - Gabriel Lukas Vacek - Kunal Grover - Andrew Dalke + Omer Katz + Jacek Generowicz Sylvain Thenault Jakub Stasiak + Stefan Beyer + Andrew Dalke + Alejandro J. Cura + Vladimir Kryachko + Gabriel + Mark Williams + Kunal Grover Nathan Taylor - Vladimir Kryachko - Omer Katz - Mark Williams - Jacek Generowicz - Alejandro J. Cura + Travis Francis Athougies + Yasir Suhail + Sergey Kishchenko + Martin Blais + Lutz Paelike + Ian Foote + Philipp Rustemeuer + Catalin Gabriel Manciu Jacob Oscarson - Travis Francis Athougies Ryan Gonzalez - Ian Foote Kristjan Valur Jonsson + Lucio Torre + Richard Lancaster + Dan Buch + Lene Wagner + Tomo Cocoa + Alecsandru Patrascu David Lievens Neil Blakey-Milner - Lutz Paelike - Lucio Torre + Henrik Vendelbo Lars Wassermann - Philipp Rustemeuer - Henrik Vendelbo - Richard Lancaster - Yasir Suhail - Dan Buch + Ignas Mikalajunas + Christoph Gerum Miguel de Val Borro Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Alecsandru Patrascu - Christoph Gerum - Martin Blais - Lene Wagner - Catalin Gabriel Manciu - Tomo Cocoa - Kim Jin Su - rafalgalczynski at gmail.com Toni Mattis - Amber Brown + Laurens Van Houtven + Bobby Impollonia + Roberto De Ioris + Jeong YunWon + Christopher Armstrong + Aaron Tubbs + Vasantha Ganesh K + Jason Michalski + Markus Holtermann + Andrew Thompson + Yusei Tahara + Ruochen Huang + Fabio Niephaus + Akira Li + Gustavo Niemeyer + Rafał Gałczyński + Logan Chien Lucas Stadler - Julian Berman - Markus Holtermann roberto at goyle + Matt Bogosian Yury V. Zaytsev - Anna Katrina Dominguez - Bobby Impollonia - Vasantha Ganesh K - Andrew Thompson florinpapa - Yusei Tahara - Aaron Tubbs - Ben Darnell - Roberto De Ioris - Logan Chien - Juan Francisco Cantero Hurtado - Ruochen Huang - Jeong YunWon - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle Anders Sigfridsson Nikolay Zinov - Jason Michalski + rafalgalczynski at gmail.com + Joshua Gilbert + Anna Katrina Dominguez + Kim Jin Su + Amber Brown + Ben Darnell + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Julian Berman + Michael Hudson-Doyle Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer Stephan Busemann - Rafał Gałczyński - Matt Bogosian + Dan Colish timo - Christian Muirhead - Berker Peksag - James Lan Volodymyr Vladymyrov - shoma hosaka - Ben Mather - Niclas Olofsson - Matthew Miller - Rodrigo Araújo + Daniel Neuhäuser + Flavio Percoco halgari - Boglarka Vezer - Chris Pressey - Buck Golemon - Diana Popa - Konrad Delong - Dinu Gherman + Jim Baker Chris Lambacher coolbutuseless at gmail.com + Mike Bayer + Rodrigo Araújo Daniil Yarancev - Jim Baker + OlivierBlanvillain + Jonas Pfannschmidt + Zearin + Andrey Churin Dan Crosta - Nikolaos-Digenis Karagiannis - James Robert - Armin Ronacher - Brett Cannon - Donald Stufft - yrttyr - aliceinwire - OlivierBlanvillain - Dan Sanders - Zooko Wilcox-O Hearn + reubano at gmail.com + Julien Phalip + Roman Podoliaka + Eli Stevens + Boglarka Vezer + PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio - jiaaro - Mads Kiilerich Antony Lee - Jason Madden - Daniel Neuh�user - reubano at gmail.com - Yaroslav Fedevych Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - squeaky - soareschen - Jonas Pfannschmidt - Kurt Griffiths - Mike Bayer - Stefan Marr - Flavio Percoco - Kristoffer Kleine + shoma hosaka + Buck Golemon + JohnDoe + yrttyr Michael Chermside Anna Ravencroft + remarkablerocket + Berker Peksag + Christian Muirhead + soareschen + Matthew Miller + Konrad Delong + Dinu Gherman pizi - remarkablerocket - Andrey Churin - Zearin - Eli Stevens - Tobias Diaz - Julien Phalip - Roman Podoliaka + James Robert + Armin Ronacher + Diana Popa + Mads Kiilerich + Brett Cannon + aliceinwire + Zooko Wilcox-O Hearn + James Lan + jiaaro + Markus Unterwaditzer + Kristoffer Kleine + Graham Markall Dan Loewenherz werat + Niclas Olofsson + Chris Pressey + Tobias Diaz + Nikolaos-Digenis Karagiannis + Kurt Griffiths + Ben Mather + Donald Stufft + Dan Sanders + Jason Madden + Yaroslav Fedevych + Even Wiik Thomassen + Stefan Marr Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -11,9 +11,9 @@ Antonio Cuni Samuele Pedroni Matti Picus + Ronan Lamy Alex Gaynor Philip Jenvey - Ronan Lamy Brian Kearns Richard Plangger Michael Hudson @@ -24,12 +24,12 @@ Hakan Ardo Benjamin Peterson Anders Chrigstrom + Wim Lavrijsen Eric van Riet Paap - Wim Lavrijsen Richard Emslie Alexander Schremmer + Remi Meier Dan Villiom Podlaski Christiansen - Remi Meier Lukas Diekmann Sven Hager Anders Lehmann @@ -52,8 +52,8 @@ Lawrence Oluyede Bartosz Skowron Daniel Roberts + Adrien Di Mascio Niko Matsakis - Adrien Di Mascio Alexander Hesse Ludovic Aubry Jacob Hallen @@ -69,8 +69,8 @@ Michael Foord Stephan Diehl Stefan Schwarzer + Tomek Meka Valentino Volonghi - Tomek Meka Stefano Rivera Patrick Maupin Devin Jeanpierre @@ -78,267 +78,270 @@ Bruno Gola David Malcolm Jean-Paul Calderone - Timo Paulssen Edd Barrett Squeaky + Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton + Nicolas Truessel Martin Matusiak - Nicolas Truessel + Wenzhu Man Konstantin Lopuhin - Wenzhu Man John Witulski Laurence Tratt + Greg Price Ivan Sichmann Freitas - Greg Price Dario Bertini + Jeremy Thurgood Mark Pearse Simon Cross - Jeremy Thurgood + Tobias Pape Andreas Stührk - Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski + William Leslie + marky1991 + Ilya Osadchiy + Tobias Oberstein Paul deGrandis - Ilya Osadchiy - marky1991 - Tobias Oberstein + Boris Feigin + Taavi Burns Adrian Kuhn - Boris Feigin tav - Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp + Mike Blume Gerald Klix - Mike Blume Oscar Nierstrasz + Rami Chowdhury Stefan H. Muller - Rami Chowdhury + Joannah Nanjekye Eugene Oden + Tim Felgentreff + Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton - Jeff Terrace - Tim Felgentreff Dusty Phillips Lukas Renggli Guenter Jantzen - William Leslie Ned Batchelder + Amit Regmi Anton Gulenko - Amit Regmi - Ben Young + Sergey Matyunin Jasper Schulz + Andrew Chambers Nicolas Chauvat Andrew Durdin - Andrew Chambers - Sergey Matyunin + Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu + Michael Twomey Rocco Moretti Gintautas Miliauskas - Michael Twomey Lucian Branescu Mihaila anatoly techtonik + Karl Bartel Gabriel Lavoie + Jared Grubb Olivier Dormond - Jared Grubb - Karl Bartel Wouter van Heyst + Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina - Sebastian Pawluś - Stuart Williams - Daniel Patrick Aaron Iles Toby Watson + Daniel Patrick + Stuart Williams Antoine Pitrou Christian Hudon + Justas Sadzevicius + Neil Shepperd Michael Cheng - Justas Sadzevicius + Mikael Schönenberg + Stanislaw Halik + Berkin Ilbeyi Gasper Zejn - Neil Shepperd - Stanislaw Halik - Mikael Schönenberg - Berkin Ilbeyi Faye Zhao Elmo Mäntynen - Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani + Jonathan David Riehl Beatrice During Alex Perry + p_zieschang at yahoo.de + Robert Zaremba + Alan McIntyre + Alexander Sedov Vaibhav Sood - Alan McIntyre Reuben Cummings - Alexander Sedov - p_zieschang at yahoo.de Attila Gobi Christopher Pope - Aaron Gallagher + Tristan Arthur + Christian Tismer + Dan Stromberg + Carl Meyer Florin Papa - Christian Tismer - Marc Abramowitz - Dan Stromberg - Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Marc Abramowitz + Arjun Naik + Aaron Gallagher Alexis Daboville - Jens-Uwe Mager - Carl Meyer + Pieter Zieschang Karl Ramm - Pieter Zieschang - Gabriel Lukas Vacek - Kunal Grover - Andrew Dalke + Omer Katz + Jacek Generowicz Sylvain Thenault Jakub Stasiak + Stefan Beyer + Andrew Dalke + Alejandro J. Cura + Vladimir Kryachko + Gabriel + Mark Williams + Kunal Grover Nathan Taylor - Vladimir Kryachko - Omer Katz - Mark Williams - Jacek Generowicz - Alejandro J. Cura + Travis Francis Athougies + Yasir Suhail + Sergey Kishchenko + Martin Blais + Lutz Paelike + Ian Foote + Philipp Rustemeuer + Catalin Gabriel Manciu Jacob Oscarson - Travis Francis Athougies Ryan Gonzalez - Ian Foote Kristjan Valur Jonsson + Lucio Torre + Richard Lancaster + Dan Buch + Lene Wagner + Tomo Cocoa + Alecsandru Patrascu David Lievens Neil Blakey-Milner - Lutz Paelike - Lucio Torre + Henrik Vendelbo Lars Wassermann - Philipp Rustemeuer - Henrik Vendelbo - Richard Lancaster - Yasir Suhail - Dan Buch + Ignas Mikalajunas + Christoph Gerum Miguel de Val Borro Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Alecsandru Patrascu - Christoph Gerum - Martin Blais - Lene Wagner - Catalin Gabriel Manciu - Tomo Cocoa - Kim Jin Su - rafalgalczynski at gmail.com Toni Mattis - Amber Brown + Laurens Van Houtven + Bobby Impollonia + Roberto De Ioris + Jeong YunWon + Christopher Armstrong + Aaron Tubbs + Vasantha Ganesh K + Jason Michalski + Markus Holtermann + Andrew Thompson + Yusei Tahara + Ruochen Huang + Fabio Niephaus + Akira Li + Gustavo Niemeyer + Rafał Gałczyński + Logan Chien Lucas Stadler - Julian Berman - Markus Holtermann roberto at goyle + Matt Bogosian Yury V. Zaytsev - Anna Katrina Dominguez - Bobby Impollonia - Vasantha Ganesh K - Andrew Thompson florinpapa - Yusei Tahara - Aaron Tubbs - Ben Darnell - Roberto De Ioris - Logan Chien - Juan Francisco Cantero Hurtado - Ruochen Huang - Jeong YunWon - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle Anders Sigfridsson Nikolay Zinov - Jason Michalski + rafalgalczynski at gmail.com + Joshua Gilbert + Anna Katrina Dominguez + Kim Jin Su + Amber Brown + Ben Darnell + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Julian Berman + Michael Hudson-Doyle Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer Stephan Busemann - Rafał Gałczyński - Matt Bogosian + Dan Colish timo - Christian Muirhead - Berker Peksag - James Lan Volodymyr Vladymyrov - shoma hosaka - Ben Mather - Niclas Olofsson - Matthew Miller - Rodrigo Araújo + Daniel Neuhäuser + Flavio Percoco halgari - Boglarka Vezer - Chris Pressey - Buck Golemon - Diana Popa - Konrad Delong - Dinu Gherman + Jim Baker Chris Lambacher coolbutuseless at gmail.com + Mike Bayer + Rodrigo Araújo Daniil Yarancev - Jim Baker + OlivierBlanvillain + Jonas Pfannschmidt + Zearin + Andrey Churin Dan Crosta - Nikolaos-Digenis Karagiannis - James Robert - Armin Ronacher - Brett Cannon - Donald Stufft - yrttyr - aliceinwire - OlivierBlanvillain - Dan Sanders - Zooko Wilcox-O Hearn + reubano at gmail.com + Julien Phalip + Roman Podoliaka + Eli Stevens + Boglarka Vezer + PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio - jiaaro - Mads Kiilerich Antony Lee - Jason Madden - Daniel Neuhäuser - reubano at gmail.com - Yaroslav Fedevych Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - squeaky - soareschen - Jonas Pfannschmidt - Kurt Griffiths - Mike Bayer - Stefan Marr - Flavio Percoco - Kristoffer Kleine + shoma hosaka + Buck Golemon + JohnDoe + yrttyr Michael Chermside Anna Ravencroft + remarkablerocket + Berker Peksag + Christian Muirhead + soareschen + Matthew Miller + Konrad Delong + Dinu Gherman pizi - remarkablerocket - Andrey Churin - Zearin - Eli Stevens - Tobias Diaz - Julien Phalip - Roman Podoliaka + James Robert + Armin Ronacher + Diana Popa + Mads Kiilerich + Brett Cannon + aliceinwire + Zooko Wilcox-O Hearn + James Lan + jiaaro + Markus Unterwaditzer + Kristoffer Kleine + Graham Markall Dan Loewenherz werat - - + Niclas Olofsson + Chris Pressey + Tobias Diaz + Nikolaos-Digenis Karagiannis + Kurt Griffiths + Ben Mather + Donald Stufft + Dan Sanders + Jason Madden + Yaroslav Fedevych + Even Wiik Thomassen + Stefan Marr diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -75,9 +75,10 @@ 'Spenser Bauman':['Spenser Andrew Bauman'], 'Raffael Tfirst':['raffael.tfirst at gmail.com'], 'timo':['timo at eistee.fritz.box'], - 'Jasper Schulz':['Jasper.Schulz'], + 'Jasper Schulz':['Jasper.Schulz', 'jbs'], 'Aaron Gallagher':['"Aaron Gallagher'], 'Yasir Suhail':['yasirs'], + 'Squeaky', ['squeaky'], } alias_map = {} From pypy.commits at gmail.com Wed Mar 15 04:07:51 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 01:07:51 -0700 (PDT) Subject: [pypy-commit] cffi release-1.10: release branch Message-ID: <58c8f657.02502e0a.16894.057b@mx.google.com> Author: Armin Rigo Branch: release-1.10 Changeset: r2910:a051c529f3b3 Date: 2017-03-15 09:07 +0100 http://bitbucket.org/cffi/cffi/changeset/a051c529f3b3/ Log: release branch From pypy.commits at gmail.com Wed Mar 15 06:29:42 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 03:29:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Match the description of '-u' to its more correct variant in the python Message-ID: <58c91796.44db190a.d26e.1852@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90694:45a687d2768f Date: 2017-03-15 11:28 +0100 http://bitbucket.org/pypy/pypy/changeset/45a687d2768f/ Log: Match the description of '-u' to its more correct variant in the python man page, and not the error-inducing one in 'python3 --help'. See also http://bugs.python.org/issue28647 diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -19,8 +19,9 @@ -q : don't print version and copyright messages on interactive startup -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE -S : don't imply 'import site' on initialization --u : unbuffered binary stdout and stderr, stdin always buffered; - also PYTHONUNBUFFERED=x +-u : force the binary I/O layers of stdout and stderr to be unbuffered. + stdin is always buffered. the text I/O layer will still be + line-buffered. see also PYTHONUNBUFFERED=x -v : verbose (trace import statements); also PYTHONVERBOSE=x can be supplied multiple times to increase verbosity -V : print the Python version number and exit (also --version) From pypy.commits at gmail.com Wed Mar 15 07:01:53 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 04:01:53 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Copy more closely the logic of sysconfig_cpython.py. Fix for the Message-ID: <58c91f21.011c190a.b1c04.1952@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90695:b517a201935d Date: 2017-03-15 12:01 +0100 http://bitbucket.org/pypy/pypy/changeset/b517a201935d/ Log: Copy more closely the logic of sysconfig_cpython.py. Fix for the 'venv' standard module on pypy3. diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py --- a/lib-python/3/distutils/sysconfig_pypy.py +++ b/lib-python/3/distutils/sysconfig_pypy.py @@ -19,13 +19,16 @@ PREFIX = os.path.normpath(sys.prefix) EXEC_PREFIX = os.path.normpath(sys.exec_prefix) +BASE_PREFIX = os.path.normpath(sys.base_prefix) +BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) project_base = os.path.dirname(os.path.abspath(sys.executable)) python_build = False def get_python_inc(plat_specific=0, prefix=None): - from os.path import join as j - return j(sys.prefix, 'include') + if prefix is None: + prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX + return os.path.join(prefix, 'include') def get_python_version(): """Return a string containing the major and minor Python version, From pypy.commits at gmail.com Wed Mar 15 08:08:39 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 15 Mar 2017 05:08:39 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: merge default Message-ID: <58c92ec7.039d190a.1d9a7.1e74@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90696:107955f416ca Date: 2017-03-14 14:30 +0100 http://bitbucket.org/pypy/pypy/changeset/107955f416ca/ Log: merge default diff too long, truncating to 2000 out of 37236 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -60,6 +60,9 @@ ^lib_pypy/ctypes_config_cache/_.+_cache\.py$ ^lib_pypy/ctypes_config_cache/_.+_.+_\.py$ ^lib_pypy/_libmpdec/.+.o$ +^lib_pypy/.+.c$ +^lib_pypy/.+.o$ +^lib_pypy/.+.so$ ^pypy/doc/discussion/.+\.html$ ^include/.+\.h$ ^include/.+\.inl$ @@ -74,7 +77,7 @@ ^rpython/doc/_build/.*$ ^compiled ^.git/ -^.hypothesis/ +.hypothesis/ ^release/ ^rpython/_cache$ diff --git a/extra_tests/README.txt b/extra_tests/README.txt new file mode 100644 --- /dev/null +++ b/extra_tests/README.txt @@ -0,0 +1,5 @@ +The tests in this directory are a complement to lib-python/3/test/. + +They are meant to run on top of a compiled pypy3 or CPython3.5 in an +environment containing at least pytest and hypothesis, using a command like +'pytest extra_tests/'. diff --git a/extra_tests/pytest.ini b/extra_tests/pytest.ini new file mode 100644 diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_unicode.py @@ -0,0 +1,34 @@ +import pytest +from hypothesis import strategies as st +from hypothesis import given, settings, example + +from unicodedata import normalize + +# For every (n1, n2, n3) triple, applying n1 then n2 must be the same +# as applying n3. +# Reference: http://unicode.org/reports/tr15/#Design_Goals +compositions = [ + ('NFC', 'NFC', 'NFC'), + ('NFC', 'NFD', 'NFD'), + ('NFC', 'NFKC', 'NFKC'), + ('NFC', 'NFKD', 'NFKD'), + ('NFD', 'NFC', 'NFC'), + ('NFD', 'NFD', 'NFD'), + ('NFD', 'NFKC', 'NFKC'), + ('NFD', 'NFKD', 'NFKD'), + ('NFKC', 'NFC', 'NFKC'), + ('NFKC', 'NFD', 'NFKD'), + ('NFKC', 'NFKC', 'NFKC'), + ('NFKC', 'NFKD', 'NFKD'), + ('NFKD', 'NFC', 'NFKC'), + ('NFKD', 'NFD', 'NFKD'), + ('NFKD', 'NFKC', 'NFKC'), + ('NFKD', 'NFKD', 'NFKD'), +] + + at pytest.mark.parametrize('norm1, norm2, norm3', compositions) + at settings(max_examples=1000) + at example(s=u'---\uafb8\u11a7---') # issue 2289 + at given(s=st.text()) +def test_composition(s, norm1, norm2, norm3): + assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s) diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -33,6 +33,10 @@ from __pypy__ import reversed_dict as _reversed_dict except ImportError: _reversed_dict = None # don't have ordered dicts +try: + from __pypy__ import dict_popitem_first as _dict_popitem_first +except ImportError: + _dict_popitem_first = None try: from thread import get_ident as _get_ident @@ -44,6 +48,17 @@ ### OrderedDict ################################################################################ +if _dict_popitem_first is None: + def _dict_popitem_first(self): + it = dict.iteritems(self) + try: + k, v = it.next() + except StopIteration: + raise KeyError('dictionary is empty') + dict.__delitem__(self, k) + return (k, v) + + class OrderedDict(dict): '''Dictionary that remembers insertion order. @@ -68,12 +83,7 @@ if last: return dict.popitem(self) else: - it = dict.__iter__(self) - try: - k = it.next() - except StopIteration: - raise KeyError('dictionary is empty') - return (k, self.pop(k)) + return _dict_popitem_first(self) def __repr__(self, _repr_running={}): 'od.__repr__() <==> repr(od)' diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -369,11 +369,8 @@ def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - # in cPython, _sysconfigdata is generated at build time, see _generate_posix_vars() - # in PyPy no such module exists - #from _sysconfigdata import build_time_vars - #vars.update(build_time_vars) - return + from _sysconfigdata import build_time_vars + vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" @@ -529,7 +526,9 @@ for suffix, mode, type_ in imp.get_suffixes(): if type_ == imp.C_EXTENSION: _CONFIG_VARS['SOABI'] = suffix.split('.')[1] - break + break + _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'], + 'include') if args: vals = [] diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -604,7 +604,8 @@ """ # hack for performance: if restype is a "simple" primitive type, don't # allocate the buffer because it's going to be thrown away immediately - if self._is_primitive(restype) and not restype._is_pointer_like(): + if (self._is_primitive(restype) and restype._type_ != '?' + and not restype._is_pointer_like()): return result # shape = restype._ffishape_ diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_sysconfigdata.py @@ -0,0 +1,5 @@ +import imp + +build_time_vars = { + "SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] +} diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -570,7 +570,10 @@ # we need 'libpypy-c.{so,dylib}', which should be by # default located in 'sys.prefix/bin' for installed # systems. - pythonlib = "pypy-c" + if sys.version_info < (3,): + pythonlib = "pypy-c" + else: + pythonlib = "pypy3-c" if hasattr(sys, 'prefix'): ensure('library_dirs', os.path.join(sys.prefix, 'bin')) # On uninstalled pypy's, the libpypy-c is typically found in @@ -756,21 +759,27 @@ def _load_backend_lib(backend, name, flags): + import os if name is None: if sys.platform != "win32": return backend.load_library(None, flags) name = "c" # Windows: load_library(None) fails, but this works # (backward compatibility hack only) - try: - if '.' not in name and '/' not in name: - raise OSError("library not found: %r" % (name,)) - return backend.load_library(name, flags) - except OSError: - import ctypes.util - path = ctypes.util.find_library(name) - if path is None: - raise # propagate the original OSError - return backend.load_library(path, flags) + first_error = None + if '.' in name or '/' in name or os.sep in name: + try: + return backend.load_library(name, flags) + except OSError as e: + first_error = e + import ctypes.util + path = ctypes.util.find_library(name) + if path is None: + msg = ("ctypes.util.find_library() did not manage " + "to locate a library called %r" % (name,)) + if first_error is not None: + msg = "%s. Additionally, %s" % (first_error, msg) + raise OSError(msg) + return backend.load_library(path, flags) def _make_ffi_library(ffi, libname, flags): backend = ffi._backend diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -112,11 +112,20 @@ def _make_cmp(name): cmpfunc = getattr(operator, name) def cmp(self, other): - if isinstance(other, CTypesData): + v_is_ptr = not isinstance(self, CTypesGenericPrimitive) + w_is_ptr = (isinstance(other, CTypesData) and + not isinstance(other, CTypesGenericPrimitive)) + if v_is_ptr and w_is_ptr: return cmpfunc(self._convert_to_address(None), other._convert_to_address(None)) + elif v_is_ptr or w_is_ptr: + return NotImplemented else: - return NotImplemented + if isinstance(self, CTypesGenericPrimitive): + self = self._value + if isinstance(other, CTypesGenericPrimitive): + other = other._value + return cmpfunc(self, other) cmp.func_name = name return cmp @@ -128,7 +137,7 @@ __ge__ = _make_cmp('__ge__') def __hash__(self): - return hash(type(self)) ^ hash(self._convert_to_address(None)) + return hash(self._convert_to_address(None)) def _to_string(self, maxlen): raise TypeError("string(): %r" % (self,)) @@ -137,14 +146,8 @@ class CTypesGenericPrimitive(CTypesData): __slots__ = [] - def __eq__(self, other): - return self is other - - def __ne__(self, other): - return self is not other - def __hash__(self): - return object.__hash__(self) + return hash(self._value) def _get_own_repr(self): return repr(self._from_ctypes(self._value)) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -34,6 +34,9 @@ r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') _r_star_const_space = re.compile( # matches "* const " r"[*]\s*((const|volatile|restrict)\b\s*)+") +_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+" + r"\.\.\.") +_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.") def _get_parser(): global _parser_cache @@ -180,6 +183,10 @@ assert csource[p:p+3] == '...' csource = '%s __dotdotdot%d__ %s' % (csource[:p], number, csource[p+3:]) + # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__" + csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource) + # Replace "float ..." or "double..." with "__dotdotdotfloat__" + csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) # Replace all remaining "..." with the same name, "__dotdotdot__", # which is declared with a typedef for the purpose of C parsing. return csource.replace('...', ' __dotdotdot__ '), macros @@ -252,7 +259,8 @@ typenames += sorted(ctn) # csourcelines = ['typedef int %s;' % typename for typename in typenames] - csourcelines.append('typedef int __dotdotdot__;') + csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,' + ' __dotdotdot__;') csourcelines.append(csource) csource = '\n'.join(csourcelines) if lock is not None: @@ -311,6 +319,8 @@ for decl in iterator: if decl.name == '__dotdotdot__': break + else: + assert 0 # try: self._inside_extern_python = '__cffi_extern_python_stop' @@ -322,15 +332,15 @@ raise CDefError("typedef does not declare any name", decl) quals = 0 - if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) - and decl.type.type.names[-1] == '__dotdotdot__'): + if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and + decl.type.type.names[-1].startswith('__dotdotdot')): realtype = self._get_unknown_type(decl) elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and isinstance(decl.type.type.type, pycparser.c_ast.IdentifierType) and - decl.type.type.type.names == ['__dotdotdot__']): - realtype = model.unknown_ptr_type(decl.name) + decl.type.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_ptr_type(decl) else: realtype, quals = self._get_type_and_quals( decl.type, name=decl.name, partial_length_ok=True) @@ -832,24 +842,25 @@ def _get_unknown_type(self, decl): typenames = decl.type.type.names - assert typenames[-1] == '__dotdotdot__' - if len(typenames) == 1: + if typenames == ['__dotdotdot__']: return model.unknown_type(decl.name) - if (typenames[:-1] == ['float'] or - typenames[:-1] == ['double']): - # not for 'long double' so far - result = model.UnknownFloatType(decl.name) - else: - for t in typenames[:-1]: - if t not in ['int', 'short', 'long', 'signed', - 'unsigned', 'char']: - raise FFIError(':%d: bad usage of "..."' % - decl.coord.line) - result = model.UnknownIntegerType(decl.name) + if typenames == ['__dotdotdotint__']: + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef int... %s'" % decl.name + return model.UnknownIntegerType(decl.name) - if self._uses_new_feature is None: - self._uses_new_feature = "'typedef %s... %s'" % ( - ' '.join(typenames[:-1]), decl.name) + if typenames == ['__dotdotdotfloat__']: + # note: not for 'long double' so far + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef float... %s'" % decl.name + return model.UnknownFloatType(decl.name) - return result + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) + + def _get_unknown_ptr_type(self, decl): + if decl.type.type.type.names == ['__dotdotdot__']: + return model.unknown_ptr_type(decl.name) + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -49,7 +49,7 @@ ------------------------------- (**Note**: for some hints on how to translate the Python interpreter under Windows, see the `windows document`_ . For hints on how to cross-compile in -a chroot using scratchbox2, see the `arm document`_ in the +a chroot using scratchbox2, see the `arm document`_ in the `RPython documentation`_) .. _`windows document`: windows.html @@ -57,7 +57,7 @@ .. _`RPython documentation`: http://rpython.readthedocs.org The host Python needs to have CFFI installed. If translating on PyPy, CFFI is -already installed. If translating on CPython, you need to install it, e.g. +already installed. If translating on CPython, you need to install it, e.g. using ``pip install cffi``. To build PyPy on Unix using the C translation backend, you need at least a C @@ -116,7 +116,7 @@ On Fedora:: dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \ - lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \ + sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \ gdbm-devel \ xz-devel # For lzma on PyPy3. @@ -162,7 +162,8 @@ import libraries in the `out-of-line API mode`_. This is done by the following command:: - PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py + cd pypy/goal + PYTHONPATH=../.. ./pypy-c ../tool/build_cffi_imports.py .. _`out-of-line API mode`: http://cffi.readthedocs.org/en/latest/overview.html#real-example-api-level-out-of-line @@ -185,7 +186,7 @@ imported the first time. :: - + cd pypy/tool/release ./package.py pypy-VER-PLATFORM @@ -222,5 +223,3 @@ to continue normally. If the default path is usable, most code will be fine. However, the ``sys.prefix`` will be unset and some existing libraries assume that this is never the case. - - diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -17,6 +17,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.append(os.path.abspath('.')) +sys.path.append(os.path.abspath('../../')) # -- Read The Docs theme config ------------------------------------------------ diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -483,6 +483,11 @@ the rest is kept. If you return an unexpected string from ``__hex__()`` you get an exception (or a crash before CPython 2.7.13). +* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes. + CPython allows that for module subtypes, but not for e.g. ``int`` + or ``float`` subtypes. Currently PyPy does not support the + ``__class__`` attribute assignment for any non heaptype subtype. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -156,19 +156,30 @@ Does PyPy have a GIL? Why? ------------------------------------------------- -Yes, PyPy has a GIL. Removing the GIL is very hard. The problems are -essentially the same as with CPython (including the fact that our -garbage collectors are not thread-safe so far). Fixing it is possible, -as shown by Jython and IronPython, but difficult. It would require -adapting the whole source code of PyPy, including subtle decisions about -whether some effects are ok or not for the user (i.e. the Python -programmer). +Yes, PyPy has a GIL. Removing the GIL is very hard. On top of CPython, +you have two problems: (1) GC, in this case reference counting; (2) the +whole Python language. -Instead, since 2012, there is work going on on a still very experimental -:doc:`Software Transactional Memory ` (STM) version of PyPy. This should give -an alternative PyPy which works without a GIL, while at the same time -continuing to give the Python programmer the complete illusion of having -one. +For PyPy, the hard issue is (2): by that I mean issues like what occurs +if a mutable object is changed from one thread and read from another +concurrently. This is a problem for *any* mutable type: it needs +careful review and fixes (fine-grained locks, mostly) through the +*whole* Python interpreter. It is a major effort, although not +completely impossible, as Jython/IronPython showed. This includes +subtle decisions about whether some effects are ok or not for the user +(i.e. the Python programmer). + +CPython has additionally the problem (1) of reference counting. With +PyPy, this sub-problem is simpler: we need to make our GC +multithread-aware. This is easier to do efficiently in PyPy than in +CPython. It doesn't solve the issue (2), though. + +Note that since 2012 there is work going on on a still very experimental +:doc:`Software Transactional Memory ` (STM) version of PyPy. This +should give an alternative PyPy which works without a GIL, while at the +same time continuing to give the Python programmer the complete illusion +of having one. This work is currently a bit stalled because of its own +technical difficulties. Is PyPy more clever than CPython about Tail Calls? @@ -426,3 +437,11 @@ but so far there doesn't seem to be an overwhelming commercial interest in it. .. _`to make it happen`: windows.html#what-is-missing-for-a-full-64-bit-translation + + +How long will PyPy support Python2? +----------------------------------- + +Since RPython is built on top of Python2 and that is extremely unlikely to +change, the Python2 version of PyPy will be around "forever", i.e. as long as +PyPy itself is around. diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst --- a/pypy/doc/objspace.rst +++ b/pypy/doc/objspace.rst @@ -188,6 +188,7 @@ .. py:function:: wrap(x) + **Deprecated! Eventually this method should disappear.** Returns a wrapped object that is a reference to the interpreter-level object :py:obj:`x`. This can be used either on simple immutable objects (integers, strings, etc) to create a new wrapped object, or on instances of :py:class:`W_Root` @@ -196,6 +197,35 @@ be directly exposed to application-level code in this way - functions, frames, code objects, etc. +.. py:function:: newint(i) + + Creates a wrapped object holding an integral value. `newint` creates an object + of type `W_IntObject`. + +.. py:function:: newlong(l) + + Creates a wrapped object holding an integral value. The main difference to newint + is the type of the argument (which is rpython.rlib.rbigint.rbigint). On PyPy3 this + method will return an :py:class:`int` (PyPy2 it returns a :py:class:`long`). + +.. py:function:: newbytes(t) + + The given argument is a rpython bytestring. Creates a wrapped object + of type :py:class:`bytes` (both on PyPy2 and PyPy3). + +.. py:function:: newtext(t) + + The given argument is a rpython bytestring. Creates a wrapped object + of type :py:class:`str`. On PyPy3 this will return a wrapped unicode + object. The object will hold a utf-8-nosg decoded value of `t`. + The "utf-8-nosg" codec used here is slightly different from the + "utf-8" implemented in Python 2 or Python 3: it is defined as utf-8 + without any special handling of surrogate characters. They are + encoded using the same three-bytes sequence that encodes any char in + the range from ``'\u0800'`` to ``'\uffff'``. + + PyPy2 will return a bytestring object. No encoding/decoding steps will be applied. + .. py:function:: newbool(b) Creates a wrapped :py:class:`bool` object from an :ref:`interpreter-level ` @@ -217,15 +247,18 @@ Creates a new slice object. -.. py:function:: newstring(asciilist) +.. py:function:: newunicode(ustr) - Creates a string from a list of wrapped integers. Note that this may not be - a very useful method; usually you can just write ``space.wrap("mystring")``. + Creates a Unicode string from an rpython unicode string. + This method may disappear soon and be replaced by :py:function:`newutf8()`. -.. py:function:: newunicode(codelist) +.. py:function:: newutf8(bytestr) - Creates a Unicode string from a list of integers (code points). + Creates a Unicode string from an rpython byte string, decoded as + "utf-8-nosg". On PyPy3 it is the same as :py:function:`newtext()`. +Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and +`pypy/objspace/std/objspace.py`. Conversions from Application Level to Interpreter Level ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -258,11 +291,21 @@ If :py:obj:`w_x` is an application-level integer or long, return an interpreter-level :py:class:`rbigint`. Otherwise raise :py:exc:`TypeError`. +.. automethod:: pypy.interpreter.baseobjspace.ObjSpace.bytes_w(w_x) +.. automethod:: pypy.interpreter.baseobjspace.ObjSpace.text_w(w_x) + .. py:function:: str_w(w_x) + **Deprecated. use text_w or bytes_w instead** If :py:obj:`w_x` is an application-level string, return an interpreter-level string. Otherwise raise :py:exc:`TypeError`. +.. py:function:: unicode_w(w_x) + + Takes an application level :py:class:`unicode` and return an + interpreter-level unicode string. This method may disappear soon and + be replaced by :py:function:`text_w()`. + .. py:function:: float_w(w_x) If :py:obj:`w_x` is an application-level float, integer or long, return an diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,9 +1,29 @@ -Potential project list +Potential Project List ====================== -========================== +Google Summer of Code 2017 +-------------------------- + +PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. + + +* **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. + Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. + 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. + The general procedure of this project is: Run both CPython and PyPy of the same Python version and + compare the memory usage (using Massif or other tools). + If PyPy consumes a lot more memory then find and resolve the issue. + +* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. + +* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). + +* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. + +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. + Simple tasks for newcomers -========================== +-------------------------- * Tkinter module missing support for threads: https://bitbucket.org/pypy/pypy/issue/1929/tkinter-broken-for-threaded-python-on-both @@ -15,9 +35,8 @@ https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets -================== Mid-to-large tasks -================== +------------------ Below is a list of projects that are interesting for potential contributors who are seriously interested in the PyPy project. They mostly share common @@ -81,7 +100,7 @@ module. Improving the jitviewer ------------------------- +----------------------- Analyzing performance of applications is always tricky. We have various tools, for example a `jitviewer`_ that help us analyze performance. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -142,3 +142,50 @@ it is the standard ``OrderedDict.move_to_end()`` method, but the behavior is also available on Python 2.x or for the ``dict`` type by calling ``__pypy__.move_to_end(dict, key, last=True)``. + + +.. branch optinfo-into-bridges-3 + +Improve the optimization of branchy Python code by retaining more information +across failing guards. + + +.. branch: space-newtext + +Internal refactoring of ``space.wrap()``, which is now replaced with +explicitly-typed methods. Notably, there are now ``space.newbytes()`` +and ``space.newtext()``: these two methods are identical on PyPy 2.7 but +not on PyPy 3.x. The latter is used to get an app-level unicode string +by decoding the RPython string, assumed to be utf-8. + +.. branch: space-wrap + +.. branch: fix_bool_restype + +Fix for ``ctypes.c_bool``-returning ctypes functions + +.. branch: fix-cpyext-releasebuffer + +Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews +keeping objects alive forever (missing decref), and make sure that +bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. + +.. branch: fix-global + +Fix bug (bad reported info) when asked to translate SyntaxWarning to +SyntaxError. + +.. branch: optinfo-into-bridges-3 + +Improve the optimization of branchy Python code by retaining more +information across failing guards. This is done by appending some +carefully encoded extra information into the resume code. + +.. branch: shadowstack-perf-2 + +Two changes that together bring the performance of shadowstack close to +asmgcc---close enough that we can now make shadowstack the default even +on Linux. This should remove a whole class of rare bugs introduced by +asmgcc. + +.. branch: fniephaus/fix-typo-1488123166752 diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -4,16 +4,24 @@ import os import py +TAR_OPTIONS = '-x -v --strip-components=2' +TAR = 'tar {options} -f {tarfile} {files}' + +def untar(tarfile, files): + cmd = TAR.format(options=TAR_OPTIONS, tarfile=tarfile, files=files) + os.system(cmd) + if sys.platform.startswith('linux'): arch = 'linux' cmd = 'wget "%s"' - tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'" + TAR_OPTIONS += ' --wildcards' + binfiles = "'*/bin/pypy' '*/bin/libpypy-c.so'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): arch = 'osx' cmd = 'curl -O "%s"' - tar = "tar -x -v --strip-components=2 -f %s '*/bin/pypy'" + binfiles = "'*/bin/pypy'" else: print 'Cannot determine the platform, please update this script' sys.exit(1) @@ -34,6 +42,7 @@ filename = 'pypy-c-%s-latest-%s.tar.bz2' % (kind, arch) url = 'http://buildbot.pypy.org/nightly/%s/%s' % (branch, filename) tmp = py.path.local.mkdtemp() +pypy_latest = tmp.join(filename) mydir = tmp.chdir() print 'Downloading pypy to', tmp if os.system(cmd % url) != 0: @@ -41,4 +50,10 @@ print 'Extracting pypy binary' mydir.chdir() -os.system(tar % tmp.join(filename)) +untar(pypy_latest, binfiles) +include_dir = py.path.local('../../include') +if include_dir.check(dir=True): + include_dir.chdir() + untar(pypy_latest, '*/include/*') +else: + print 'WARNING: could not find the include/ dir' diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -31,9 +31,9 @@ def create_entry_point(space, w_dict): if w_dict is not None: # for tests - w_entry_point = space.getitem(w_dict, space.wrap('entry_point')) - w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel')) - w_initstdio = space.getitem(w_dict, space.wrap('initstdio')) + w_entry_point = space.getitem(w_dict, space.newtext('entry_point')) + w_run_toplevel = space.getitem(w_dict, space.newtext('run_toplevel')) + w_initstdio = space.getitem(w_dict, space.newtext('initstdio')) withjit = space.config.objspace.usemodules.pypyjit hashfunc = space.config.objspace.hash else: @@ -65,8 +65,8 @@ try: try: space.startup() - w_executable = space.wrap(argv[0]) - w_argv = space.newlist([space.wrap(s) for s in argv[1:]]) + w_executable = space.newtext(argv[0]) + w_argv = space.newlist([space.newtext(s) for s in argv[1:]]) w_exitcode = space.call_function(w_entry_point, w_executable, w_argv) exitcode = space.int_w(w_exitcode) # try to pull it all in @@ -76,7 +76,7 @@ except OperationError as e: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) - debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) + debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space)))) return 1 finally: try: @@ -84,7 +84,7 @@ except OperationError as e: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) - debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) + debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space)))) return 1 return exitcode @@ -123,7 +123,7 @@ try: # initialize sys.{path,executable,stdin,stdout,stderr} # (in unbuffered mode, to avoid troubles) and import site - space.appexec([w_path, space.wrap(home), w_initstdio], + space.appexec([w_path, space.newtext(home), w_initstdio], r"""(path, home, initstdio): import sys sys.path[:] = path @@ -141,7 +141,7 @@ if verbose: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) - debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) + debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space)))) return rffi.cast(rffi.INT, -1) finally: if must_leave: @@ -180,11 +180,11 @@ def _pypy_execute_source(source, c_argument): try: w_globals = space.newdict(module=True) - space.setitem(w_globals, space.wrap('__builtins__'), + space.setitem(w_globals, space.newtext('__builtins__'), space.builtin_modules['__builtin__']) - space.setitem(w_globals, space.wrap('c_argument'), - space.wrap(c_argument)) - space.appexec([space.wrap(source), w_globals], """(src, glob): + space.setitem(w_globals, space.newtext('c_argument'), + space.newint(c_argument)) + space.appexec([space.newtext(source), w_globals], """(src, glob): import sys stmt = compile(src, 'c callback', 'exec') if not hasattr(sys, '_pypy_execute_source'): @@ -195,7 +195,7 @@ except OperationError as e: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) - debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) + debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space)))) return -1 return 0 @@ -323,7 +323,7 @@ # obscure hack to stuff the translation options into the translated PyPy import pypy.module.sys options = make_dict(config) - wrapstr = 'space.wrap(%r)' % (options) + wrapstr = 'space.wrap(%r)' % (options) # import time pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr if config.objspace.usemodules._cffi_backend: self.hack_for_cffi_modules(driver) diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -363,7 +363,7 @@ i = 0 for w_key in keys_w: try: - key = space.str_w(w_key) + key = space.text_w(w_key) except OperationError as e: if e.match(space, space.w_TypeError): raise oefmt(space.w_TypeError, "keywords must be strings") @@ -547,11 +547,11 @@ except IndexError: name = '?' else: - w_enc = space.wrap(space.sys.defaultencoding) - w_err = space.wrap("replace") + w_enc = space.newtext(space.sys.defaultencoding) + w_err = space.newtext("replace") w_name = space.call_method(w_name, "encode", w_enc, w_err) - name = space.str_w(w_name) + name = space.text_w(w_name) break self.kwd_name = name diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -266,8 +266,8 @@ else: w_key = space.newtuple([obj, space.w_float]) elif space.is_w(w_type, space.w_complex): - w_real = space.getattr(obj, space.wrap("real")) - w_imag = space.getattr(obj, space.wrap("imag")) + w_real = space.getattr(obj, space.newtext("real")) + w_imag = space.getattr(obj, space.newtext("imag")) real = space.float_w(w_real) imag = space.float_w(w_imag) real_negzero = (real == 0.0 and @@ -366,7 +366,7 @@ space = self.space consts_w = [space.w_None] * space.len_w(w_consts) w_iter = space.iter(w_consts) - first = space.wrap(0) + first = space.newint(0) while True: try: w_key = space.next(w_iter) diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -13,7 +13,7 @@ "field %s is required for %T", name, w_obj) def check_string(space, w_obj): - if not (space.isinstance_w(w_obj, space.w_str) or + if not (space.isinstance_w(w_obj, space.w_bytes) or space.isinstance_w(w_obj, space.w_unicode)): raise oefmt(space.w_TypeError, "AST string must be of type str or unicode") @@ -48,10 +48,10 @@ "Hack around the fact we can't store tuples on a TypeDef." def __init__(self, fields): - self.fields = fields - - def __spacebind__(self, space): - return space.newtuple([space.wrap(field) for field in self.fields]) + assert fields == [] + + def spacebind(self, space): + return space.newtuple([]) class W_AST(W_Root): @@ -67,14 +67,14 @@ if w_dict is None: w_dict = space.newdict() w_type = space.type(self) - w_fields = space.getattr(w_type, space.wrap("_fields")) + w_fields = space.getattr(w_type, space.newtext("_fields")) for w_name in space.fixedview(w_fields): try: space.setitem(w_dict, w_name, space.getattr(self, w_name)) except OperationError: pass - w_attrs = space.findattr(w_type, space.wrap("_attributes")) + w_attrs = space.findattr(w_type, space.newtext("_attributes")) if w_attrs: for w_name in space.fixedview(w_attrs): try: @@ -93,12 +93,12 @@ def W_AST_new(space, w_type, __args__): node = space.allocate_instance(W_AST, w_type) - return space.wrap(node) + return node def W_AST_init(space, w_self, __args__): args_w, kwargs_w = __args__.unpack() fields_w = space.fixedview(space.getattr(space.type(w_self), - space.wrap("_fields"))) + space.newtext("_fields"))) num_fields = len(fields_w) if fields_w else 0 if args_w and len(args_w) != num_fields: if num_fields == 0: @@ -114,7 +114,7 @@ for i, w_field in enumerate(fields_w): space.setattr(w_self, w_field, args_w[i]) for field, w_value in kwargs_w.iteritems(): - space.setattr(w_self, space.wrap(field), w_value) + space.setattr(w_self, space.newtext(field), w_value) W_AST.typedef = typedef.TypeDef("_ast.AST", @@ -143,16 +143,16 @@ def make_new_type(self, space, name, base, fields, attributes): w_base = getattr(self, 'w_%s' % base) w_dict = space.newdict() - space.setitem_str(w_dict, '__module__', space.wrap('_ast')) + space.setitem_str(w_dict, '__module__', space.newtext('_ast')) if fields is not None: space.setitem_str(w_dict, "_fields", - space.newtuple([space.wrap(f) for f in fields])) + space.newtuple([space.newtext(f) for f in fields])) if attributes is not None: space.setitem_str(w_dict, "_attributes", - space.newtuple([space.wrap(a) for a in attributes])) + space.newtuple([space.newtext(a) for a in attributes])) w_type = space.call_function( space.w_type, - space.wrap(name), space.newtuple([w_base]), w_dict) + space.newtext(name), space.newtuple([w_base]), w_dict) setattr(self, 'w_%s' % name, w_type) def get(space): @@ -196,7 +196,7 @@ else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) return w_node @staticmethod @@ -230,7 +230,7 @@ else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) return w_node @staticmethod @@ -258,7 +258,7 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Expression) w_body = self.body.to_object(space) # expr - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) return w_node @staticmethod @@ -293,7 +293,7 @@ else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) return w_node @staticmethod @@ -390,26 +390,26 @@ def to_object(self, space): w_node = space.call_function(get(space).w_FunctionDef) - w_name = space.wrap(self.name) # identifier - space.setattr(w_node, space.wrap('name'), w_name) + w_name = space.newtext(self.name) # identifier + space.setattr(w_node, space.newtext('name'), w_name) w_args = self.args.to_object(space) # arguments - space.setattr(w_node, space.wrap('args'), w_args) + space.setattr(w_node, space.newtext('args'), w_args) if self.body is None: body_w = [] else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) if self.decorator_list is None: decorator_list_w = [] else: decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr w_decorator_list = space.newlist(decorator_list_w) - space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -420,7 +420,7 @@ w_decorator_list = get_field(space, w_node, 'decorator_list', False) w_lineno = get_field(space, w_node, 'lineno', False) w_col_offset = get_field(space, w_node, 'col_offset', False) - _name = space.realstr_w(w_name) + _name = space.realtext_w(w_name) if _name is None: raise_required_value(space, w_node, 'name') _args = arguments.from_object(space, w_args) @@ -463,30 +463,30 @@ def to_object(self, space): w_node = space.call_function(get(space).w_ClassDef) - w_name = space.wrap(self.name) # identifier - space.setattr(w_node, space.wrap('name'), w_name) + w_name = space.newtext(self.name) # identifier + space.setattr(w_node, space.newtext('name'), w_name) if self.bases is None: bases_w = [] else: bases_w = [node.to_object(space) for node in self.bases] # expr w_bases = space.newlist(bases_w) - space.setattr(w_node, space.wrap('bases'), w_bases) + space.setattr(w_node, space.newtext('bases'), w_bases) if self.body is None: body_w = [] else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) if self.decorator_list is None: decorator_list_w = [] else: decorator_list_w = [node.to_object(space) for node in self.decorator_list] # expr w_decorator_list = space.newlist(decorator_list_w) - space.setattr(w_node, space.wrap('decorator_list'), w_decorator_list) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('decorator_list'), w_decorator_list) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -497,7 +497,7 @@ w_decorator_list = get_field(space, w_node, 'decorator_list', False) w_lineno = get_field(space, w_node, 'lineno', False) w_col_offset = get_field(space, w_node, 'col_offset', False) - _name = space.realstr_w(w_name) + _name = space.realtext_w(w_name) if _name is None: raise_required_value(space, w_node, 'name') bases_w = space.unpackiterable(w_bases) @@ -530,11 +530,11 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Return) w_value = self.value.to_object(space) if self.value is not None else space.w_None # expr - space.setattr(w_node, space.wrap('value'), w_value) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('value'), w_value) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -572,11 +572,11 @@ else: targets_w = [node.to_object(space) for node in self.targets] # expr w_targets = space.newlist(targets_w) - space.setattr(w_node, space.wrap('targets'), w_targets) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('targets'), w_targets) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -617,13 +617,13 @@ else: targets_w = [node.to_object(space) for node in self.targets] # expr w_targets = space.newlist(targets_w) - space.setattr(w_node, space.wrap('targets'), w_targets) + space.setattr(w_node, space.newtext('targets'), w_targets) w_value = self.value.to_object(space) # expr - space.setattr(w_node, space.wrap('value'), w_value) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('value'), w_value) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -663,15 +663,15 @@ def to_object(self, space): w_node = space.call_function(get(space).w_AugAssign) w_target = self.target.to_object(space) # expr - space.setattr(w_node, space.wrap('target'), w_target) + space.setattr(w_node, space.newtext('target'), w_target) w_op = operator_to_class[self.op - 1]().to_object(space) # operator - space.setattr(w_node, space.wrap('op'), w_op) + space.setattr(w_node, space.newtext('op'), w_op) w_value = self.value.to_object(space) # expr - space.setattr(w_node, space.wrap('value'), w_value) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('value'), w_value) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -719,19 +719,19 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Print) w_dest = self.dest.to_object(space) if self.dest is not None else space.w_None # expr - space.setattr(w_node, space.wrap('dest'), w_dest) + space.setattr(w_node, space.newtext('dest'), w_dest) if self.values is None: values_w = [] else: values_w = [node.to_object(space) for node in self.values] # expr w_values = space.newlist(values_w) - space.setattr(w_node, space.wrap('values'), w_values) - w_nl = space.wrap(self.nl) # bool - space.setattr(w_node, space.wrap('nl'), w_nl) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('values'), w_values) + w_nl = space.newbool(self.nl) # bool + space.setattr(w_node, space.newtext('nl'), w_nl) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -780,25 +780,25 @@ def to_object(self, space): w_node = space.call_function(get(space).w_For) w_target = self.target.to_object(space) # expr - space.setattr(w_node, space.wrap('target'), w_target) + space.setattr(w_node, space.newtext('target'), w_target) w_iter = self.iter.to_object(space) # expr - space.setattr(w_node, space.wrap('iter'), w_iter) + space.setattr(w_node, space.newtext('iter'), w_iter) if self.body is None: body_w = [] else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) if self.orelse is None: orelse_w = [] else: orelse_w = [node.to_object(space) for node in self.orelse] # stmt w_orelse = space.newlist(orelse_w) - space.setattr(w_node, space.wrap('orelse'), w_orelse) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('orelse'), w_orelse) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -850,23 +850,23 @@ def to_object(self, space): w_node = space.call_function(get(space).w_While) w_test = self.test.to_object(space) # expr - space.setattr(w_node, space.wrap('test'), w_test) + space.setattr(w_node, space.newtext('test'), w_test) if self.body is None: body_w = [] else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) if self.orelse is None: orelse_w = [] else: orelse_w = [node.to_object(space) for node in self.orelse] # stmt w_orelse = space.newlist(orelse_w) - space.setattr(w_node, space.wrap('orelse'), w_orelse) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('orelse'), w_orelse) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -914,23 +914,23 @@ def to_object(self, space): w_node = space.call_function(get(space).w_If) w_test = self.test.to_object(space) # expr - space.setattr(w_node, space.wrap('test'), w_test) + space.setattr(w_node, space.newtext('test'), w_test) if self.body is None: body_w = [] else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) if self.orelse is None: orelse_w = [] else: orelse_w = [node.to_object(space) for node in self.orelse] # stmt w_orelse = space.newlist(orelse_w) - space.setattr(w_node, space.wrap('orelse'), w_orelse) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('orelse'), w_orelse) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -977,19 +977,19 @@ def to_object(self, space): w_node = space.call_function(get(space).w_With) w_context_expr = self.context_expr.to_object(space) # expr - space.setattr(w_node, space.wrap('context_expr'), w_context_expr) + space.setattr(w_node, space.newtext('context_expr'), w_context_expr) w_optional_vars = self.optional_vars.to_object(space) if self.optional_vars is not None else space.w_None # expr - space.setattr(w_node, space.wrap('optional_vars'), w_optional_vars) + space.setattr(w_node, space.newtext('optional_vars'), w_optional_vars) if self.body is None: body_w = [] else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('body'), w_body) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1035,15 +1035,15 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Raise) w_type = self.type.to_object(space) if self.type is not None else space.w_None # expr - space.setattr(w_node, space.wrap('type'), w_type) + space.setattr(w_node, space.newtext('type'), w_type) w_inst = self.inst.to_object(space) if self.inst is not None else space.w_None # expr - space.setattr(w_node, space.wrap('inst'), w_inst) + space.setattr(w_node, space.newtext('inst'), w_inst) w_tback = self.tback.to_object(space) if self.tback is not None else space.w_None # expr - space.setattr(w_node, space.wrap('tback'), w_tback) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('tback'), w_tback) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1093,23 +1093,23 @@ else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) if self.handlers is None: handlers_w = [] else: handlers_w = [node.to_object(space) for node in self.handlers] # excepthandler w_handlers = space.newlist(handlers_w) - space.setattr(w_node, space.wrap('handlers'), w_handlers) + space.setattr(w_node, space.newtext('handlers'), w_handlers) if self.orelse is None: orelse_w = [] else: orelse_w = [node.to_object(space) for node in self.orelse] # stmt w_orelse = space.newlist(orelse_w) - space.setattr(w_node, space.wrap('orelse'), w_orelse) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('orelse'), w_orelse) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1158,17 +1158,17 @@ else: body_w = [node.to_object(space) for node in self.body] # stmt w_body = space.newlist(body_w) - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) if self.finalbody is None: finalbody_w = [] else: finalbody_w = [node.to_object(space) for node in self.finalbody] # stmt w_finalbody = space.newlist(finalbody_w) - space.setattr(w_node, space.wrap('finalbody'), w_finalbody) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('finalbody'), w_finalbody) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1207,13 +1207,13 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Assert) w_test = self.test.to_object(space) # expr - space.setattr(w_node, space.wrap('test'), w_test) + space.setattr(w_node, space.newtext('test'), w_test) w_msg = self.msg.to_object(space) if self.msg is not None else space.w_None # expr - space.setattr(w_node, space.wrap('msg'), w_msg) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('msg'), w_msg) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1255,11 +1255,11 @@ else: names_w = [node.to_object(space) for node in self.names] # alias w_names = space.newlist(names_w) - space.setattr(w_node, space.wrap('names'), w_names) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('names'), w_names) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1295,20 +1295,20 @@ def to_object(self, space): w_node = space.call_function(get(space).w_ImportFrom) - w_module = space.wrap(self.module) # identifier - space.setattr(w_node, space.wrap('module'), w_module) + w_module = space.newtext_or_none(self.module) # identifier + space.setattr(w_node, space.newtext('module'), w_module) if self.names is None: names_w = [] else: names_w = [node.to_object(space) for node in self.names] # alias w_names = space.newlist(names_w) - space.setattr(w_node, space.wrap('names'), w_names) - w_level = space.wrap(self.level) # int - space.setattr(w_node, space.wrap('level'), w_level) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('names'), w_names) + w_level = space.newint(self.level) # int + space.setattr(w_node, space.newtext('level'), w_level) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1318,7 +1318,7 @@ w_level = get_field(space, w_node, 'level', True) w_lineno = get_field(space, w_node, 'lineno', False) w_col_offset = get_field(space, w_node, 'col_offset', False) - _module = space.str_or_None_w(w_module) + _module = space.realtext_w(w_module) if not space.is_none(w_module) else None names_w = space.unpackiterable(w_names) _names = [alias.from_object(space, w_item) for w_item in names_w] _level = space.int_w(w_level) @@ -1351,15 +1351,15 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Exec) w_body = self.body.to_object(space) # expr - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) w_globals = self.globals.to_object(space) if self.globals is not None else space.w_None # expr - space.setattr(w_node, space.wrap('globals'), w_globals) + space.setattr(w_node, space.newtext('globals'), w_globals) w_locals = self.locals.to_object(space) if self.locals is not None else space.w_None # expr - space.setattr(w_node, space.wrap('locals'), w_locals) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('locals'), w_locals) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1398,13 +1398,13 @@ if self.names is None: names_w = [] else: - names_w = [space.wrap(node) for node in self.names] # identifier + names_w = [space.newtext(node) for node in self.names] # identifier w_names = space.newlist(names_w) - space.setattr(w_node, space.wrap('names'), w_names) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('names'), w_names) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1413,7 +1413,7 @@ w_lineno = get_field(space, w_node, 'lineno', False) w_col_offset = get_field(space, w_node, 'col_offset', False) names_w = space.unpackiterable(w_names) - _names = [space.realstr_w(w_item) for w_item in names_w] + _names = [space.realtext_w(w_item) for w_item in names_w] _lineno = space.int_w(w_lineno) _col_offset = space.int_w(w_col_offset) return Global(_names, _lineno, _col_offset) @@ -1437,11 +1437,11 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Expr) w_value = self.value.to_object(space) # expr - space.setattr(w_node, space.wrap('value'), w_value) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('value'), w_value) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1472,10 +1472,10 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Pass) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1502,10 +1502,10 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Break) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1532,10 +1532,10 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Continue) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1628,17 +1628,17 @@ def to_object(self, space): w_node = space.call_function(get(space).w_BoolOp) w_op = boolop_to_class[self.op - 1]().to_object(space) # boolop - space.setattr(w_node, space.wrap('op'), w_op) + space.setattr(w_node, space.newtext('op'), w_op) if self.values is None: values_w = [] else: values_w = [node.to_object(space) for node in self.values] # expr w_values = space.newlist(values_w) - space.setattr(w_node, space.wrap('values'), w_values) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('values'), w_values) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1678,15 +1678,15 @@ def to_object(self, space): w_node = space.call_function(get(space).w_BinOp) w_left = self.left.to_object(space) # expr - space.setattr(w_node, space.wrap('left'), w_left) + space.setattr(w_node, space.newtext('left'), w_left) w_op = operator_to_class[self.op - 1]().to_object(space) # operator - space.setattr(w_node, space.wrap('op'), w_op) + space.setattr(w_node, space.newtext('op'), w_op) w_right = self.right.to_object(space) # expr - space.setattr(w_node, space.wrap('right'), w_right) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('right'), w_right) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1729,13 +1729,13 @@ def to_object(self, space): w_node = space.call_function(get(space).w_UnaryOp) w_op = unaryop_to_class[self.op - 1]().to_object(space) # unaryop - space.setattr(w_node, space.wrap('op'), w_op) + space.setattr(w_node, space.newtext('op'), w_op) w_operand = self.operand.to_object(space) # expr - space.setattr(w_node, space.wrap('operand'), w_operand) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('operand'), w_operand) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1775,13 +1775,13 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Lambda) w_args = self.args.to_object(space) # arguments - space.setattr(w_node, space.wrap('args'), w_args) + space.setattr(w_node, space.newtext('args'), w_args) w_body = self.body.to_object(space) # expr - space.setattr(w_node, space.wrap('body'), w_body) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('body'), w_body) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1823,15 +1823,15 @@ def to_object(self, space): w_node = space.call_function(get(space).w_IfExp) w_test = self.test.to_object(space) # expr - space.setattr(w_node, space.wrap('test'), w_test) + space.setattr(w_node, space.newtext('test'), w_test) w_body = self.body.to_object(space) # expr - space.setattr(w_node, space.wrap('body'), w_body) + space.setattr(w_node, space.newtext('body'), w_body) w_orelse = self.orelse.to_object(space) # expr - space.setattr(w_node, space.wrap('orelse'), w_orelse) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('orelse'), w_orelse) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1883,17 +1883,17 @@ else: keys_w = [node.to_object(space) for node in self.keys] # expr w_keys = space.newlist(keys_w) - space.setattr(w_node, space.wrap('keys'), w_keys) + space.setattr(w_node, space.newtext('keys'), w_keys) if self.values is None: values_w = [] else: values_w = [node.to_object(space) for node in self.values] # expr w_values = space.newlist(values_w) - space.setattr(w_node, space.wrap('values'), w_values) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('values'), w_values) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1935,11 +1935,11 @@ else: elts_w = [node.to_object(space) for node in self.elts] # expr w_elts = space.newlist(elts_w) - space.setattr(w_node, space.wrap('elts'), w_elts) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('elts'), w_elts) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -1976,17 +1976,17 @@ def to_object(self, space): w_node = space.call_function(get(space).w_ListComp) w_elt = self.elt.to_object(space) # expr - space.setattr(w_node, space.wrap('elt'), w_elt) + space.setattr(w_node, space.newtext('elt'), w_elt) if self.generators is None: generators_w = [] else: generators_w = [node.to_object(space) for node in self.generators] # comprehension w_generators = space.newlist(generators_w) - space.setattr(w_node, space.wrap('generators'), w_generators) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('generators'), w_generators) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -2027,17 +2027,17 @@ def to_object(self, space): w_node = space.call_function(get(space).w_SetComp) w_elt = self.elt.to_object(space) # expr - space.setattr(w_node, space.wrap('elt'), w_elt) + space.setattr(w_node, space.newtext('elt'), w_elt) if self.generators is None: generators_w = [] else: generators_w = [node.to_object(space) for node in self.generators] # comprehension w_generators = space.newlist(generators_w) - space.setattr(w_node, space.wrap('generators'), w_generators) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('generators'), w_generators) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -2080,19 +2080,19 @@ def to_object(self, space): w_node = space.call_function(get(space).w_DictComp) w_key = self.key.to_object(space) # expr - space.setattr(w_node, space.wrap('key'), w_key) + space.setattr(w_node, space.newtext('key'), w_key) w_value = self.value.to_object(space) # expr - space.setattr(w_node, space.wrap('value'), w_value) + space.setattr(w_node, space.newtext('value'), w_value) if self.generators is None: generators_w = [] else: generators_w = [node.to_object(space) for node in self.generators] # comprehension w_generators = space.newlist(generators_w) - space.setattr(w_node, space.wrap('generators'), w_generators) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('generators'), w_generators) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -2137,17 +2137,17 @@ def to_object(self, space): w_node = space.call_function(get(space).w_GeneratorExp) w_elt = self.elt.to_object(space) # expr - space.setattr(w_node, space.wrap('elt'), w_elt) + space.setattr(w_node, space.newtext('elt'), w_elt) if self.generators is None: generators_w = [] else: generators_w = [node.to_object(space) for node in self.generators] # comprehension w_generators = space.newlist(generators_w) - space.setattr(w_node, space.wrap('generators'), w_generators) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('generators'), w_generators) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -2185,11 +2185,11 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Yield) w_value = self.value.to_object(space) if self.value is not None else space.w_None # expr - space.setattr(w_node, space.wrap('value'), w_value) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('value'), w_value) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -2226,23 +2226,23 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Compare) w_left = self.left.to_object(space) # expr - space.setattr(w_node, space.wrap('left'), w_left) + space.setattr(w_node, space.newtext('left'), w_left) if self.ops is None: ops_w = [] else: ops_w = [cmpop_to_class[node - 1]().to_object(space) for node in self.ops] # cmpop w_ops = space.newlist(ops_w) - space.setattr(w_node, space.wrap('ops'), w_ops) + space.setattr(w_node, space.newtext('ops'), w_ops) if self.comparators is None: comparators_w = [] else: comparators_w = [node.to_object(space) for node in self.comparators] # expr w_comparators = space.newlist(comparators_w) - space.setattr(w_node, space.wrap('comparators'), w_comparators) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('comparators'), w_comparators) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -2296,27 +2296,27 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Call) w_func = self.func.to_object(space) # expr - space.setattr(w_node, space.wrap('func'), w_func) + space.setattr(w_node, space.newtext('func'), w_func) if self.args is None: args_w = [] else: args_w = [node.to_object(space) for node in self.args] # expr w_args = space.newlist(args_w) - space.setattr(w_node, space.wrap('args'), w_args) + space.setattr(w_node, space.newtext('args'), w_args) if self.keywords is None: keywords_w = [] else: keywords_w = [node.to_object(space) for node in self.keywords] # keyword w_keywords = space.newlist(keywords_w) - space.setattr(w_node, space.wrap('keywords'), w_keywords) + space.setattr(w_node, space.newtext('keywords'), w_keywords) w_starargs = self.starargs.to_object(space) if self.starargs is not None else space.w_None # expr - space.setattr(w_node, space.wrap('starargs'), w_starargs) + space.setattr(w_node, space.newtext('starargs'), w_starargs) w_kwargs = self.kwargs.to_object(space) if self.kwargs is not None else space.w_None # expr - space.setattr(w_node, space.wrap('kwargs'), w_kwargs) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('kwargs'), w_kwargs) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -2360,11 +2360,11 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Repr) w_value = self.value.to_object(space) # expr - space.setattr(w_node, space.wrap('value'), w_value) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('value'), w_value) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) return w_node @staticmethod @@ -2397,11 +2397,11 @@ def to_object(self, space): w_node = space.call_function(get(space).w_Num) w_n = self.n # object - space.setattr(w_node, space.wrap('n'), w_n) - w_lineno = space.wrap(self.lineno) # int - space.setattr(w_node, space.wrap('lineno'), w_lineno) - w_col_offset = space.wrap(self.col_offset) # int - space.setattr(w_node, space.wrap('col_offset'), w_col_offset) + space.setattr(w_node, space.newtext('n'), w_n) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) From pypy.commits at gmail.com Wed Mar 15 08:08:47 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 15 Mar 2017 05:08:47 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: copy over the changes from vmprof-python/f7df918fbdd Message-ID: <58c92ecf.574a190a.0ab8.1ce7@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90699:aa5c858b1622 Date: 2017-03-15 12:56 +0100 http://bitbucket.org/pypy/pypy/changeset/aa5c858b1622/ Log: copy over the changes from vmprof-python/f7df918fbdd diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c --- a/rpython/rlib/rvmprof/src/shared/_vmprof.c +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c @@ -248,14 +248,15 @@ static PyObject * -sample_stack_now(PyObject *module, PyObject *args) +sample_stack_now(PyObject *module, PyObject * args) { PyThreadState * tstate = NULL; - PyObject * list; + PyObject * list = NULL; int i; int entry_count; void ** m; void * routine_ip; + long skip = 0; // stop any signal to occur vmprof_ignore_signals(1); @@ -265,6 +266,10 @@ goto error; } + if (!PyArg_ParseTuple(args, "l", &skip)) { + goto error; + } + tstate = PyGILState_GetThisThreadState(); m = (void**)malloc(SINGLE_BUF_SIZE); if (m == NULL) { @@ -272,7 +277,7 @@ vmprof_ignore_signals(0); return NULL; } - entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, 0, 0); + entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, skip, 0); for (i = 0; i < entry_count; i++) { routine_ip = m[i]; @@ -337,7 +342,7 @@ {"disable", disable_vmprof, METH_NOARGS, "Disable profiling."}, {"write_all_code_objects", write_all_code_objects, METH_NOARGS, "Write eagerly all the IDs of code objects"}, - {"sample_stack_now", sample_stack_now, METH_NOARGS, "Sample the stack now"}, + {"sample_stack_now", sample_stack_now, METH_VARARGS, "Sample the stack now"}, #ifdef VMP_SUPPORTS_NATIVE_PROFILING {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"}, #endif diff --git a/rpython/rlib/rvmprof/src/shared/compat.h b/rpython/rlib/rvmprof/src/shared/compat.h --- a/rpython/rlib/rvmprof/src/shared/compat.h +++ b/rpython/rlib/rvmprof/src/shared/compat.h @@ -13,6 +13,7 @@ #define PyStr_GET_SIZE PyString_GET_SIZE #define PyStr_NEW PyString_FromString #define PyLong_NEW PyInt_FromSsize_t + #define PyLong_AsLong PyInt_AsLong # endif #endif diff --git a/rpython/rlib/rvmprof/src/shared/machine.c b/rpython/rlib/rvmprof/src/shared/machine.c --- a/rpython/rlib/rvmprof/src/shared/machine.c +++ b/rpython/rlib/rvmprof/src/shared/machine.c @@ -27,14 +27,3 @@ #endif } -#ifdef VMP_SUPPORTS_NATIVE_PROFILING -#include "libudis86/udis86.h" -unsigned int vmp_machine_code_instr_length(char* pc) -{ - struct ud u; - ud_init(&u); - ud_set_input_buffer(&u, (uint8_t*)pc, 12); - ud_set_mode(&u, vmp_machine_bits()); - return ud_decode(&u); -} -#endif diff --git a/rpython/rlib/rvmprof/src/shared/machine.h b/rpython/rlib/rvmprof/src/shared/machine.h --- a/rpython/rlib/rvmprof/src/shared/machine.h +++ b/rpython/rlib/rvmprof/src/shared/machine.h @@ -10,11 +10,3 @@ */ const char * vmp_machine_os_name(void); -/** - * How many bytes does the x86 instruction take at pc[0..]. - * - * Returns 0 on failure. - */ -#ifdef VMP_SUPPORTS_NATIVE_PROFILING -unsigned int vmp_machine_code_instr_length(char* pc); -#endif diff --git a/rpython/rlib/rvmprof/src/shared/trampoline.c b/rpython/rlib/rvmprof/src/shared/trampoline.c --- a/rpython/rlib/rvmprof/src/shared/trampoline.c +++ b/rpython/rlib/rvmprof/src/shared/trampoline.c @@ -121,6 +121,16 @@ } #endif +#include "libudis86/udis86.h" +unsigned int vmp_machine_code_instr_length(char* pc) +{ + struct ud u; + ud_init(&u); + ud_set_input_buffer(&u, (uint8_t*)pc, 12); + ud_set_mode(&u, vmp_machine_bits()); + return ud_decode(&u); +} + // a hilarious typo, tramp -> trump :) int _redirect_trampoline_and_back(char * eval, char * trump, char * vmprof_eval) { diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -15,24 +15,18 @@ #include "compat.h" #ifdef VMP_SUPPORTS_NATIVE_PROFILING -#define UNW_LOCAL_ONLY -#include "unwind/libunwind.h" -# ifdef X86_64 -# define REG_RBX UNW_X86_64_RBX -# elif defined(X86_32) -# define REG_RBX UNW_X86_EDI -# endif +#include "unwind/vmprof_unwind.h" + +typedef mcontext_t unw_context_t; // functions copied from libunwind using dlopen -#ifdef VMPROF_UNIX static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL; static int (*unw_step)(unw_cursor_t*) = NULL; static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL; static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; static int (*unw_is_signal_frame)(unw_cursor_t *) = NULL; -static int (*unw_getcontext)(unw_cursor_t *) = NULL; -#endif +static int (*unw_getcontext)(unw_context_t *) = NULL; #endif @@ -176,30 +170,39 @@ return 0; } + if (signal < 0) { + while (signal < 0) { + int err = unw_step(&cursor); + if (err <= 0) { + return 0; + } + signal++; + } + } else { #ifdef VMPROF_LINUX - while (signal) { - int is_signal_frame = unw_is_signal_frame(&cursor); - if (is_signal_frame) { - unw_step(&cursor); // step once more discard signal frame - break; + while (signal) { + int is_signal_frame = unw_is_signal_frame(&cursor); + if (is_signal_frame) { + unw_step(&cursor); // step once more discard signal frame + break; + } + int err = unw_step(&cursor); + if (err <= 0) { + return 0; + } } - int err = unw_step(&cursor); - if (err <= 0) { - return 0; +#else + // who would have guessed that unw_is_signal_frame does not work on mac os x + if (signal) { + unw_step(&cursor); // vmp_walk_and_record_stack + // get_stack_trace is inlined + unw_step(&cursor); // _vmprof_sample_stack + unw_step(&cursor); // sigprof_handler + unw_step(&cursor); // _sigtramp } +#endif } -#else - // who would have guessed that unw_is_signal_frame does not work on mac os x - if (signal) { - unw_step(&cursor); // vmp_walk_and_record_stack - // get_stack_trace is inlined - unw_step(&cursor); // _vmprof_sample_stack - unw_step(&cursor); // sigprof_handler - unw_step(&cursor); // _sigtramp - } -#endif - //printf("stack trace:\n"); int depth = 0; PY_STACK_FRAME_T * top_most_frame = frame; while (depth < max_depth) { @@ -223,84 +226,39 @@ // printf("func_addr is 0, now %p\n", rip); //} +#ifdef PYPY + unw_word_t rip = 0; + if (unw_get_reg(&cursor, UNW_REG_IP, &rip) < 0) { + return 0; + } +#endif if (IS_VMPROF_EVAL((void*)pip.start_ip)) { // yes we found one stack entry of the python frames! -#ifndef RPYTHON_VMPROF - unw_word_t rbx = 0; - if (unw_get_reg(&cursor, REG_RBX, &rbx) < 0) { - break; - } - if (rbx != (unw_word_t)top_most_frame) { - // uh we are screwed! the ip indicates we are have context - // to a PyEval_EvalFrameEx function, but when we tried to retrieve - // the stack located py frame it has a different address than the - // current top_most_frame - return 0; - } else { -#else - { + return vmp_walk_and_record_python_stack_only(frame, result, max_depth, depth, pc); +#ifdef PYPY + } else if (IS_JIT_FRAME((void*)rip)) { + depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth); + return vmp_walk_and_record_python_stack_only(frame, result, max_depth, depth, pc); #endif - if (top_most_frame != NULL) { - top_most_frame = _write_python_stack_entry(top_most_frame, result, &depth, max_depth); - } else { - // Signals can occur at the two places (1) and (2), that will - // have added a stack entry, but the function __vmprof_eval_vmprof - // is not entered. This behaviour will swallow one Python stack frames - // - // (1) PyPy: enter_code happened, but __vmprof_eval_vmprof was not called - // (2) PyPy: __vmprof_eval_vmprof was returned, but exit_code was not called - // - // destroy this sample, as it would display a strage sample - return 0; - } - } - } else if (vmp_ignore_ip((intptr_t)func_addr)) { - // this is an instruction pointer that should be ignored - // (that is any function name in the mapping range of - // cpython or libpypy-c.so, but of course not - // extenstions in site-packages) } else { // mark native routines with the first bit set, // this is possible because compiler align to 8 bytes. // -#ifdef PYPY_JIT_CODEMAP - // see vmp_dynamic.c on line ip->flags = DYN_JIT_FLAG - if (pip.flags == DYN_JIT_FLAG) { - if (top_most_frame->kind != VMPROF_JITTED_TAG) { - // if this is encountered frequently something is wrong with - // the stack building - return 0; - } - intptr_t pc = ((intptr_t*)(top_most_frame->value - sizeof(intptr_t)))[0]; - depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth); - top_most_frame = FRAME_STEP(top_most_frame); - } else if (func_addr != 0x0) { - depth = _write_native_stack((void*)(func_addr | 0x1), result, depth); - } -#else if (func_addr != 0x0) { depth = _write_native_stack((void*)(func_addr | 0x1), result, depth); } -#endif } int err = unw_step(&cursor); if (err == 0) { - //printf("sample ended\n"); break; } else if (err < 0) { return 0; // this sample is broken, cannot walk it fully } } - if (top_most_frame == NULL) { - return depth; - } - // Whenever the trampoline is inserted, there might be a view python - // stack levels that do not have the trampoline! - // they should not be consumed, because the let native symbols flow forward. - return depth; //vmp_walk_and_record_python_stack_only(top_most_frame, result, max_depth, depth); + return 0; // kill this sample, no python level was found #else return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc); #endif @@ -492,30 +450,47 @@ static const char * vmprof_error = NULL; + +#ifdef VMPROF_LINUX +#define LIBUNWIND "libunwind.so" +#ifdef __i386__ +#define PREFIX "x86" +#elif __x86_64__ +#define PREFIX "x86_64" +#endif +#define U_PREFIX "_U" +#define UL_PREFIX "_UL" +#else +#define LIBUNWIND "/usr/lib/system/libunwind.dylib" +#define PREFIX "unw" +#define U_PREFIX "" +#define UL_PREFIX "" +#endif + int vmp_native_enable(void) { void * libhandle; vmp_native_traces_enabled = 1; if (!unw_get_reg) { - if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) { + if (!(libhandle = dlopen(LIBUNWIND, RTLD_LAZY | RTLD_LOCAL))) { goto bail_out; } - if (!(unw_getcontext = dlsym(libhandle, "_ULx86_64_getcontext"))) { + if (!(unw_get_reg = dlsym(libhandle, UL_PREFIX PREFIX "_get_reg"))) { goto bail_out; } - if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) { + if (!(unw_get_proc_info = dlsym(libhandle, UL_PREFIX PREFIX "_get_proc_info"))){ goto bail_out; } - if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))){ + if (!(unw_init_local = dlsym(libhandle, UL_PREFIX PREFIX "_init_local"))) { goto bail_out; } - if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) { + if (!(unw_step = dlsym(libhandle, UL_PREFIX PREFIX "_step"))) { goto bail_out; } - if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) { + if (!(unw_is_signal_frame = dlsym(libhandle, UL_PREFIX PREFIX "_is_signal_frame"))) { goto bail_out; } - if (!(unw_is_signal_frame = dlsym(libhandle, "_ULx86_64_is_signal_frame"))) { + if (!(unw_getcontext = dlsym(libhandle, U_PREFIX PREFIX "_getcontext"))) { goto bail_out; } if (dlclose(libhandle)) { @@ -529,7 +504,9 @@ return vmp_read_vmaps(NULL); #endif bail_out: + vmprof_error = dlerror(); fprintf(stderr, "could not load libunwind at runtime. error: %s\n", vmprof_error); + vmp_native_traces_enabled = 0; return 0; } diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h --- a/rpython/rlib/rvmprof/src/shared/vmprof.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof.h @@ -61,9 +61,13 @@ #define PY_THREAD_STATE_T PyThreadState #define FRAME_STEP(f) f->f_back #define FRAME_CODE(f) f->f_code -PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag); -#define VMPROF_EVAL() vmprof_eval -#define IS_VMPROF_EVAL(PTR) PTR == (void*)vmprof_eval + +#if CPYTHON_HAS_FRAME_EVALUATION +#define IS_VMPROF_EVAL(PTR) PTR == (void*)_PyEval_EvalFrameDefault +#else +#define IS_VMPROF_EVAL(PTR) (PTR == (void*)PyEval_EvalFrameEx || PTR == (void*)PyEval_EvalFrame) #endif +#endif + diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h @@ -337,39 +337,12 @@ vmp_native_disable(); return; } -#if CPYTHON_HAS_FRAME_EVALUATION - PyThreadState *tstate = PyThreadState_GET(); - tstate->interp->eval_frame = vmprof_eval; - _default_eval_loop = _PyEval_EvalFrameDefault; -#elif defined(RPYTHON_VMPROF) - // do nothing here, the stack is maintained by rpython - // no need for a trampoline -#else - if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx, - vmprof_eval, (void*)&_default_eval_loop) == 0) { - } else { - fprintf(stderr, "FATAL: could not insert trampline, try with --no-native\n"); - // TODO dump the first few bytes and tell them to create an issue! - exit(-1); - } -#endif vmp_native_enable(); } static void disable_cpyprof(void) { vmp_native_disable(); -#if CPYTHON_HAS_FRAME_EVALUATION - PyThreadState *tstate = PyThreadState_GET(); - tstate->interp->eval_frame = _PyEval_EvalFrameDefault; -#elif defined(RPYTHON_VMPROF) - // TODO nothing? -#else - if (vmp_unpatch_callee_trampoline(PyEval_EvalFrameEx) > 0) { - fprintf(stderr, "FATAL: could not remove trampoline\n"); - exit(-1); - } -#endif dump_native_symbols(vmp_profile_fileno()); } #endif From pypy.commits at gmail.com Wed Mar 15 08:08:42 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 15 Mar 2017 05:08:42 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: copy over changes made to vmprof-python Message-ID: <58c92eca.49a9190a.b8798.1e19@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90697:ac30c079910e Date: 2017-03-14 15:00 +0100 http://bitbucket.org/pypy/pypy/changeset/ac30c079910e/ Log: copy over changes made to vmprof-python diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h --- a/rpython/rlib/rvmprof/src/vmprof_main.h +++ b/rpython/rlib/rvmprof/src/vmprof_main.h @@ -1,3 +1,5 @@ +#pragma once + /* VMPROF * * statistical sampling profiler specifically designed to profile programs @@ -10,45 +12,49 @@ * * Tested only on gcc, linux, x86_64. * - * Copyright (C) 2014-2015 + * Copyright (C) 2014-2017 * Antonio Cuni - anto.cuni at gmail.com * Maciej Fijalkowski - fijall at gmail.com * Armin Rigo - arigo at tunes.org + * Richard Plangger - planrichi at gmail.com * */ #define _GNU_SOURCE 1 #include +#include +#include #include -#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "vmprof_stack.h" + +#include "vmprof.h" + +#include "vmp_stack.h" #include "vmprof_getpc.h" #include "vmprof_mt.h" -#include "vmprof_get_custom_offset.h" #include "vmprof_common.h" +#include "compat.h" + +#if defined(__unix__) +#include "rss_unix.h" +#elif defined(__APPLE__) +#include "rss_darwin.h" +#endif + /************************************************************/ -static long prepare_interval_usec; -static long saved_profile_file; -static struct profbuf_s *volatile current_codes; static void *(*mainloop_get_virtual_ip)(char *) = 0; - -static int opened_profile(char *interp_name); +static int opened_profile(const char *interp_name, int memory, int proflines, int native); static void flush_codes(void); - /************************************************************/ /* value: last bit is 1 if signals must be ignored; all other bits @@ -79,24 +85,26 @@ static char atfork_hook_installed = 0; -static intptr_t get_current_thread_id(void) +/* ************************************************************* + * functions to dump the stack trace + * ************************************************************* + */ + +int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc) { - /* xxx This function is a hack on two fronts: - - - It assumes that pthread_self() is async-signal-safe. This - should be true on Linux. I hope it is also true elsewhere. - - - It abuses pthread_self() by assuming it just returns an - integer. According to comments in CPython's source code, the - platforms where it is not the case are rare nowadays. - - An alternative would be to try to look if the information is - available in the ucontext_t in the caller. - */ - return (intptr_t)pthread_self(); + PY_STACK_FRAME_T * frame; +#ifdef RPYTHON_VMPROF + // do nothing here, + frame = (PY_STACK_FRAME_T*)current; +#else + if (!current) { + return 0; + } + frame = current->frame; +#endif + return vmp_walk_and_record_stack(frame, result, max_depth, 1, pc); } - /* ************************************************************* * the signal handler * ************************************************************* @@ -112,9 +120,67 @@ longjmp(restore_point, SIGSEGV); } +int _vmprof_sample_stack(struct profbuf_s *p, PY_THREAD_STATE_T * tstate, ucontext_t * uc) +{ + int depth; + struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data; + st->marker = MARKER_STACKTRACE; + st->count = 1; +#ifdef RPYTHON_VMPROF + depth = get_stack_trace(get_vmprof_stack(), st->stack, MAX_STACK_DEPTH-1, (intptr_t)GetPC(uc)); +#else + depth = get_stack_trace(tstate, st->stack, MAX_STACK_DEPTH-1, (intptr_t)NULL); +#endif + if (depth == 0) { + return 0; + } + st->depth = depth; + st->stack[depth++] = tstate; + long rss = get_current_proc_rss(); + if (rss >= 0) + st->stack[depth++] = (void*)rss; + p->data_offset = offsetof(struct prof_stacktrace_s, marker); + p->data_size = (depth * sizeof(void *) + + sizeof(struct prof_stacktrace_s) - + offsetof(struct prof_stacktrace_s, marker)); + return 1; +} + +#ifndef RPYTHON_VMPROF +static PY_THREAD_STATE_T * _get_pystate_for_this_thread(void) { + // see issue 116 on github.com/vmprof/vmprof-python. + // PyGILState_GetThisThreadState(); can hang forever + // + PyInterpreterState * istate; + PyThreadState * state; + long mythread_id; + + istate = PyInterpreterState_Head(); + if (istate == NULL) { + return NULL; + } + mythread_id = PyThread_get_thread_ident(); + // fish fish fish, it will NOT lock the keymutex in pythread + do { + state = PyInterpreterState_ThreadHead(istate); + do { + if (state->thread_id == mythread_id) { + return state; + } + } while ((state = PyThreadState_Next(state)) != NULL); + } while ((istate = PyInterpreterState_Next(istate)) != NULL); + + // uh? not found? + return NULL; +} +#endif + static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { -#ifdef __APPLE__ + int commit; + PY_THREAD_STATE_T * tstate = NULL; + void (*prevhandler)(int); +#ifndef RPYTHON_VMPROF // TERRIBLE HACK AHEAD // on OS X, the thread local storage is sometimes uninitialized // when the signal handler runs - it means it's impossible to read errno @@ -122,48 +188,46 @@ // it seems impossible to read the register gs. // here we register segfault handler (all guarded by a spinlock) and call // longjmp in case segfault happens while reading a thread local + // + // We do the same error detection for linux to ensure that + // get_current_thread_state returns a sane result while (__sync_lock_test_and_set(&spinlock, 1)) { } - signal(SIGSEGV, &segfault_handler); + prevhandler = signal(SIGSEGV, &segfault_handler); int fault_code = setjmp(restore_point); if (fault_code == 0) { pthread_self(); - get_current_thread_id(); + tstate = _get_pystate_for_this_thread(); } else { - signal(SIGSEGV, SIG_DFL); - __sync_synchronize(); - spinlock = 0; - return; + signal(SIGSEGV, prevhandler); + __sync_lock_release(&spinlock); + return; } - signal(SIGSEGV, SIG_DFL); - __sync_synchronize(); - spinlock = 0; + signal(SIGSEGV, prevhandler); + __sync_lock_release(&spinlock); #endif + long val = __sync_fetch_and_add(&signal_handler_value, 2L); if ((val & 1) == 0) { int saved_errno = errno; - int fd = profile_file; + int fd = vmp_profile_fileno(); assert(fd >= 0); struct profbuf_s *p = reserve_buffer(fd); if (p == NULL) { /* ignore this signal: there are no free buffers right now */ - } - else { - int depth; - struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data; - st->marker = MARKER_STACKTRACE; - st->count = 1; - depth = get_stack_trace(get_vmprof_stack(), st->stack, - MAX_STACK_DEPTH-2, GetPC((ucontext_t*)ucontext)); - st->depth = depth; - st->stack[depth++] = get_current_thread_id(); - p->data_offset = offsetof(struct prof_stacktrace_s, marker); - p->data_size = (depth * sizeof(void *) + - sizeof(struct prof_stacktrace_s) - - offsetof(struct prof_stacktrace_s, marker)); - commit_buffer(fd, p); + } else { +#ifdef RPYTHON_VMPORF + commit = _vmprof_sample_stack(p, NULL, (ucontext_t*)ucontext); +#else + commit = _vmprof_sample_stack(p, tstate, (ucontext_t*)ucontext); +#endif + if (commit) { + commit_buffer(fd, p); + } else { + cancel_buffer(p); + } } errno = saved_errno; @@ -173,6 +237,7 @@ } + /* ************************************************************* * the setup and teardown functions * ************************************************************* @@ -197,58 +262,53 @@ return 0; } -static int itimer_which = ITIMER_PROF; - static int install_sigprof_timer(void) { - struct itimerval timer; + static struct itimerval timer; timer.it_interval.tv_sec = 0; timer.it_interval.tv_usec = profile_interval_usec; timer.it_value = timer.it_interval; - if (setitimer(itimer_which, &timer, NULL) == 0) - return 0; /* normal path */ - - if (errno == EINVAL) { - /* on WSL, only ITIMER_REAL is supported */ - if (setitimer(ITIMER_REAL, &timer, NULL) == 0) { - fprintf(stderr, "warning: setitimer(): ITIMER_PROF not " - "available, using ITIMER_REAL instead. " - "Multithreaded programs and programs " - "doing a lot of I/O won't give correct " - "results.\n"); - itimer_which = ITIMER_REAL; - return 0; - } - } - return -1; + if (setitimer(ITIMER_PROF, &timer, NULL) != 0) + return -1; + return 0; } static int remove_sigprof_timer(void) { - struct itimerval timer; + static struct itimerval timer; timer.it_interval.tv_sec = 0; timer.it_interval.tv_usec = 0; timer.it_value.tv_sec = 0; timer.it_value.tv_usec = 0; - if (setitimer(itimer_which, &timer, NULL) != 0) + if (setitimer(ITIMER_PROF, &timer, NULL) != 0) return -1; return 0; } static void atfork_disable_timer(void) { if (profile_interval_usec > 0) { - saved_profile_file = profile_file; - profile_file = -1; remove_sigprof_timer(); +#ifndef RPYTHON_VMPROF + is_enabled = 0; +#endif } } static void atfork_enable_timer(void) { if (profile_interval_usec > 0) { - profile_file = saved_profile_file; install_sigprof_timer(); +#ifndef RPYTHON_VMPROF + is_enabled = 1; +#endif } } +static void atfork_close_profile_file(void) { + int fd = vmp_profile_fileno(); + if (fd != -1) + close(fd); + vmp_set_profile_fileno(-1); +} + static int install_pthread_atfork_hooks(void) { /* this is needed to prevent the problems described there: - http://code.google.com/p/gperftools/issues/detail?id=278 @@ -262,20 +322,69 @@ */ if (atfork_hook_installed) return 0; - int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL); + int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, atfork_close_profile_file); if (ret != 0) return -1; atfork_hook_installed = 1; return 0; } +#ifdef VMP_SUPPORTS_NATIVE_PROFILING +void init_cpyprof(int native) +{ + // skip this if native should not be enabled + if (!native) { + vmp_native_disable(); + return; + } +#if CPYTHON_HAS_FRAME_EVALUATION + PyThreadState *tstate = PyThreadState_GET(); + tstate->interp->eval_frame = vmprof_eval; + _default_eval_loop = _PyEval_EvalFrameDefault; +#elif defined(RPYTHON_VMPROF) + // do nothing here, the stack is maintained by rpython + // no need for a trampoline +#else + if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx, + vmprof_eval, (void*)&_default_eval_loop) == 0) { + } else { + fprintf(stderr, "FATAL: could not insert trampline, try with --no-native\n"); + // TODO dump the first few bytes and tell them to create an issue! + exit(-1); + } +#endif + vmp_native_enable(); +} + +static void disable_cpyprof(void) +{ + vmp_native_disable(); +#if CPYTHON_HAS_FRAME_EVALUATION + PyThreadState *tstate = PyThreadState_GET(); + tstate->interp->eval_frame = _PyEval_EvalFrameDefault; +#elif defined(RPYTHON_VMPROF) + // TODO nothing? +#else + if (vmp_unpatch_callee_trampoline(PyEval_EvalFrameEx) > 0) { + fprintf(stderr, "FATAL: could not remove trampoline\n"); + exit(-1); + } +#endif + dump_native_symbols(vmp_profile_fileno()); +} +#endif + RPY_EXTERN -int vmprof_enable(void) +int vmprof_enable(int memory, int native) { - assert(profile_file >= 0); +#ifdef VMP_SUPPORTS_NATIVE_PROFILING + init_cpyprof(native); +#endif + assert(vmp_profile_fileno() >= 0); assert(prepare_interval_usec > 0); profile_interval_usec = prepare_interval_usec; - + if (memory && setup_rss() == -1) + goto error; if (install_pthread_atfork_hooks() == -1) goto error; if (install_sigprof_handler() == -1) @@ -286,32 +395,19 @@ return 0; error: - profile_file = -1; + vmp_set_profile_fileno(-1); profile_interval_usec = 0; return -1; } -static int _write_all(const char *buf, size_t bufsize) + +int close_profile(void) { - while (bufsize > 0) { - ssize_t count = write(profile_file, buf, bufsize); - if (count <= 0) - return -1; /* failed */ - buf += count; - bufsize -= count; - } - return 0; -} + (void)vmp_write_time_now(MARKER_TRAILER); -static int close_profile(void) -{ - char marker = MARKER_TRAILER; - - if (_write_all(&marker, 1) < 0) - return -1; - + teardown_rss(); /* don't close() the file descriptor from here */ - profile_file = -1; + vmp_set_profile_fileno(-1); return 0; } @@ -320,29 +416,29 @@ { vmprof_ignore_signals(1); profile_interval_usec = 0; +#ifdef VMP_SUPPORTS_NATIVE_PROFILING + disable_cpyprof(); +#endif if (remove_sigprof_timer() == -1) return -1; if (remove_sigprof_handler() == -1) return -1; flush_codes(); - if (shutdown_concurrent_bufs(profile_file) < 0) + if (shutdown_concurrent_bufs(vmp_profile_fileno()) < 0) return -1; return close_profile(); } RPY_EXTERN -int vmprof_register_virtual_function(char *code_name, long code_uid, +int vmprof_register_virtual_function(char *code_name, intptr_t code_uid, int auto_retry) { long namelen = strnlen(code_name, 1023); - long blocklen = 1 + 2 * sizeof(long) + namelen; + long blocklen = 1 + sizeof(intptr_t) + sizeof(long) + namelen; struct profbuf_s *p; char *t; - if (profile_file == -1) - return 0; // silently don't write it - retry: p = current_codes; if (p != NULL) { @@ -352,7 +448,7 @@ size_t freesize = SINGLE_BUF_SIZE - p->data_size; if (freesize < (size_t)blocklen) { /* full: flush it */ - commit_buffer(profile_file, p); + commit_buffer(vmp_profile_fileno(), p); p = NULL; } } @@ -363,7 +459,7 @@ } if (p == NULL) { - p = reserve_buffer(profile_file); + p = reserve_buffer(vmp_profile_fileno()); if (p == NULL) { /* can't get a free block; should almost never be the case. Spin loop if allowed, or return a failure code @@ -381,14 +477,14 @@ p->data_size += blocklen; assert(p->data_size <= SINGLE_BUF_SIZE); *t++ = MARKER_VIRTUAL_IP; - memcpy(t, &code_uid, sizeof(long)); t += sizeof(long); + memcpy(t, &code_uid, sizeof(intptr_t)); t += sizeof(intptr_t); memcpy(t, &namelen, sizeof(long)); t += sizeof(long); memcpy(t, code_name, namelen); /* try to reattach 'p' to 'current_codes' */ if (!__sync_bool_compare_and_swap(¤t_codes, NULL, p)) { /* failed, flush it */ - commit_buffer(profile_file, p); + commit_buffer(vmp_profile_fileno(), p); } return 0; } @@ -398,6 +494,6 @@ struct profbuf_s *p = current_codes; if (p != NULL) { current_codes = NULL; - commit_buffer(profile_file, p); + commit_buffer(vmp_profile_fileno(), p); } } From pypy.commits at gmail.com Wed Mar 15 08:08:45 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 15 Mar 2017 05:08:45 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: copy dynamic loading for libunwind functions from pypy's history Message-ID: <58c92ecd.8f57190a.66016.1d87@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90698:c6cdfd64054c Date: 2017-03-14 15:12 +0100 http://bitbucket.org/pypy/pypy/changeset/c6cdfd64054c/ Log: copy dynamic loading for libunwind functions from pypy's history diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c --- a/rpython/rlib/rvmprof/src/shared/_vmprof.c +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c @@ -276,7 +276,7 @@ for (i = 0; i < entry_count; i++) { routine_ip = m[i]; - PyList_Append(list, PyLong_NEW((long)routine_ip)); + PyList_Append(list, PyLong_NEW((ssize_t)routine_ip)); } free(m); diff --git a/rpython/rlib/rvmprof/src/shared/compat.h b/rpython/rlib/rvmprof/src/shared/compat.h --- a/rpython/rlib/rvmprof/src/shared/compat.h +++ b/rpython/rlib/rvmprof/src/shared/compat.h @@ -7,12 +7,12 @@ #define PyStr_AS_STRING PyBytes_AS_STRING #define PyStr_GET_SIZE PyBytes_GET_SIZE #define PyStr_NEW PyUnicode_FromString - #define PyLong_NEW PyLong_FromLong + #define PyLong_NEW PyLong_FromSsize_t # else #define PyStr_AS_STRING PyString_AS_STRING #define PyStr_GET_SIZE PyString_GET_SIZE #define PyStr_NEW PyString_FromString - #define PyLong_NEW PyInt_FromLong + #define PyLong_NEW PyInt_FromSsize_t # endif #endif diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -16,12 +16,24 @@ #ifdef VMP_SUPPORTS_NATIVE_PROFILING #define UNW_LOCAL_ONLY -#include +#include "unwind/libunwind.h" # ifdef X86_64 # define REG_RBX UNW_X86_64_RBX # elif defined(X86_32) # define REG_RBX UNW_X86_EDI # endif + +// functions copied from libunwind using dlopen + +#ifdef VMPROF_UNIX +static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL; +static int (*unw_step)(unw_cursor_t*) = NULL; +static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL; +static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; +static int (*unw_is_signal_frame)(unw_cursor_t *) = NULL; +static int (*unw_getcontext)(unw_cursor_t *) = NULL; +#endif + #endif #ifdef __APPLE__ @@ -59,7 +71,7 @@ int len; int addr; int j; - long line; + uint64_t line; char *lnotab; #ifndef RPYTHON_VMPROF // pypy does not support line profiling @@ -76,7 +88,7 @@ lnotab = PyStr_AS_STRING(frame->f_code->co_lnotab); if (lnotab != NULL) { - line = (long)frame->f_lineno; + line = (uint64_t)frame->f_lineno; addr = 0; len = (int)PyStr_GET_SIZE(frame->f_code->co_lnotab); @@ -202,7 +214,6 @@ // printf(" %s %p\n", name, func_addr); //} - //if (func_addr == 0) { // unw_word_t rip = 0; // if (unw_get_reg(&cursor, UNW_REG_IP, &rip) < 0) { @@ -479,14 +490,47 @@ } #endif +static const char * vmprof_error = NULL; + int vmp_native_enable(void) { + void * libhandle; vmp_native_traces_enabled = 1; + if (!unw_get_reg) { + if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) { + goto bail_out; + } + if (!(unw_getcontext = dlsym(libhandle, "_ULx86_64_getcontext"))) { + goto bail_out; + } + if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) { + goto bail_out; + } + if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))){ + goto bail_out; + } + if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) { + goto bail_out; + } + if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) { + goto bail_out; + } + if (!(unw_is_signal_frame = dlsym(libhandle, "_ULx86_64_is_signal_frame"))) { + goto bail_out; + } + if (dlclose(libhandle)) { + goto bail_out; + } + } + #if defined(__unix__) return vmp_read_vmaps("/proc/self/maps"); #elif defined(__APPLE__) return vmp_read_vmaps(NULL); #endif +bail_out: + fprintf(stderr, "could not load libunwind at runtime. error: %s\n", vmprof_error); + return 0; } void vmp_native_disable(void) { diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h --- a/rpython/rlib/rvmprof/src/shared/vmprof.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof.h @@ -1,6 +1,8 @@ #pragma once +#ifdef VMPROF_UNIX #include +#endif // common defines #define MARKER_STACKTRACE '\x01' @@ -52,6 +54,7 @@ // for cpython #include "_vmprof.h" #include +#include #include #define PY_STACK_FRAME_T PyFrameObject #define PY_EVAL_RETURN_T PyObject diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h @@ -48,7 +48,13 @@ * is 4, but fails on win32 */ typedef struct prof_stacktrace_s { +#ifdef VMPROF_WINDOWS + // if padding is 8 bytes, then on both 32bit and 64bit, the + // stack field is aligned + char padding[sizeof(void*) - 1]; +#else char padding[sizeof(long) - 1]; +#endif char marker; long count, depth; void *stack[]; @@ -96,16 +102,15 @@ } header; const char * machine; + size_t namelen = strnlen(interp_name, 255); machine = vmp_machine_os_name(); - size_t namelen = strnlen(interp_name, 255); - header.hdr[0] = 0; header.hdr[1] = 3; header.hdr[2] = 0; header.hdr[3] = prepare_interval_usec; - if (strstr(machine, "win") != 0) { + if (strstr(machine, "win64") != 0) { header.hdr[4] = 1; } else { header.hdr[4] = 0; diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h @@ -12,10 +12,11 @@ * * Tested only on gcc, linux, x86_64. * - * Copyright (C) 2014-2015 + * Copyright (C) 2014-2017 * Antonio Cuni - anto.cuni at gmail.com * Maciej Fijalkowski - fijall at gmail.com * Armin Rigo - arigo at tunes.org + * Richard Plangger - planrichi at gmail.com * */ @@ -145,6 +146,35 @@ return 1; } +#ifndef RPYTHON_VMPROF +static PY_THREAD_STATE_T * _get_pystate_for_this_thread(void) { + // see issue 116 on github.com/vmprof/vmprof-python. + // PyGILState_GetThisThreadState(); can hang forever + // + PyInterpreterState * istate; + PyThreadState * state; + long mythread_id; + + istate = PyInterpreterState_Head(); + if (istate == NULL) { + return NULL; + } + mythread_id = PyThread_get_thread_ident(); + // fish fish fish, it will NOT lock the keymutex in pythread + do { + state = PyInterpreterState_ThreadHead(istate); + do { + if (state->thread_id == mythread_id) { + return state; + } + } while ((state = PyThreadState_Next(state)) != NULL); + } while ((istate = PyInterpreterState_Next(istate)) != NULL); + + // uh? not found? + return NULL; +} +#endif + static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { int commit; @@ -167,7 +197,7 @@ int fault_code = setjmp(restore_point); if (fault_code == 0) { pthread_self(); - tstate = PyGILState_GetThisThreadState(); + tstate = _get_pystate_for_this_thread(); } else { signal(SIGSEGV, prevhandler); __sync_lock_release(&spinlock); diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c --- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c +++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c @@ -1,4 +1,4 @@ - +// cannot include this header because it also has definitions #include "windows.h" #include "compat.h" #include "vmp_stack.h" @@ -12,7 +12,6 @@ return 0; } -#include "vmprof_common.h" #include int vmp_write_all(const char *buf, size_t bufsize) diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h @@ -48,9 +48,9 @@ return -1; // possible, e.g. attached debugger or thread alread suspended // find the correct thread depth = vmp_walk_and_record_stack(tstate->frame, stack->stack, - MAX_STACK_DEPTH, 0); + MAX_STACK_DEPTH, 0, 0); stack->depth = depth; - stack->stack[depth++] = (void*)thread_id; + stack->stack[depth++] = (void*)((ULONG_PTR)thread_id); stack->count = 1; stack->marker = MARKER_STACKTRACE; ResumeThread(hThread); @@ -86,16 +86,14 @@ continue; depth = vmprof_snapshot_thread(tstate->thread_id, tstate, stack); if (depth > 0) { - // see note in vmprof_common.h on the prof_stacktrace_s struct why - // there are two vmpr_write_all calls - vmp_write_all((char*)stack + offsetof(prof_stacktrace_s, marker), SIZEOF_PROF_STACKTRACE); - vmp_write_all((char*)stack->stack, depth * sizeof(void*)); + vmp_write_all((char*)stack + offsetof(prof_stacktrace_s, marker), + SIZEOF_PROF_STACKTRACE + depth * sizeof(void*)); } } } RPY_EXTERN -int vmprof_enable(int memory) +int vmprof_enable(int memory, int native) { if (!thread_started) { if (!CreateThread(NULL, 0, vmprof_mainloop, NULL, 0, NULL)) { @@ -121,4 +119,5 @@ RPY_EXTERN void vmprof_ignore_signals(int ignored) { + enabled = !ignored; } From pypy.commits at gmail.com Wed Mar 15 08:08:50 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 15 Mar 2017 05:08:50 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: revert the changes done to rpython/jit/* to support _U_dyn_register and _U_dyn_cancel (does not exist on mac os x) Message-ID: <58c92ed2.01462e0a.b44bb.1e63@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90700:abc1cf9c6e4a Date: 2017-03-15 13:07 +0100 http://bitbucket.org/pypy/pypy/changeset/abc1cf9c6e4a/ Log: revert the changes done to rpython/jit/* to support _U_dyn_register and _U_dyn_cancel (does not exist on mac os x) diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -80,7 +80,7 @@ libssl _vmprof - libunwind + libunwind (optional, loaded dynamically at runtime) Make sure to have these libraries (with development headers) installed before building PyPy, otherwise the resulting binary will not contain diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -491,6 +491,7 @@ if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) + self.setup(looptoken) if self.cpu.HAS_CODEMAP: self.codemap_builder.enter_portal_frame(jd_id, unique_id, diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -162,10 +162,9 @@ jitcell_token.outermost_jitdriver_sd = jitdriver_sd return jitcell_token -def record_loop_or_bridge(metainterp_sd, loop, asminfo): +def record_loop_or_bridge(metainterp_sd, loop): """Do post-backend recordings and cleanups on 'loop'. """ - from rpython.rlib.rvmprof import rvmprof # get the original jitcell token corresponding to jitcell form which # this trace starts original_jitcell_token = loop.original_jitcell_token @@ -175,11 +174,6 @@ wref = weakref.ref(original_jitcell_token) clt = original_jitcell_token.compiled_loop_token clt.loop_token_wref = wref - - rvmprof.dyn_register_jit_page(original_jitcell_token, asminfo.asmaddr, - asminfo.asmaddr+asminfo.asmlen) - - for op in loop.operations: descr = op.getdescr() # not sure what descr.index is about @@ -250,9 +244,9 @@ if not we_are_translated(): loop.check_consistency() jitcell_token.target_tokens = [target_token] - asminfo = send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, - loop, "loop", runtime_args, metainterp.box_names_memo) - record_loop_or_bridge(metainterp_sd, loop, asminfo) + send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop", + runtime_args, metainterp.box_names_memo) + record_loop_or_bridge(metainterp_sd, loop) return target_token def compile_loop(metainterp, greenkey, start, inputargs, jumpargs, @@ -345,9 +339,9 @@ loop_info.extra_before_label + [loop_info.label_op] + loop_ops) if not we_are_translated(): loop.check_consistency() - asminfo = send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, - loop, "loop", inputargs, metainterp.box_names_memo) - record_loop_or_bridge(metainterp_sd, loop, asminfo) + send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop", + inputargs, metainterp.box_names_memo) + record_loop_or_bridge(metainterp_sd, loop) loop_info.post_loop_compilation(loop, jitdriver_sd, metainterp, jitcell_token) return start_descr @@ -422,9 +416,9 @@ loop.quasi_immutable_deps = quasi_immutable_deps target_token = loop.operations[-1].getdescr() - asminfo = resumekey.compile_and_attach(metainterp, loop, inputargs) + resumekey.compile_and_attach(metainterp, loop, inputargs) - record_loop_or_bridge(metainterp_sd, loop, asminfo) + record_loop_or_bridge(metainterp_sd, loop) return target_token def get_box_replacement(op, allow_none=False): @@ -509,12 +503,11 @@ 'compiling', None, name, memo) _log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE_OPT, metainterp_sd, None) _log.write(inputargs, operations) - asminfo = metainterp_sd.cpu.compile_loop(inputargs, + return metainterp_sd.cpu.compile_loop(inputargs, operations, looptoken, jd_id=jd_id, unique_id=unique_id, - log=log, name=name, logger=metainterp_sd.jitlog) - - return asminfo + log=log, name=name, + logger=metainterp_sd.jitlog) def do_compile_bridge(metainterp_sd, faildescr, inputargs, operations, original_loop_token, log=True, memo=None): @@ -524,10 +517,9 @@ _log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE_OPT, metainterp_sd, None) _log.write(inputargs, operations) assert isinstance(faildescr, AbstractFailDescr) - asminfo = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, + return metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, original_loop_token, log=log, logger=metainterp_sd.jitlog) - return asminfo def forget_optimization_info(lst, reset_values=False): for item in lst: @@ -597,8 +589,6 @@ if metainterp_sd.warmrunnerdesc is not None: # for tests metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(original_jitcell_token) - return asminfo - def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs, operations, original_loop_token, memo): forget_optimization_info(operations) @@ -844,10 +834,10 @@ self._debug_subinputargs = new_loop.inputargs self._debug_suboperations = new_loop.operations propagate_original_jitcell_token(new_loop) - return send_bridge_to_backend(metainterp.jitdriver_sd, - metainterp.staticdata, self, inputargs, - new_loop.operations, new_loop.original_jitcell_token, - metainterp.box_names_memo) + send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata, + self, inputargs, new_loop.operations, + new_loop.original_jitcell_token, + metainterp.box_names_memo) def make_a_counter_per_value(self, guard_value_op, index): assert guard_value_op.getopnum() == rop.GUARD_VALUE @@ -1045,14 +1035,13 @@ jitdriver_sd = metainterp.jitdriver_sd new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd) propagate_original_jitcell_token(new_loop) - asminfo = send_loop_to_backend(self.original_greenkey, - metainterp.jitdriver_sd, metainterp_sd, new_loop, - "entry bridge", orig_inputargs, metainterp.box_names_memo) + send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd, + metainterp_sd, new_loop, "entry bridge", + orig_inputargs, metainterp.box_names_memo) # send the new_loop to warmspot.py, to be called directly the next time jitdriver_sd.warmstate.attach_procedure_to_interp( self.original_greenkey, jitcell_token) metainterp_sd.stats.add_jitcell_token(jitcell_token) - return asminfo def compile_trace(metainterp, resumekey, runtime_boxes): @@ -1122,8 +1111,8 @@ if info.final(): new_trace.inputargs = info.inputargs target_token = new_trace.operations[-1].getdescr() - asminfo = resumekey.compile_and_attach(metainterp, new_trace, inputargs) - record_loop_or_bridge(metainterp_sd, new_trace, asminfo) + resumekey.compile_and_attach(metainterp, new_trace, inputargs) + record_loop_or_bridge(metainterp_sd, new_trace) return target_token new_trace.inputargs = info.renamed_inputargs metainterp.retrace_needed(new_trace, info) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -419,14 +419,6 @@ def __init__(self): # For memory management of assembled loops self._keepalive_jitcell_tokens = {} # set of other JitCellToken - self._rvmprof_references = [] - - def rvmprof_register(self, ref): - """ Call this method for every loop or bridge that hangs on this - token. Otherwise the information tracked by libunwind will - not be freed. - """ - self._rvmprof_references.append(ref) def record_jump_to(self, jitcell_token): assert isinstance(jitcell_token, JitCellToken) diff --git a/rpython/jit/metainterp/memmgr.py b/rpython/jit/metainterp/memmgr.py --- a/rpython/jit/metainterp/memmgr.py +++ b/rpython/jit/metainterp/memmgr.py @@ -2,7 +2,6 @@ from rpython.rlib.rarithmetic import r_int64 from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.rvmprof import rvmprof # # Logic to decide which loops are old and not used any more. @@ -71,7 +70,6 @@ for looptoken in self.alive_loops.keys(): if (0 <= looptoken.generation < max_generation or looptoken.invalidated): - rvmprof.dyn_cancel(looptoken) del self.alive_loops[looptoken] newtotal = len(self.alive_loops) debug_print("Loop tokens freed: ", oldtotal - newtotal) diff --git a/rpython/jit/metainterp/optimizeopt/version.py b/rpython/jit/metainterp/optimizeopt/version.py --- a/rpython/jit/metainterp/optimizeopt/version.py +++ b/rpython/jit/metainterp/optimizeopt/version.py @@ -76,7 +76,7 @@ descr, vl.inputargs, vl.operations, jitcell_token, metainterp.box_names_memo) - record_loop_or_bridge(metainterp_sd, vl, asminfo) + record_loop_or_bridge(metainterp_sd, vl) assert asminfo is not None compiled[version] = (asminfo, descr, version, jitcell_token) else: diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1823,12 +1823,6 @@ self.jitlog.logger_noopt = self.logger_noopt self.jitlog.logger_ops = self.logger_ops - from rpython.rlib.rvmprof import rvmprof, cintf - try: - self.vmprof = rvmprof._get_vmprof() - except cintf.VMProfPlatformUnsupported: - self.vmprof = None - self.profiler = ProfilerClass() self.profiler.cpu = cpu self.warmrunnerdesc = warmrunnerdesc diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -15,7 +15,6 @@ ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof') SRC = ROOT.join('src') SHARED = SRC.join('shared') -UDIS86 = SHARED.join('libudis86') BACKTRACE = SHARED.join('libbacktrace') compile_extra = ['-DRPYTHON_VMPROF', '-g', '-O1'] @@ -45,7 +44,7 @@ _libs = [] eci_kwds = dict( - include_dirs = [SRC, SHARED, BACKTRACE, UDIS86], + include_dirs = [SRC, SHARED, BACKTRACE], includes = ['rvmprof.h','vmprof_stack.h'], libraries = _libs, separate_module_files = [ @@ -55,10 +54,6 @@ SHARED.join('symboltable.c'), SHARED.join('vmp_stack.c'), SHARED.join('vmp_dynamic.c'), - # udis86 - SHARED.join('libudis86/decode.c'), - SHARED.join('libudis86/itab.c'), - SHARED.join('libudis86/udis86.c'), ] + separate_module_files, post_include_bits=[], compile_extra=compile_extra From pypy.commits at gmail.com Wed Mar 15 08:26:58 2017 From: pypy.commits at gmail.com (tobweber) Date: Wed, 15 Mar 2017 05:26:58 -0700 (PDT) Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Fix missing duration measurement of inevitable transactions' validation and initialization of duration struct Message-ID: <58c93312.065c2e0a.a1214.20fb@mx.google.com> Author: Tobias Weber Branch: c8-overheads-instrumentation Changeset: r2031:a73dbda25da5 Date: 2017-03-14 21:25 +0100 http://bitbucket.org/pypy/stmgc/changeset/a73dbda25da5/ Log: Fix missing duration measurement of inevitable transactions' validation and initialization of duration struct diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -156,10 +156,10 @@ static bool _stm_validate(void) { + /* returns true if we reached a valid state, or false if + we need to abort now */ start_timer(); - /* returns true if we reached a valid state, or false if - we need to abort now */ dprintf(("_stm_validate() at cl=%p, rev=%lu\n", STM_PSEGMENT->last_commit_log_entry, STM_PSEGMENT->last_commit_log_entry->rev_num)); /* go from last known entry in commit log to the @@ -173,6 +173,8 @@ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { assert(first_cl->next == INEV_RUNNING); + + stop_timer_and_publish(STM_DURATION_VALIDATION); return true; } diff --git a/c8/stm/timing.h b/c8/stm/timing.h --- a/c8/stm/timing.h +++ b/c8/stm/timing.h @@ -5,30 +5,31 @@ /* Use raw monotonic time, i.e., solely based on local hardware (no NTP adjustments) as in prof.c to obtain values comparable with total program runtime. */ -#define start_timer() struct timespec start, stop, duration = { 0, 0 }; \ +#define start_timer() struct timespec start, stop; \ + struct timespec duration = { .tv_sec = 0, .tv_nsec = 0 };\ continue_timer() /* Must use start_timer before using this macro. */ -#define get_duration() duration.tv_sec = \ - stop.tv_sec - start.tv_sec + duration.tv_sec; \ - duration.tv_nsec = \ +#define get_duration() duration.tv_sec = \ + stop.tv_sec - start.tv_sec + duration.tv_sec; \ + duration.tv_nsec = \ stop.tv_nsec - start.tv_nsec + duration.tv_nsec; -#define pause_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &stop); \ +#define pause_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &stop); \ get_duration() -#define stm_duration_payload(duration) \ - stm_timing_event_payload_data_t stm_duration_data = \ - { .duration = &duration }; \ - stm_timing_event_payload_t stm_duration_payload = \ +#define stm_duration_payload(duration) \ + stm_timing_event_payload_data_t stm_duration_data = \ + { .duration = &duration }; \ + stm_timing_event_payload_t stm_duration_payload = \ { STM_EVENT_PAYLOAD_DURATION, stm_duration_data }; -#define publish_event(event) \ - (timing_enabled() ? \ - stmcb_timing_event( \ - STM_SEGMENT->running_thread, event, &stm_duration_payload) : \ +#define publish_event(event) \ + (timing_enabled() ? \ + stmcb_timing_event( \ + STM_SEGMENT->running_thread, event, &stm_duration_payload) : \ (void)0); -#define stop_timer_and_publish(event) pause_timer() \ - stm_duration_payload(duration) \ +#define stop_timer_and_publish(event) pause_timer() \ + stm_duration_payload(duration) \ publish_event(event) From pypy.commits at gmail.com Wed Mar 15 08:27:01 2017 From: pypy.commits at gmail.com (tobweber) Date: Wed, 15 Mar 2017 05:27:01 -0700 (PDT) Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Fix logger accesses to running thread info after it has already been reset Message-ID: <58c93315.010e2e0a.8b228.1fe6@mx.google.com> Author: Tobias Weber Branch: c8-overheads-instrumentation Changeset: r2032:6c8ad9d4223c Date: 2017-03-15 13:25 +0100 http://bitbucket.org/pypy/stmgc/changeset/6c8ad9d4223c/ Log: Fix logger accesses to running thread info after it has already been reset diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -159,6 +159,7 @@ /* returns true if we reached a valid state, or false if we need to abort now */ start_timer(); + stm_thread_local_t *thread_local_for_logging = STM_SEGMENT->running_thread; dprintf(("_stm_validate() at cl=%p, rev=%lu\n", STM_PSEGMENT->last_commit_log_entry, STM_PSEGMENT->last_commit_log_entry->rev_num)); @@ -174,7 +175,10 @@ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { assert(first_cl->next == INEV_RUNNING); - stop_timer_and_publish(STM_DURATION_VALIDATION); + if (thread_local_for_logging != NULL) { + stop_timer_and_publish_for_thread( + thread_local_for_logging, STM_DURATION_VALIDATION); + } return true; } @@ -342,7 +346,10 @@ release_privatization_lock(my_segnum); } - stop_timer_and_publish(STM_DURATION_VALIDATION); + if (thread_local_for_logging != NULL) { + stop_timer_and_publish_for_thread( + thread_local_for_logging, STM_DURATION_VALIDATION); + } return !needs_abort; } @@ -1219,6 +1226,7 @@ static void _core_commit_transaction(bool external) { start_timer(); + stm_thread_local_t *thread_local_for_logging = STM_SEGMENT->running_thread; exec_local_finalizers(); @@ -1316,7 +1324,8 @@ s_mutex_unlock(); - stop_timer_and_publish(STM_DURATION_COMMIT_EXCEPT_GC); + stop_timer_and_publish_for_thread( + thread_local_for_logging, STM_DURATION_COMMIT_EXCEPT_GC); /* between transactions, call finalizers. this will execute a transaction itself */ diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -747,6 +747,8 @@ dprintf((" .----- major collection -----------------------\n")); assert(_has_mutex()); + stm_thread_local_t *thread_local_for_logging = STM_SEGMENT->running_thread; + /* first, force a minor collection in each of the other segments */ major_do_validation_and_minor_collections(); @@ -784,8 +786,9 @@ if (must_abort()) abort_with_mutex(); - stop_timer_and_publish(STM_DURATION_MAJOR_GC_LOG_ONLY); - + stop_timer_and_publish_for_thread( + thread_local_for_logging, STM_DURATION_MAJOR_GC_LOG_ONLY); + return; #endif } @@ -843,5 +846,6 @@ if (must_abort()) abort_with_mutex(); - stop_timer_and_publish(STM_DURATION_MAJOR_GC_FULL); + stop_timer_and_publish_for_thread( + thread_local_for_logging, STM_DURATION_MAJOR_GC_FULL); } diff --git a/c8/stm/timing.h b/c8/stm/timing.h --- a/c8/stm/timing.h +++ b/c8/stm/timing.h @@ -10,10 +10,8 @@ continue_timer() /* Must use start_timer before using this macro. */ -#define get_duration() duration.tv_sec = \ - stop.tv_sec - start.tv_sec + duration.tv_sec; \ - duration.tv_nsec = \ - stop.tv_nsec - start.tv_nsec + duration.tv_nsec; +#define get_duration() duration.tv_sec += stop.tv_sec - start.tv_sec; \ + duration.tv_nsec += stop.tv_nsec - start.tv_nsec; #define pause_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &stop); \ get_duration() @@ -24,12 +22,16 @@ stm_timing_event_payload_t stm_duration_payload = \ { STM_EVENT_PAYLOAD_DURATION, stm_duration_data }; -#define publish_event(event) \ +#define publish_event(thread_local, event) \ (timing_enabled() ? \ - stmcb_timing_event( \ - STM_SEGMENT->running_thread, event, &stm_duration_payload) : \ + stmcb_timing_event(thread_local, event, &stm_duration_payload) : \ (void)0); -#define stop_timer_and_publish(event) pause_timer() \ - stm_duration_payload(duration) \ - publish_event(event) +#define stop_timer_and_publish_for_thread(thread_local, event) \ + pause_timer() \ + stm_duration_payload(duration) \ + assert(thread_local != NULL); \ + publish_event(thread_local, event) + +#define stop_timer_and_publish(event) \ + stop_timer_and_publish_for_thread(STM_SEGMENT->running_thread, event) From pypy.commits at gmail.com Wed Mar 15 09:38:39 2017 From: pypy.commits at gmail.com (rlamy) Date: Wed, 15 Mar 2017 06:38:39 -0700 (PDT) Subject: [pypy-commit] pypy stricter-encode: Try to correctly call error handlers on pypy3 in utf16 and utf32 encoders Message-ID: <58c943df.4920190a.115c.26ed@mx.google.com> Author: Ronan Lamy Branch: stricter-encode Changeset: r90701:af44b848333d Date: 2017-03-14 17:45 +0000 http://bitbucket.org/pypy/pypy/changeset/af44b848333d/ Log: Try to correctly call error handlers on pypy3 in utf16 and utf32 encoders diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -604,14 +604,19 @@ _STORECHAR(result, 0xFEFF, BYTEORDER) byteorder = BYTEORDER - i = 0 - while i < size: - ch = ord(s[i]) - i += 1 + pos = 0 + while pos < size: + ch = ord(s[pos]) + pos += 1 ch2 = 0 if 0xD800 <= ch < 0xDFFF: - errorhandler( - errors, 'utf16', 'surrogates not allowed', s, i - 1, i) + ru, rs, pos = errorhandler( + errors, 'utf16', 'surrogates not allowed', s, pos - 1, pos) + if rs is not None: + result.append(rs) + continue + else: + pass # XXX if ch >= 0x10000: ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) ch = 0xD800 | ((ch-0x10000) >> 10) @@ -772,19 +777,24 @@ _STORECHAR32(result, 0xFEFF, BYTEORDER) byteorder = BYTEORDER - i = 0 - while i < size: - ch = ord(s[i]) - i += 1 + pos = 0 + while pos < size: + ch = ord(s[pos]) + pos += 1 ch2 = 0 if 0xD800 <= ch < 0xDFFF: - errorhandler( - errors, 'utf32', 'surrogates not allowed', s, i - 1, i) - if MAXUNICODE < 65536 and 0xD800 <= ch <= 0xDBFF and i < size: - ch2 = ord(s[i]) + ru, rs, pos = errorhandler( + errors, 'utf32', 'surrogates not allowed', s, pos - 1, pos) + if rs is not None: + result.append(rs) + continue + else: + pass # XXX + if MAXUNICODE < 65536 and 0xD800 <= ch <= 0xDBFF and pos < size: + ch2 = ord(s[pos]) if 0xDC00 <= ch2 <= 0xDFFF: ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000; - i += 1 + pos += 1 _STORECHAR32(result, ch, byteorder) return result.build() From pypy.commits at gmail.com Wed Mar 15 10:01:20 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 07:01:20 -0700 (PDT) Subject: [pypy-commit] pypy default: add missing names Message-ID: <58c94930.84202e0a.5c9f.253f@mx.google.com> Author: Armin Rigo Branch: Changeset: r90702:05b724e21868 Date: 2017-03-15 14:56 +0100 http://bitbucket.org/pypy/pypy/changeset/05b724e21868/ Log: add missing names diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -609,6 +609,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', ] TYPES = {} FORWARD_DECLS = [] From pypy.commits at gmail.com Wed Mar 15 10:01:22 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 07:01:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <58c94932.0be3190a.d4c92.2529@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90703:ed16c1e6473c Date: 2017-03-15 14:59 +0100 http://bitbucket.org/pypy/pypy/changeset/ed16c1e6473c/ Log: hg merge default diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,5 @@ +#encoding utf-8 + License ======= @@ -42,9 +44,9 @@ Antonio Cuni Samuele Pedroni Matti Picus + Ronan Lamy Alex Gaynor Philip Jenvey - Ronan Lamy Brian Kearns Richard Plangger Michael Hudson @@ -55,12 +57,12 @@ Hakan Ardo Benjamin Peterson Anders Chrigstrom + Wim Lavrijsen Eric van Riet Paap - Wim Lavrijsen Richard Emslie Alexander Schremmer + Remi Meier Dan Villiom Podlaski Christiansen - Remi Meier Lukas Diekmann Sven Hager Anders Lehmann @@ -83,8 +85,8 @@ Lawrence Oluyede Bartosz Skowron Daniel Roberts + Adrien Di Mascio Niko Matsakis - Adrien Di Mascio Alexander Hesse Ludovic Aubry Jacob Hallen @@ -100,8 +102,8 @@ Michael Foord Stephan Diehl Stefan Schwarzer + Tomek Meka Valentino Volonghi - Tomek Meka Stefano Rivera Patrick Maupin Devin Jeanpierre @@ -109,268 +111,273 @@ Bruno Gola David Malcolm Jean-Paul Calderone - Timo Paulssen Edd Barrett Squeaky + Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton + Nicolas Truessel Martin Matusiak - Nicolas Truessel + Wenzhu Man Konstantin Lopuhin - Wenzhu Man John Witulski Laurence Tratt + Greg Price Ivan Sichmann Freitas - Greg Price Dario Bertini + Jeremy Thurgood Mark Pearse Simon Cross - Jeremy Thurgood + Tobias Pape Andreas Stührk - Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski + William Leslie + marky1991 + Ilya Osadchiy + Tobias Oberstein Paul deGrandis - Ilya Osadchiy - marky1991 - Tobias Oberstein + Boris Feigin + Taavi Burns Adrian Kuhn - Boris Feigin tav - Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp + Mike Blume Gerald Klix - Mike Blume Oscar Nierstrasz + Rami Chowdhury Stefan H. Muller - Rami Chowdhury + Joannah Nanjekye Eugene Oden + Tim Felgentreff + Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton - Jeff Terrace - Tim Felgentreff Dusty Phillips Lukas Renggli Guenter Jantzen - William Leslie Ned Batchelder + Amit Regmi Anton Gulenko - Amit Regmi - Ben Young + Sergey Matyunin Jasper Schulz + Andrew Chambers Nicolas Chauvat Andrew Durdin - Andrew Chambers - Sergey Matyunin + Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu + Michael Twomey Rocco Moretti Gintautas Miliauskas - Michael Twomey Lucian Branescu Mihaila anatoly techtonik + Karl Bartel Gabriel Lavoie + Jared Grubb Olivier Dormond - Jared Grubb - Karl Bartel Wouter van Heyst + Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina - Sebastian Pawluś - Stuart Williams - Daniel Patrick Aaron Iles Toby Watson + Daniel Patrick + Stuart Williams Antoine Pitrou Christian Hudon + Justas Sadzevicius + Neil Shepperd Michael Cheng - Justas Sadzevicius + Mikael Schönenberg + Stanislaw Halik + Berkin Ilbeyi Gasper Zejn - Neil Shepperd - Stanislaw Halik - Mikael Schönenberg - Berkin Ilbeyi Faye Zhao Elmo Mäntynen - Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani + Jonathan David Riehl Beatrice During Alex Perry + p_zieschang at yahoo.de + Robert Zaremba + Alan McIntyre + Alexander Sedov Vaibhav Sood - Alan McIntyre Reuben Cummings - Alexander Sedov - p_zieschang at yahoo.de Attila Gobi Christopher Pope - Aaron Gallagher + Tristan Arthur + Christian Tismer + Dan Stromberg + Carl Meyer Florin Papa - Christian Tismer - Marc Abramowitz - Dan Stromberg - Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Marc Abramowitz + Arjun Naik + Aaron Gallagher Alexis Daboville - Jens-Uwe Mager - Carl Meyer + Pieter Zieschang Karl Ramm - Pieter Zieschang - Gabriel Lukas Vacek - Kunal Grover - Andrew Dalke + Omer Katz + Jacek Generowicz Sylvain Thenault Jakub Stasiak + Stefan Beyer + Andrew Dalke + Alejandro J. Cura + Vladimir Kryachko + Gabriel + Mark Williams + Kunal Grover Nathan Taylor - Vladimir Kryachko - Omer Katz - Mark Williams - Jacek Generowicz - Alejandro J. Cura + Travis Francis Athougies + Yasir Suhail + Sergey Kishchenko + Martin Blais + Lutz Paelike + Ian Foote + Philipp Rustemeuer + Catalin Gabriel Manciu Jacob Oscarson - Travis Francis Athougies Ryan Gonzalez - Ian Foote Kristjan Valur Jonsson + Lucio Torre + Richard Lancaster + Dan Buch + Lene Wagner + Tomo Cocoa + Alecsandru Patrascu David Lievens Neil Blakey-Milner - Lutz Paelike - Lucio Torre + Henrik Vendelbo Lars Wassermann - Philipp Rustemeuer - Henrik Vendelbo - Richard Lancaster - Yasir Suhail - Dan Buch + Ignas Mikalajunas + Christoph Gerum Miguel de Val Borro Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Alecsandru Patrascu - Christoph Gerum - Martin Blais - Lene Wagner - Catalin Gabriel Manciu - Tomo Cocoa - Kim Jin Su - rafalgalczynski at gmail.com Toni Mattis - Amber Brown + Laurens Van Houtven + Bobby Impollonia + Roberto De Ioris + Jeong YunWon + Christopher Armstrong + Aaron Tubbs + Vasantha Ganesh K + Jason Michalski + Markus Holtermann + Andrew Thompson + Yusei Tahara + Ruochen Huang + Fabio Niephaus + Akira Li + Gustavo Niemeyer + Rafał Gałczyński + Logan Chien Lucas Stadler - Julian Berman - Markus Holtermann roberto at goyle + Matt Bogosian Yury V. Zaytsev - Anna Katrina Dominguez - Bobby Impollonia - Vasantha Ganesh K - Andrew Thompson florinpapa - Yusei Tahara - Aaron Tubbs - Ben Darnell - Roberto De Ioris - Logan Chien - Juan Francisco Cantero Hurtado - Ruochen Huang - Jeong YunWon - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle Anders Sigfridsson Nikolay Zinov - Jason Michalski + rafalgalczynski at gmail.com + Joshua Gilbert + Anna Katrina Dominguez + Kim Jin Su + Amber Brown + Ben Darnell + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Julian Berman + Michael Hudson-Doyle Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer Stephan Busemann - Rafał Gałczyński - Matt Bogosian + Dan Colish timo - Christian Muirhead - Berker Peksag - James Lan Volodymyr Vladymyrov - shoma hosaka - Ben Mather - Niclas Olofsson - Matthew Miller - Rodrigo Araújo + Daniel Neuhäuser + Flavio Percoco halgari - Boglarka Vezer - Chris Pressey - Buck Golemon - Diana Popa - Konrad Delong - Dinu Gherman + Jim Baker Chris Lambacher coolbutuseless at gmail.com + Mike Bayer + Rodrigo Araújo Daniil Yarancev - Jim Baker + OlivierBlanvillain + Jonas Pfannschmidt + Zearin + Andrey Churin Dan Crosta - Nikolaos-Digenis Karagiannis - James Robert - Armin Ronacher - Brett Cannon - Donald Stufft - yrttyr - aliceinwire - OlivierBlanvillain - Dan Sanders - Zooko Wilcox-O Hearn + reubano at gmail.com + Julien Phalip + Roman Podoliaka + Eli Stevens + Boglarka Vezer + PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio - jiaaro - Mads Kiilerich Antony Lee - Jason Madden - Daniel Neuh�user - reubano at gmail.com - Yaroslav Fedevych Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - squeaky - soareschen - Jonas Pfannschmidt - Kurt Griffiths - Mike Bayer - Stefan Marr - Flavio Percoco - Kristoffer Kleine + shoma hosaka + Buck Golemon + JohnDoe + yrttyr Michael Chermside Anna Ravencroft + remarkablerocket + Berker Peksag + Christian Muirhead + soareschen + Matthew Miller + Konrad Delong + Dinu Gherman pizi - remarkablerocket - Andrey Churin - Zearin - Eli Stevens - Tobias Diaz - Julien Phalip - Roman Podoliaka + James Robert + Armin Ronacher + Diana Popa + Mads Kiilerich + Brett Cannon + aliceinwire + Zooko Wilcox-O Hearn + James Lan + jiaaro + Markus Unterwaditzer + Kristoffer Kleine + Graham Markall Dan Loewenherz werat + Niclas Olofsson + Chris Pressey + Tobias Diaz + Nikolaos-Digenis Karagiannis + Kurt Griffiths + Ben Mather + Donald Stufft + Dan Sanders + Jason Madden + Yaroslav Fedevych + Even Wiik Thomassen + Stefan Marr Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -11,9 +11,9 @@ Antonio Cuni Samuele Pedroni Matti Picus + Ronan Lamy Alex Gaynor Philip Jenvey - Ronan Lamy Brian Kearns Richard Plangger Michael Hudson @@ -24,12 +24,12 @@ Hakan Ardo Benjamin Peterson Anders Chrigstrom + Wim Lavrijsen Eric van Riet Paap - Wim Lavrijsen Richard Emslie Alexander Schremmer + Remi Meier Dan Villiom Podlaski Christiansen - Remi Meier Lukas Diekmann Sven Hager Anders Lehmann @@ -52,8 +52,8 @@ Lawrence Oluyede Bartosz Skowron Daniel Roberts + Adrien Di Mascio Niko Matsakis - Adrien Di Mascio Alexander Hesse Ludovic Aubry Jacob Hallen @@ -64,14 +64,13 @@ Michal Bendowski stian Jan de Mooij - Spenser Bauman Tyler Wade Vincent Legoll Michael Foord Stephan Diehl Stefan Schwarzer + Tomek Meka Valentino Volonghi - Tomek Meka Stefano Rivera Patrick Maupin Devin Jeanpierre @@ -79,268 +78,270 @@ Bruno Gola David Malcolm Jean-Paul Calderone - Timo Paulssen Edd Barrett Squeaky + Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton + Nicolas Truessel Martin Matusiak - Nicolas Truessel + Wenzhu Man Konstantin Lopuhin - Wenzhu Man John Witulski Laurence Tratt + Greg Price Ivan Sichmann Freitas - Greg Price Dario Bertini + Jeremy Thurgood Mark Pearse Simon Cross - Jeremy Thurgood + Tobias Pape Andreas Stührk - Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski + William Leslie + marky1991 + Ilya Osadchiy + Tobias Oberstein Paul deGrandis - Ilya Osadchiy - marky1991 - Tobias Oberstein + Boris Feigin + Taavi Burns Adrian Kuhn - Boris Feigin tav - Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp + Mike Blume Gerald Klix - Mike Blume Oscar Nierstrasz + Rami Chowdhury Stefan H. Muller - Rami Chowdhury + Joannah Nanjekye Eugene Oden + Tim Felgentreff + Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton - Jeff Terrace - Tim Felgentreff Dusty Phillips Lukas Renggli Guenter Jantzen - William Leslie Ned Batchelder + Amit Regmi Anton Gulenko - Amit Regmi - Ben Young + Sergey Matyunin Jasper Schulz + Andrew Chambers Nicolas Chauvat Andrew Durdin - Andrew Chambers - Sergey Matyunin + Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu + Michael Twomey Rocco Moretti Gintautas Miliauskas - Michael Twomey Lucian Branescu Mihaila anatoly techtonik + Karl Bartel Gabriel Lavoie + Jared Grubb Olivier Dormond - Jared Grubb - Karl Bartel Wouter van Heyst + Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina - Sebastian Pawluś - Stuart Williams - Daniel Patrick Aaron Iles Toby Watson + Daniel Patrick + Stuart Williams Antoine Pitrou Christian Hudon + Justas Sadzevicius + Neil Shepperd Michael Cheng - Justas Sadzevicius + Mikael Schönenberg + Stanislaw Halik + Berkin Ilbeyi Gasper Zejn - Neil Shepperd - Stanislaw Halik - Mikael Schönenberg - Berkin Ilbeyi Faye Zhao Elmo Mäntynen - Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani + Jonathan David Riehl Beatrice During Alex Perry + p_zieschang at yahoo.de + Robert Zaremba + Alan McIntyre + Alexander Sedov Vaibhav Sood - Alan McIntyre Reuben Cummings - Alexander Sedov - p_zieschang at yahoo.de Attila Gobi Christopher Pope - Aaron Gallagher + Tristan Arthur + Christian Tismer + Dan Stromberg + Carl Meyer Florin Papa - Christian Tismer - Marc Abramowitz - Dan Stromberg - Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Marc Abramowitz + Arjun Naik + Aaron Gallagher Alexis Daboville - Jens-Uwe Mager - Carl Meyer + Pieter Zieschang Karl Ramm - Pieter Zieschang - Gabriel Lukas Vacek - Kunal Grover - Andrew Dalke + Omer Katz + Jacek Generowicz Sylvain Thenault Jakub Stasiak + Stefan Beyer + Andrew Dalke + Alejandro J. Cura + Vladimir Kryachko + Gabriel + Mark Williams + Kunal Grover Nathan Taylor - Vladimir Kryachko - Omer Katz - Mark Williams - Jacek Generowicz - Alejandro J. Cura + Travis Francis Athougies + Yasir Suhail + Sergey Kishchenko + Martin Blais + Lutz Paelike + Ian Foote + Philipp Rustemeuer + Catalin Gabriel Manciu Jacob Oscarson - Travis Francis Athougies Ryan Gonzalez - Ian Foote Kristjan Valur Jonsson + Lucio Torre + Richard Lancaster + Dan Buch + Lene Wagner + Tomo Cocoa + Alecsandru Patrascu David Lievens Neil Blakey-Milner - Lutz Paelike - Lucio Torre + Henrik Vendelbo Lars Wassermann - Philipp Rustemeuer - Henrik Vendelbo - Richard Lancaster - Yasir Suhail - Dan Buch + Ignas Mikalajunas + Christoph Gerum Miguel de Val Borro Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Alecsandru Patrascu - Christoph Gerum - Martin Blais - Lene Wagner - Catalin Gabriel Manciu - Tomo Cocoa - Kim Jin Su - rafalgalczynski at gmail.com Toni Mattis - Amber Brown + Laurens Van Houtven + Bobby Impollonia + Roberto De Ioris + Jeong YunWon + Christopher Armstrong + Aaron Tubbs + Vasantha Ganesh K + Jason Michalski + Markus Holtermann + Andrew Thompson + Yusei Tahara + Ruochen Huang + Fabio Niephaus + Akira Li + Gustavo Niemeyer + Rafał Gałczyński + Logan Chien Lucas Stadler - Julian Berman - Markus Holtermann roberto at goyle + Matt Bogosian Yury V. Zaytsev - Anna Katrina Dominguez - Bobby Impollonia - Vasantha Ganesh K - Andrew Thompson florinpapa - Yusei Tahara - Aaron Tubbs - Ben Darnell - Roberto De Ioris - Logan Chien - Juan Francisco Cantero Hurtado - Ruochen Huang - Jeong YunWon - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle Anders Sigfridsson Nikolay Zinov - Jason Michalski + rafalgalczynski at gmail.com + Joshua Gilbert + Anna Katrina Dominguez + Kim Jin Su + Amber Brown + Ben Darnell + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Julian Berman + Michael Hudson-Doyle Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer Stephan Busemann - Rafał Gałczyński - Matt Bogosian + Dan Colish timo - Christian Muirhead - Berker Peksag - James Lan Volodymyr Vladymyrov - shoma hosaka - Ben Mather - Niclas Olofsson - Matthew Miller - Rodrigo Araújo + Daniel Neuhäuser + Flavio Percoco halgari - Boglarka Vezer - Chris Pressey - Buck Golemon - Diana Popa - Konrad Delong - Dinu Gherman + Jim Baker Chris Lambacher coolbutuseless at gmail.com + Mike Bayer + Rodrigo Araújo Daniil Yarancev - Jim Baker + OlivierBlanvillain + Jonas Pfannschmidt + Zearin + Andrey Churin Dan Crosta - Nikolaos-Digenis Karagiannis - James Robert - Armin Ronacher - Brett Cannon - Donald Stufft - yrttyr - aliceinwire - OlivierBlanvillain - Dan Sanders - Zooko Wilcox-O Hearn + reubano at gmail.com + Julien Phalip + Roman Podoliaka + Eli Stevens + Boglarka Vezer + PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio - jiaaro - Mads Kiilerich + Antony Lee + Jim Hunziker + shoma hosaka + Buck Golemon JohnDoe - Antony Lee - Jason Madden - Daniel Neuhäuser - reubano at gmail.com - Yaroslav Fedevych - Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - squeaky - soareschen - Jonas Pfannschmidt - Kurt Griffiths - Mike Bayer - Stefan Marr - Flavio Percoco - Kristoffer Kleine + yrttyr Michael Chermside Anna Ravencroft + remarkablerocket + Berker Peksag + Christian Muirhead + soareschen + Matthew Miller + Konrad Delong + Dinu Gherman pizi - remarkablerocket - Andrey Churin - Zearin - Eli Stevens - Tobias Diaz - Julien Phalip - Roman Podoliaka + James Robert + Armin Ronacher + Diana Popa + Mads Kiilerich + Brett Cannon + aliceinwire + Zooko Wilcox-O Hearn + James Lan + jiaaro + Markus Unterwaditzer + Kristoffer Kleine + Graham Markall Dan Loewenherz werat - - + Niclas Olofsson + Chris Pressey + Tobias Diaz + Nikolaos-Digenis Karagiannis + Kurt Griffiths + Ben Mather + Donald Stufft + Dan Sanders + Jason Madden + Yaroslav Fedevych + Even Wiik Thomassen + Stefan Marr diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.7.0.rst @@ -0,0 +1,183 @@ +============================================= +PyPy2.7 and PyPy3.5 v5.7 - two in one release +============================================= + +We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7. +This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and +PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version +3.5.3. + +We continue to make incremental improvements to our C-API +compatibility layer (cpyext). PyPy2 can now import and run many c-extension +packages, among the most notable are numpy, cython, and pandas. Performance may +be slower than CPython, especially for frequently-called short C functions. +Please let us know if your use case is slow, we have ideas how to make things +faster but need real-world examples (not micro-benchmarks) of problematic code. + +Work proceeds at a good pace on the PyPy3.5 +version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta +release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and +as `these benchmarks show`_ it already gives a nice speed bump. +We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise +"PyPy3.5" supports the Python 3.5 language). + +CFFI_ has been updated to 1.10, improving an already great package for +interfacing with C. + +As always, this release fixed many issues and bugs raised by the +growing community of PyPy users. We strongly recommend updating. + +You can download the v5.7 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html +.. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) +============================================================================================= + +See also issues that were resolved_ + +* New features and cleanups + + * update the format of the PYPYLOG file and improvements to vmprof + * improve the consistency of RPython annotation unions + * emit more sysconfig values for downstream cextension packages + * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*, + _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork, + * add translation option --keepgoing to continue after the first AnnotationError + * detect and raise on recreation of a PyPy object from a PyObject during + tp_dealloc + * refactor and clean up poor handling of unicode exposed in work on py3.5 + * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) + * add translation time --disable_entrypoints option for embedding PyPy together + with another RPython VM + * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14 + * support translations with cpyext and the Boehm GC (for special cases like + revdb + * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is + now possible to obtain the address of any readonly object without pinning it + * refactor the initialization code in translating cpyext + * fix ``"".replace("", "x", num)`` to give the same result as CPython + * use a cffi-style C parser to create rffi objects in cpyext, now the + translating python must have cffi available + * add a rpython implementation of siphash24, allow choosing hash algorithm + randomizing the seed + * make ``attach_gdb`` work on Windows (with Visual Studio Debugger) + * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make + available as ``__pypy__.move_to_end`` and, on py3.5, + ``OrderedDict.move_to_end()`` + * remove completely RPython ``space.wrap`` in a major cleanup, differentiate + between ``space.newtext`` and ``space.newbytes`` on py3.5 + * improve shadowstack to where it is now the default in place of asmgcc + +* Bug Fixes + + * any uncaught RPython exception in the interpreter is turned into a + SystemError (rather than a segfault) + * create log files without the executable bit + * disable clock_gettime() on OS/X, since we support 10.11 and it was only + added in 10.12 + * support HAVE_FSTATVFS which was unintentionally always false + * fix user-created C-API heaptype, issue #2434 + * fix PyDict_Update is not actually the same as dict.update + * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute + issue #2446 + * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``, + ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` + * improve support for creating c-extension objects from app-level classes, + filling more slots especially ``tp_new`` and ``tp_dealloc`` + * add rstack.stack_almost_full() and use it to avoid stack overflow due to + the JIT where possible + * fix for ctypes.c_bool returning bool restype issue #2475 + * fix in corner cases with the GIL and C-API functions + +* Performance improvements: + + * clean-ups in the jit optimizeopt + * optimize ``if x is not None: return x`` or ``if x != 0: return x`` + * add ``jit.conditional_call_elidable()``, a way to tell the JIT + "conditonally call this function" returning a result + * try harder to propagate ``can_be_None=False`` information + * add ``rarithmetic.ovfcheck_int32_add/sub/mul`` + * add and use ``rgc.may_ignore_finalizer()``: an optimization hint that makes + the GC stop tracking the object + * replace malloc+memset with a single calloc, useful for large allocations? + * linux: try to implement os.urandom() as the syscall getrandom() if available + * propagate ``debug.ll_assert_not_none()`` through the JIT to reduce number of + guards + * improve the performance of ``PyDict_Next`` + * improve ``dict.pop()`` + * improve the optimization of branchy Python code by retaining more + information across failing guards + * add optimized "zero-copy" path for ``io.FileIO.readinto`` + +Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) +========================================================= + +Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo + +* New features + + * this first PyPy3.5 release implements much, but not all, of Python 3.5.3 + * PEP 456 allowing secure and interchangable hash algorithms + * use cryptography_'s cffi backend for SSL + +* Bug Fixes + + * implement fixes for some CPython issues that arose since the last release + * solve deadlocks in thread locking mechanism + +* Performance improvements: + + * do not create a list whenever descr_new of a bytesobject is called + * + * + * + +.. _resolved: whatsnew-pypy2-5.7.0.html +.. _cryptography: https://cryptography.io +.. _cppyy: cppyy.html + +Please update, and continue to help us make PyPy better. + +Cheers diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -75,9 +75,10 @@ 'Spenser Bauman':['Spenser Andrew Bauman'], 'Raffael Tfirst':['raffael.tfirst at gmail.com'], 'timo':['timo at eistee.fritz.box'], - 'Jasper Schulz':['Jasper.Schulz'], + 'Jasper Schulz':['Jasper.Schulz', 'jbs'], 'Aaron Gallagher':['"Aaron Gallagher'], 'Yasir Suhail':['yasirs'], + 'Squeaky', ['squeaky'], } alias_map = {} diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -619,6 +619,7 @@ 'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree', 'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free', 'PyObject_CallFinalizerFromDealloc', + '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', ] TYPES = {} FORWARD_DECLS = [] diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1197,3 +1197,25 @@ pass bases = module.foo(C) assert bases == (A, B) + + def test_multiple_inheritance_old_style_base(self): + module = self.import_extension('foo', [ + ("foo", "METH_O", + ''' + PyTypeObject *tp; + tp = (PyTypeObject*)args; + Py_INCREF(tp->tp_bases); + return tp->tp_bases; + ''' + )]) + # used to segfault after some iterations + for i in range(11): + print i + class A(object): + pass + class B: + pass + class C(A, B): + pass + bases = module.foo(C) + assert bases == (A, B) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -832,7 +832,10 @@ if base: inherit_special(space, pto, base) for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): - inherit_slots(space, pto, w_base) + if isinstance(w_base, W_TypeObject): + inherit_slots(space, pto, w_base) + #else: + # w_base is a W_ClassObject, ignore it if not pto.c_tp_setattro: from pypy.module.cpyext.object import PyObject_GenericSetAttr From pypy.commits at gmail.com Wed Mar 15 10:01:25 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 07:01:25 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: add missing names Message-ID: <58c94935.0f472e0a.65dbb.24cd@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r90704:f4793ab18123 Date: 2017-03-15 15:00 +0100 http://bitbucket.org/pypy/pypy/changeset/f4793ab18123/ Log: add missing names diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -620,6 +620,7 @@ 'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free', 'PyObject_CallFinalizerFromDealloc', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', + 'PyBytes_FromFormat', 'PyBytes_FromFormatV', ] TYPES = {} FORWARD_DECLS = [] From pypy.commits at gmail.com Wed Mar 15 10:25:51 2017 From: pypy.commits at gmail.com (rlamy) Date: Wed, 15 Mar 2017 07:25:51 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Use correct struct definitions for modules Message-ID: <58c94eef.45cc190a.89638.270d@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90705:4838a4613d1c Date: 2017-03-15 14:25 +0000 http://bitbucket.org/pypy/pypy/changeset/4838a4613d1c/ Log: Use correct struct definitions for modules diff --git a/pypy/module/cpyext/include/moduleobject.h b/pypy/module/cpyext/include/moduleobject.h --- a/pypy/module/cpyext/include/moduleobject.h +++ b/pypy/module/cpyext/include/moduleobject.h @@ -6,31 +6,7 @@ extern "C" { #endif -typedef struct PyModuleDef_Base { - PyObject_HEAD - PyObject* (*m_init)(void); - Py_ssize_t m_index; - PyObject* m_copy; -} PyModuleDef_Base; - -#define PyModuleDef_HEAD_INIT { \ - PyObject_HEAD_INIT(NULL) \ - NULL, /* m_init */ \ - 0, /* m_index */ \ - NULL, /* m_copy */ \ - } - -typedef struct PyModuleDef{ - PyModuleDef_Base m_base; - const char* m_name; - const char* m_doc; - Py_ssize_t m_size; - PyMethodDef *m_methods; - inquiry m_reload; - traverseproc m_traverse; - inquiry m_clear; - freefunc m_free; -}PyModuleDef; +#include "cpyext_moduleobject.h" #ifdef __cplusplus } diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -1,6 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, cpython_struct, \ - METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING +from pypy.module.cpyext.api import ( + cpython_api, METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cts, + parse_dir) from pypy.module.cpyext.pyobject import PyObject, as_pyobj from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( @@ -10,18 +11,8 @@ from pypy.module.cpyext.state import State from pypy.interpreter.error import oefmt -PyModuleDef_BaseStruct = cpython_struct( - 'PyModuleDef_Base', - []) - -PyModuleDefStruct = cpython_struct( - 'PyModuleDef', - [('m_base', PyModuleDef_BaseStruct), - ('m_name', rffi.CCHARP), - ('m_doc', rffi.CCHARP), - ('m_methods', lltype.Ptr(PyMethodDef)), - ], level=2) -PyModuleDef = lltype.Ptr(PyModuleDefStruct) +cts.parse_header(parse_dir / 'cpyext_moduleobject.h') +PyModuleDef = cts.gettype('PyModuleDef *') @cpython_api([PyModuleDef, rffi.INT_real], PyObject) def PyModule_Create2(space, module, api_version): diff --git a/pypy/module/cpyext/parse/cpyext_moduleobject.h b/pypy/module/cpyext/parse/cpyext_moduleobject.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/parse/cpyext_moduleobject.h @@ -0,0 +1,38 @@ +typedef struct PyModuleDef_Base { + PyObject_HEAD + PyObject* (*m_init)(void); + Py_ssize_t m_index; + PyObject* m_copy; +} PyModuleDef_Base; + +#define PyModuleDef_HEAD_INIT { \ + PyObject_HEAD_INIT(NULL) \ + NULL, /* m_init */ \ + 0, /* m_index */ \ + NULL, /* m_copy */ \ + } + +struct PyModuleDef_Slot; +/* New in 3.5 */ +typedef struct PyModuleDef_Slot{ + int slot; + void *value; +} PyModuleDef_Slot; + +#define Py_mod_create 1 +#define Py_mod_exec 2 + +#define _Py_mod_LAST_SLOT 2 + + +typedef struct PyModuleDef{ + PyModuleDef_Base m_base; + const char* m_name; + const char* m_doc; + Py_ssize_t m_size; + PyMethodDef *m_methods; + struct PyModuleDef_Slot* m_slots; + traverseproc m_traverse; + inquiry m_clear; + freefunc m_free; +} PyModuleDef; From pypy.commits at gmail.com Wed Mar 15 12:22:49 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 15 Mar 2017 09:22:49 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: remove files not needed by vmprof anymore Message-ID: <58c96a59.e5d7190a.d7911.2f13@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90706:47e816fb4917 Date: 2017-03-15 15:44 +0100 http://bitbucket.org/pypy/pypy/changeset/47e816fb4917/ Log: remove files not needed by vmprof anymore diff too long, truncating to 2000 out of 10237 lines diff --git a/rpython/rlib/rvmprof/src/shared/libudis86/Makefile.am b/rpython/rlib/rvmprof/src/shared/libudis86/Makefile.am deleted file mode 100644 --- a/rpython/rlib/rvmprof/src/shared/libudis86/Makefile.am +++ /dev/null @@ -1,51 +0,0 @@ -# -# -- udis86/libudis86 -# - -PYTHON = @PYTHON@ -OPTABLE = @top_srcdir@/docs/x86/optable.xml - -MAINTAINERCLEANFILES = Makefile.in - -lib_LTLIBRARIES = libudis86.la - -libudis86_la_SOURCES = \ - itab.c \ - decode.c \ - syn.c \ - syn-intel.c \ - syn-att.c \ - udis86.c \ - udint.h \ - syn.h \ - decode.h - -include_ladir = ${includedir}/libudis86 -include_la_HEADERS = \ - types.h \ - extern.h \ - itab.h - - -BUILT_SOURCES = \ - itab.c \ - itab.h - -# -# DLLs may not contain undefined symbol references. -# We have the linker check this explicitly. -# -if TARGET_WINDOWS -libudis86_la_LDFLAGS = -no-undefined -version-info 0:0:0 -endif - -itab.c itab.h: $(OPTABLE) \ - $(top_srcdir)/scripts/ud_itab.py \ - $(top_srcdir)/scripts/ud_opcode.py - $(PYTHON) $(top_srcdir)/scripts/ud_itab.py $(OPTABLE) $(srcdir) - - -clean-local: - rm -rf $(BUILT_SOURCES) - -maintainer-clean-local: diff --git a/rpython/rlib/rvmprof/src/shared/libudis86/README.md b/rpython/rlib/rvmprof/src/shared/libudis86/README.md deleted file mode 100644 --- a/rpython/rlib/rvmprof/src/shared/libudis86/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# COPYING - -Copied from https://github.com/vmt/udis86 and ran the script command to generate itab.h/c - -# LICENCE - -BSD 2-clause license diff --git a/rpython/rlib/rvmprof/src/shared/libudis86/decode.c b/rpython/rlib/rvmprof/src/shared/libudis86/decode.c deleted file mode 100644 --- a/rpython/rlib/rvmprof/src/shared/libudis86/decode.c +++ /dev/null @@ -1,1266 +0,0 @@ -/* udis86 - libudis86/decode.c - * - * Copyright (c) 2002-2009 Vivek Thampi - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#include "udint.h" -#include "types.h" -#include "extern.h" -#include "decode.h" - -#ifndef __UD_STANDALONE__ -# include -#endif /* __UD_STANDALONE__ */ - -/* The max number of prefixes to an instruction */ -#define MAX_PREFIXES 15 - -/* rex prefix bits */ -#define REX_W(r) ( ( 0xF & ( r ) ) >> 3 ) -#define REX_R(r) ( ( 0x7 & ( r ) ) >> 2 ) -#define REX_X(r) ( ( 0x3 & ( r ) ) >> 1 ) -#define REX_B(r) ( ( 0x1 & ( r ) ) >> 0 ) -#define REX_PFX_MASK(n) ( ( P_REXW(n) << 3 ) | \ - ( P_REXR(n) << 2 ) | \ - ( P_REXX(n) << 1 ) | \ - ( P_REXB(n) << 0 ) ) - -/* scable-index-base bits */ -#define SIB_S(b) ( ( b ) >> 6 ) -#define SIB_I(b) ( ( ( b ) >> 3 ) & 7 ) -#define SIB_B(b) ( ( b ) & 7 ) - -/* modrm bits */ -#define MODRM_REG(b) ( ( ( b ) >> 3 ) & 7 ) -#define MODRM_NNN(b) ( ( ( b ) >> 3 ) & 7 ) -#define MODRM_MOD(b) ( ( ( b ) >> 6 ) & 3 ) -#define MODRM_RM(b) ( ( b ) & 7 ) - -static int decode_ext(struct ud *u, uint16_t ptr); -static int decode_opcode(struct ud *u); - -enum reg_class { /* register classes */ - REGCLASS_GPR, - REGCLASS_MMX, - REGCLASS_CR, - REGCLASS_DB, - REGCLASS_SEG, - REGCLASS_XMM -}; - - /* - * inp_start - * Should be called before each de-code operation. - */ -static void -inp_start(struct ud *u) -{ - u->inp_ctr = 0; -} - -static uint8_t -inp_peek(struct ud *u) -{ - if (u->inp_end == 0) { - if (u->inp_buf != NULL) { - if (u->inp_buf_index < u->inp_buf_size) { - return u->inp_buf[u->inp_buf_index]; - } - } else if (u->inp_peek != UD_EOI) { - return u->inp_peek; - } else { - int c; - if ((c = u->inp_hook(u)) != UD_EOI) { - u->inp_peek = c; - return u->inp_peek; - } - } - } - u->inp_end = 1; - UDERR(u, "byte expected, eoi received\n"); - return 0; -} - -static uint8_t -inp_next(struct ud *u) -{ - if (u->inp_end == 0) { - if (u->inp_buf != NULL) { - if (u->inp_buf_index < u->inp_buf_size) { - u->inp_ctr++; - return (u->inp_curr = u->inp_buf[u->inp_buf_index++]); - } - } else { - int c = u->inp_peek; - if (c != UD_EOI || (c = u->inp_hook(u)) != UD_EOI) { - u->inp_peek = UD_EOI; - u->inp_curr = c; - u->inp_sess[u->inp_ctr++] = u->inp_curr; - return u->inp_curr; - } - } - } - u->inp_end = 1; - UDERR(u, "byte expected, eoi received\n"); - return 0; -} - -static uint8_t -inp_curr(struct ud *u) -{ - return u->inp_curr; -} - - -/* - * inp_uint8 - * int_uint16 - * int_uint32 - * int_uint64 - * Load little-endian values from input - */ -static uint8_t -inp_uint8(struct ud* u) -{ - return inp_next(u); -} - -static uint16_t -inp_uint16(struct ud* u) -{ - uint16_t r, ret; - - ret = inp_next(u); - r = inp_next(u); - return ret | (r << 8); -} - -static uint32_t -inp_uint32(struct ud* u) -{ - uint32_t r, ret; - - ret = inp_next(u); - r = inp_next(u); - ret = ret | (r << 8); - r = inp_next(u); - ret = ret | (r << 16); - r = inp_next(u); - return ret | (r << 24); -} - -static uint64_t -inp_uint64(struct ud* u) -{ - uint64_t r, ret; - - ret = inp_next(u); - r = inp_next(u); - ret = ret | (r << 8); - r = inp_next(u); - ret = ret | (r << 16); - r = inp_next(u); - ret = ret | (r << 24); - r = inp_next(u); - ret = ret | (r << 32); - r = inp_next(u); - ret = ret | (r << 40); - r = inp_next(u); - ret = ret | (r << 48); - r = inp_next(u); - return ret | (r << 56); -} - - -static UD_INLINE int -eff_opr_mode(int dis_mode, int rex_w, int pfx_opr) -{ - if (dis_mode == 64) { - return rex_w ? 64 : (pfx_opr ? 16 : 32); - } else if (dis_mode == 32) { - return pfx_opr ? 16 : 32; - } else { - UD_ASSERT(dis_mode == 16); - return pfx_opr ? 32 : 16; - } -} - - -static UD_INLINE int -eff_adr_mode(int dis_mode, int pfx_adr) -{ - if (dis_mode == 64) { - return pfx_adr ? 32 : 64; - } else if (dis_mode == 32) { - return pfx_adr ? 16 : 32; - } else { - UD_ASSERT(dis_mode == 16); - return pfx_adr ? 32 : 16; - } -} - - -/* - * decode_prefixes - * - * Extracts instruction prefixes. - */ -static int -decode_prefixes(struct ud *u) -{ - int done = 0; - uint8_t curr = 0, last = 0; - UD_RETURN_ON_ERROR(u); - - do { - last = curr; - curr = inp_next(u); - UD_RETURN_ON_ERROR(u); - if (u->inp_ctr == MAX_INSN_LENGTH) { - UD_RETURN_WITH_ERROR(u, "max instruction length"); - } - - switch (curr) - { - case 0x2E: - u->pfx_seg = UD_R_CS; - break; - case 0x36: - u->pfx_seg = UD_R_SS; - break; - case 0x3E: - u->pfx_seg = UD_R_DS; - break; - case 0x26: - u->pfx_seg = UD_R_ES; - break; - case 0x64: - u->pfx_seg = UD_R_FS; - break; - case 0x65: - u->pfx_seg = UD_R_GS; - break; - case 0x67: /* adress-size override prefix */ - u->pfx_adr = 0x67; - break; - case 0xF0: - u->pfx_lock = 0xF0; - break; - case 0x66: - u->pfx_opr = 0x66; - break; - case 0xF2: - u->pfx_str = 0xf2; - break; - case 0xF3: - u->pfx_str = 0xf3; - break; - default: - /* consume if rex */ - done = (u->dis_mode == 64 && (curr & 0xF0) == 0x40) ? 0 : 1; - break; - } - } while (!done); - /* rex prefixes in 64bit mode, must be the last prefix */ - if (u->dis_mode == 64 && (last & 0xF0) == 0x40) { - u->pfx_rex = last; - } - return 0; -} - - -/* - * vex_l, vex_w - * Return the vex.L and vex.W bits - */ -static UD_INLINE uint8_t -vex_l(const struct ud *u) -{ - UD_ASSERT(u->vex_op != 0); - return ((u->vex_op == 0xc4 ? u->vex_b2 : u->vex_b1) >> 2) & 1; -} - -static UD_INLINE uint8_t -vex_w(const struct ud *u) -{ - UD_ASSERT(u->vex_op != 0); - return u->vex_op == 0xc4 ? ((u->vex_b2 >> 7) & 1) : 0; -} - - -static UD_INLINE uint8_t -modrm(struct ud * u) -{ - if ( !u->have_modrm ) { - u->modrm = inp_next( u ); - u->modrm_offset = (uint8_t) (u->inp_ctr - 1); - u->have_modrm = 1; - } - return u->modrm; -} - - -static unsigned int -resolve_operand_size(const struct ud* u, ud_operand_size_t osize) -{ - switch (osize) { - case SZ_V: - return u->opr_mode; - case SZ_Z: - return u->opr_mode == 16 ? 16 : 32; - case SZ_Y: - return u->opr_mode == 16 ? 32 : u->opr_mode; - case SZ_RDQ: - return u->dis_mode == 64 ? 64 : 32; - case SZ_X: - UD_ASSERT(u->vex_op != 0); - return (P_VEXL(u->itab_entry->prefix) && vex_l(u)) ? SZ_QQ : SZ_DQ; - default: - return osize; - } -} - - -static int resolve_mnemonic( struct ud* u ) -{ - /* resolve 3dnow weirdness. */ - if ( u->mnemonic == UD_I3dnow ) { - u->mnemonic = ud_itab[ u->le->table[ inp_curr( u ) ] ].mnemonic; - } - /* SWAPGS is only valid in 64bits mode */ - if ( u->mnemonic == UD_Iswapgs && u->dis_mode != 64 ) { - UDERR(u, "swapgs invalid in 64bits mode\n"); - return -1; - } - - if (u->mnemonic == UD_Ixchg) { - if ((u->operand[0].type == UD_OP_REG && u->operand[0].base == UD_R_AX && - u->operand[1].type == UD_OP_REG && u->operand[1].base == UD_R_AX) || - (u->operand[0].type == UD_OP_REG && u->operand[0].base == UD_R_EAX && - u->operand[1].type == UD_OP_REG && u->operand[1].base == UD_R_EAX)) { - u->operand[0].type = UD_NONE; - u->operand[1].type = UD_NONE; - u->mnemonic = UD_Inop; - } - } - - if (u->mnemonic == UD_Inop && u->pfx_repe) { - u->pfx_repe = 0; - u->mnemonic = UD_Ipause; - } - return 0; -} - - -/* ----------------------------------------------------------------------------- - * decode_a()- Decodes operands of the type seg:offset - * ----------------------------------------------------------------------------- - */ -static void -decode_a(struct ud* u, struct ud_operand *op) -{ - if (u->opr_mode == 16) { - /* seg16:off16 */ - op->type = UD_OP_PTR; - op->size = 32; - op->lval.ptr.off = inp_uint16(u); - op->lval.ptr.seg = inp_uint16(u); - } else { - /* seg16:off32 */ - op->type = UD_OP_PTR; - op->size = 48; - op->lval.ptr.off = inp_uint32(u); - op->lval.ptr.seg = inp_uint16(u); - } -} - -/* ----------------------------------------------------------------------------- - * decode_gpr() - Returns decoded General Purpose Register - * ----------------------------------------------------------------------------- - */ -static enum ud_type -decode_gpr(register struct ud* u, unsigned int s, unsigned char rm) -{ - switch (s) { - case 64: - return UD_R_RAX + rm; - case 32: - return UD_R_EAX + rm; - case 16: - return UD_R_AX + rm; - case 8: - if (u->dis_mode == 64 && u->pfx_rex) { - if (rm >= 4) - return UD_R_SPL + (rm-4); - return UD_R_AL + rm; - } else return UD_R_AL + rm; - case 0: - /* invalid size in case of a decode error */ - UD_ASSERT(u->error); - return UD_NONE; - default: - UD_ASSERT(!"invalid operand size"); - return UD_NONE; - } -} - -static void -decode_reg(struct ud *u, - struct ud_operand *opr, - int type, - int num, - int size) -{ - int reg; - size = resolve_operand_size(u, size); - switch (type) { - case REGCLASS_GPR : reg = decode_gpr(u, size, num); break; - case REGCLASS_MMX : reg = UD_R_MM0 + (num & 7); break; - case REGCLASS_XMM : - reg = num + (size == SZ_QQ ? UD_R_YMM0 : UD_R_XMM0); - break; - case REGCLASS_CR : reg = UD_R_CR0 + num; break; - case REGCLASS_DB : reg = UD_R_DR0 + num; break; - case REGCLASS_SEG : { - /* - * Only 6 segment registers, anything else is an error. - */ - if ((num & 7) > 5) { - UDERR(u, "invalid segment register value\n"); - return; - } else { - reg = UD_R_ES + (num & 7); - } - break; - } - default: - UD_ASSERT(!"invalid register type"); - return; - } - opr->type = UD_OP_REG; - opr->base = reg; - opr->size = size; -} - - -/* - * decode_imm - * - * Decode Immediate values. - */ -static void -decode_imm(struct ud* u, unsigned int size, struct ud_operand *op) -{ - op->size = resolve_operand_size(u, size); - op->type = UD_OP_IMM; - - switch (op->size) { - case 8: op->lval.sbyte = inp_uint8(u); break; - case 16: op->lval.uword = inp_uint16(u); break; - case 32: op->lval.udword = inp_uint32(u); break; - case 64: op->lval.uqword = inp_uint64(u); break; - default: return; - } -} - - -/* - * decode_mem_disp - * - * Decode mem address displacement. - */ -static void -decode_mem_disp(struct ud* u, unsigned int size, struct ud_operand *op) -{ - switch (size) { - case 8: - op->offset = 8; - op->lval.ubyte = inp_uint8(u); - break; - case 16: - op->offset = 16; - op->lval.uword = inp_uint16(u); - break; - case 32: - op->offset = 32; - op->lval.udword = inp_uint32(u); - break; - case 64: - op->offset = 64; - op->lval.uqword = inp_uint64(u); - break; - default: - return; - } -} - - -/* - * decode_modrm_reg - * - * Decodes reg field of mod/rm byte - * - */ -static UD_INLINE void -decode_modrm_reg(struct ud *u, - struct ud_operand *operand, - unsigned int type, - unsigned int size) -{ - uint8_t reg = (REX_R(u->_rex) << 3) | MODRM_REG(modrm(u)); - decode_reg(u, operand, type, reg, size); -} - - -/* - * decode_modrm_rm - * - * Decodes rm field of mod/rm byte - * - */ -static void -decode_modrm_rm(struct ud *u, - struct ud_operand *op, - unsigned char type, /* register type */ - unsigned int size) /* operand size */ - -{ - size_t offset = 0; - unsigned char mod, rm; - - /* get mod, r/m and reg fields */ - mod = MODRM_MOD(modrm(u)); - rm = (REX_B(u->_rex) << 3) | MODRM_RM(modrm(u)); - - /* - * If mod is 11b, then the modrm.rm specifies a register. - * - */ - if (mod == 3) { - decode_reg(u, op, type, rm, size); - return; - } - - /* - * !11b => Memory Address - */ - op->type = UD_OP_MEM; - op->size = resolve_operand_size(u, size); - - if (u->adr_mode == 64) { - op->base = UD_R_RAX + rm; - if (mod == 1) { - offset = 8; - } else if (mod == 2) { - offset = 32; - } else if (mod == 0 && (rm & 7) == 5) { - op->base = UD_R_RIP; - offset = 32; - } else { - offset = 0; - } - /* - * Scale-Index-Base (SIB) - */ - if ((rm & 7) == 4) { - inp_next(u); - - op->base = UD_R_RAX + (SIB_B(inp_curr(u)) | (REX_B(u->_rex) << 3)); - op->index = UD_R_RAX + (SIB_I(inp_curr(u)) | (REX_X(u->_rex) << 3)); - /* special conditions for base reference */ - if (op->index == UD_R_RSP) { - op->index = UD_NONE; - op->scale = UD_NONE; - } else { - op->scale = (1 << SIB_S(inp_curr(u))) & ~1; - } - - if (op->base == UD_R_RBP || op->base == UD_R_R13) { - if (mod == 0) { - op->base = UD_NONE; - } - if (mod == 1) { - offset = 8; - } else { - offset = 32; - } - } - } else { - op->scale = UD_NONE; - op->index = UD_NONE; - } - } else if (u->adr_mode == 32) { - op->base = UD_R_EAX + rm; - if (mod == 1) { - offset = 8; - } else if (mod == 2) { - offset = 32; - } else if (mod == 0 && rm == 5) { - op->base = UD_NONE; - offset = 32; - } else { - offset = 0; - } - - /* Scale-Index-Base (SIB) */ - if ((rm & 7) == 4) { - inp_next(u); - - op->scale = (1 << SIB_S(inp_curr(u))) & ~1; - op->index = UD_R_EAX + (SIB_I(inp_curr(u)) | (REX_X(u->pfx_rex) << 3)); - op->base = UD_R_EAX + (SIB_B(inp_curr(u)) | (REX_B(u->pfx_rex) << 3)); - - if (op->index == UD_R_ESP) { - op->index = UD_NONE; - op->scale = UD_NONE; - } - - /* special condition for base reference */ - if (op->base == UD_R_EBP) { - if (mod == 0) { - op->base = UD_NONE; - } - if (mod == 1) { - offset = 8; - } else { - offset = 32; - } - } - } else { - op->scale = UD_NONE; - op->index = UD_NONE; - } - } else { - const unsigned int bases[] = { UD_R_BX, UD_R_BX, UD_R_BP, UD_R_BP, - UD_R_SI, UD_R_DI, UD_R_BP, UD_R_BX }; - const unsigned int indices[] = { UD_R_SI, UD_R_DI, UD_R_SI, UD_R_DI, - UD_NONE, UD_NONE, UD_NONE, UD_NONE }; - op->base = bases[rm & 7]; - op->index = indices[rm & 7]; - op->scale = UD_NONE; - if (mod == 0 && rm == 6) { - offset = 16; - op->base = UD_NONE; - } else if (mod == 1) { - offset = 8; - } else if (mod == 2) { - offset = 16; - } - } - - if (offset) { - decode_mem_disp(u, offset, op); - } else { - op->offset = 0; - } -} - - -/* - * decode_moffset - * Decode offset-only memory operand - */ -static void -decode_moffset(struct ud *u, unsigned int size, struct ud_operand *opr) -{ - opr->type = UD_OP_MEM; - opr->base = UD_NONE; - opr->index = UD_NONE; - opr->scale = UD_NONE; - opr->size = resolve_operand_size(u, size); - decode_mem_disp(u, u->adr_mode, opr); -} - - -static void -decode_vex_vvvv(struct ud *u, struct ud_operand *opr, unsigned size) -{ - uint8_t vvvv; - UD_ASSERT(u->vex_op != 0); - vvvv = ((u->vex_op == 0xc4 ? u->vex_b2 : u->vex_b1) >> 3) & 0xf; - decode_reg(u, opr, REGCLASS_XMM, (0xf & ~vvvv), size); -} - - -/* - * decode_vex_immreg - * Decode source operand encoded in immediate byte [7:4] - */ -static int -decode_vex_immreg(struct ud *u, struct ud_operand *opr, unsigned size) -{ - uint8_t imm = inp_next(u); - uint8_t mask = u->dis_mode == 64 ? 0xf : 0x7; - UD_RETURN_ON_ERROR(u); - UD_ASSERT(u->vex_op != 0); - decode_reg(u, opr, REGCLASS_XMM, mask & (imm >> 4), size); - return 0; -} - - -/* - * decode_operand - * - * Decodes a single operand. - * Returns the type of the operand (UD_NONE if none) - */ -static int -decode_operand(struct ud *u, - struct ud_operand *operand, - enum ud_operand_code type, - unsigned int size) -{ - operand->type = UD_NONE; - operand->_oprcode = type; - - switch (type) { - case OP_A : - decode_a(u, operand); - break; - case OP_MR: - decode_modrm_rm(u, operand, REGCLASS_GPR, - MODRM_MOD(modrm(u)) == 3 ? - Mx_reg_size(size) : Mx_mem_size(size)); - break; - case OP_F: - u->br_far = 1; - /* intended fall through */ - case OP_M: - if (MODRM_MOD(modrm(u)) == 3) { - UDERR(u, "expected modrm.mod != 3\n"); - } - /* intended fall through */ - case OP_E: - decode_modrm_rm(u, operand, REGCLASS_GPR, size); - break; - case OP_G: - decode_modrm_reg(u, operand, REGCLASS_GPR, size); - break; - case OP_sI: - case OP_I: - decode_imm(u, size, operand); - break; - case OP_I1: - operand->type = UD_OP_CONST; - operand->lval.udword = 1; - break; - case OP_N: - if (MODRM_MOD(modrm(u)) != 3) { - UDERR(u, "expected modrm.mod == 3\n"); - } - /* intended fall through */ - case OP_Q: - decode_modrm_rm(u, operand, REGCLASS_MMX, size); - break; - case OP_P: - decode_modrm_reg(u, operand, REGCLASS_MMX, size); - break; - case OP_U: - if (MODRM_MOD(modrm(u)) != 3) { - UDERR(u, "expected modrm.mod == 3\n"); - } - /* intended fall through */ - case OP_W: - decode_modrm_rm(u, operand, REGCLASS_XMM, size); - break; - case OP_V: - decode_modrm_reg(u, operand, REGCLASS_XMM, size); - break; - case OP_H: - decode_vex_vvvv(u, operand, size); - break; - case OP_MU: - decode_modrm_rm(u, operand, REGCLASS_XMM, - MODRM_MOD(modrm(u)) == 3 ? - Mx_reg_size(size) : Mx_mem_size(size)); - break; - case OP_S: - decode_modrm_reg(u, operand, REGCLASS_SEG, size); - break; - case OP_O: - decode_moffset(u, size, operand); - break; - case OP_R0: - case OP_R1: - case OP_R2: - case OP_R3: - case OP_R4: - case OP_R5: - case OP_R6: - case OP_R7: - decode_reg(u, operand, REGCLASS_GPR, - (REX_B(u->_rex) << 3) | (type - OP_R0), size); - break; - case OP_AL: - case OP_AX: - case OP_eAX: - case OP_rAX: - decode_reg(u, operand, REGCLASS_GPR, 0, size); - break; - case OP_CL: - case OP_CX: - case OP_eCX: - decode_reg(u, operand, REGCLASS_GPR, 1, size); - break; - case OP_DL: - case OP_DX: - case OP_eDX: - decode_reg(u, operand, REGCLASS_GPR, 2, size); - break; - case OP_ES: - case OP_CS: - case OP_DS: - case OP_SS: - case OP_FS: - case OP_GS: - /* in 64bits mode, only fs and gs are allowed */ - if (u->dis_mode == 64) { - if (type != OP_FS && type != OP_GS) { - UDERR(u, "invalid segment register in 64bits\n"); - } - } - operand->type = UD_OP_REG; - operand->base = (type - OP_ES) + UD_R_ES; - operand->size = 16; - break; - case OP_J : - decode_imm(u, size, operand); - operand->type = UD_OP_JIMM; - break ; - case OP_R : - if (MODRM_MOD(modrm(u)) != 3) { - UDERR(u, "expected modrm.mod == 3\n"); - } - decode_modrm_rm(u, operand, REGCLASS_GPR, size); - break; - case OP_C: - decode_modrm_reg(u, operand, REGCLASS_CR, size); - break; - case OP_D: - decode_modrm_reg(u, operand, REGCLASS_DB, size); - break; - case OP_I3 : - operand->type = UD_OP_CONST; - operand->lval.sbyte = 3; - break; - case OP_ST0: - case OP_ST1: - case OP_ST2: - case OP_ST3: - case OP_ST4: - case OP_ST5: - case OP_ST6: - case OP_ST7: - operand->type = UD_OP_REG; - operand->base = (type - OP_ST0) + UD_R_ST0; - operand->size = 80; - break; - case OP_L: - decode_vex_immreg(u, operand, size); - break; - default : - operand->type = UD_NONE; - break; - } - return operand->type; -} - - -/* - * decode_operands - * - * Disassemble upto 3 operands of the current instruction being - * disassembled. By the end of the function, the operand fields - * of the ud structure will have been filled. - */ -static int -decode_operands(struct ud* u) -{ - decode_operand(u, &u->operand[0], - u->itab_entry->operand1.type, - u->itab_entry->operand1.size); - if (u->operand[0].type != UD_NONE) { - decode_operand(u, &u->operand[1], - u->itab_entry->operand2.type, - u->itab_entry->operand2.size); - } - if (u->operand[1].type != UD_NONE) { - decode_operand(u, &u->operand[2], - u->itab_entry->operand3.type, - u->itab_entry->operand3.size); - } - if (u->operand[2].type != UD_NONE) { - decode_operand(u, &u->operand[3], - u->itab_entry->operand4.type, - u->itab_entry->operand4.size); - } - return 0; -} - -/* ----------------------------------------------------------------------------- - * clear_insn() - clear instruction structure - * ----------------------------------------------------------------------------- - */ -static void -clear_insn(register struct ud* u) -{ - u->error = 0; - u->pfx_seg = 0; - u->pfx_opr = 0; - u->pfx_adr = 0; - u->pfx_lock = 0; - u->pfx_repne = 0; - u->pfx_rep = 0; - u->pfx_repe = 0; - u->pfx_rex = 0; - u->pfx_str = 0; - u->mnemonic = UD_Inone; - u->itab_entry = NULL; - u->have_modrm = 0; - u->br_far = 0; - u->vex_op = 0; - u->_rex = 0; - u->operand[0].type = UD_NONE; - u->operand[1].type = UD_NONE; - u->operand[2].type = UD_NONE; - u->operand[3].type = UD_NONE; -} - - -static UD_INLINE int -resolve_pfx_str(struct ud* u) -{ - if (u->pfx_str == 0xf3) { - if (P_STR(u->itab_entry->prefix)) { - u->pfx_rep = 0xf3; - } else { - u->pfx_repe = 0xf3; - } - } else if (u->pfx_str == 0xf2) { - u->pfx_repne = 0xf3; - } - return 0; -} - - -static int -resolve_mode( struct ud* u ) -{ - int default64; - /* if in error state, bail out */ - if ( u->error ) return -1; - - /* propagate prefix effects */ - if ( u->dis_mode == 64 ) { /* set 64bit-mode flags */ - - /* Check validity of instruction m64 */ - if ( P_INV64( u->itab_entry->prefix ) ) { - UDERR(u, "instruction invalid in 64bits\n"); - return -1; - } - - /* compute effective rex based on, - * - vex prefix (if any) - * - rex prefix (if any, and not vex) - * - allowed prefixes specified by the opcode map - */ - if (u->vex_op == 0xc4) { - /* vex has rex.rxb in 1's complement */ - u->_rex = ((~(u->vex_b1 >> 5) & 0x7) /* rex.0rxb */ | - ((u->vex_b2 >> 4) & 0x8) /* rex.w000 */); - } else if (u->vex_op == 0xc5) { - /* vex has rex.r in 1's complement */ - u->_rex = (~(u->vex_b1 >> 5)) & 4; - } else { - UD_ASSERT(u->vex_op == 0); - u->_rex = u->pfx_rex; - } - u->_rex &= REX_PFX_MASK(u->itab_entry->prefix); - - /* whether this instruction has a default operand size of - * 64bit, also hardcoded into the opcode map. - */ - default64 = P_DEF64( u->itab_entry->prefix ); - /* calculate effective operand size */ - if (REX_W(u->_rex)) { - u->opr_mode = 64; - } else if ( u->pfx_opr ) { - u->opr_mode = 16; - } else { - /* unless the default opr size of instruction is 64, - * the effective operand size in the absence of rex.w - * prefix is 32. - */ - u->opr_mode = default64 ? 64 : 32; - } - - /* calculate effective address size */ - u->adr_mode = (u->pfx_adr) ? 32 : 64; - } else if ( u->dis_mode == 32 ) { /* set 32bit-mode flags */ - u->opr_mode = ( u->pfx_opr ) ? 16 : 32; - u->adr_mode = ( u->pfx_adr ) ? 16 : 32; - } else if ( u->dis_mode == 16 ) { /* set 16bit-mode flags */ - u->opr_mode = ( u->pfx_opr ) ? 32 : 16; - u->adr_mode = ( u->pfx_adr ) ? 32 : 16; - } - - return 0; -} - - -static UD_INLINE int -decode_insn(struct ud *u, uint16_t ptr) -{ - UD_ASSERT((ptr & 0x8000) == 0); - u->itab_entry = &ud_itab[ ptr ]; - u->mnemonic = u->itab_entry->mnemonic; - return (resolve_pfx_str(u) == 0 && - resolve_mode(u) == 0 && - decode_operands(u) == 0 && - resolve_mnemonic(u) == 0) ? 0 : -1; -} - - -/* - * decode_3dnow() - * - * Decoding 3dnow is a little tricky because of its strange opcode - * structure. The final opcode disambiguation depends on the last - * byte that comes after the operands have been decoded. Fortunately, - * all 3dnow instructions have the same set of operand types. So we - * go ahead and decode the instruction by picking an arbitrarily chosen - * valid entry in the table, decode the operands, and read the final - * byte to resolve the menmonic. - */ -static UD_INLINE int -decode_3dnow(struct ud* u) -{ - uint16_t ptr; - UD_ASSERT(u->le->type == UD_TAB__OPC_3DNOW); - UD_ASSERT(u->le->table[0xc] != 0); - decode_insn(u, u->le->table[0xc]); - inp_next(u); - if (u->error) { - return -1; - } - ptr = u->le->table[inp_curr(u)]; - UD_ASSERT((ptr & 0x8000) == 0); - u->mnemonic = ud_itab[ptr].mnemonic; - return 0; -} - - -static int -decode_ssepfx(struct ud *u) -{ - uint8_t idx; - uint8_t pfx; - - /* - * String prefixes (f2, f3) take precedence over operand - * size prefix (66). - */ - pfx = u->pfx_str; - if (pfx == 0) { - pfx = u->pfx_opr; - } - idx = ((pfx & 0xf) + 1) / 2; - if (u->le->table[idx] == 0) { - idx = 0; - } - if (idx && u->le->table[idx] != 0) { - /* - * "Consume" the prefix as a part of the opcode, so it is no - * longer exported as an instruction prefix. - */ - u->pfx_str = 0; - if (pfx == 0x66) { - /* - * consume "66" only if it was used for decoding, leaving - * it to be used as an operands size override for some - * simd instructions. - */ - u->pfx_opr = 0; - } - } - return decode_ext(u, u->le->table[idx]); -} - - -static int -decode_vex(struct ud *u) -{ - uint8_t index; - if (u->dis_mode != 64 && MODRM_MOD(inp_peek(u)) != 0x3) { - index = 0; - } else { - u->vex_op = inp_curr(u); - u->vex_b1 = inp_next(u); - if (u->vex_op == 0xc4) { - uint8_t pp, m; - /* 3-byte vex */ - u->vex_b2 = inp_next(u); - UD_RETURN_ON_ERROR(u); - m = u->vex_b1 & 0x1f; - if (m == 0 || m > 3) { - UD_RETURN_WITH_ERROR(u, "reserved vex.m-mmmm value"); - } - pp = u->vex_b2 & 0x3; - index = (pp << 2) | m; - } else { - /* 2-byte vex */ - UD_ASSERT(u->vex_op == 0xc5); - index = 0x1 | ((u->vex_b1 & 0x3) << 2); - } - } - return decode_ext(u, u->le->table[index]); -} - - -/* - * decode_ext() - * - * Decode opcode extensions (if any) - */ -static int -decode_ext(struct ud *u, uint16_t ptr) -{ - uint8_t idx = 0; - if ((ptr & 0x8000) == 0) { - return decode_insn(u, ptr); - } - u->le = &ud_lookup_table_list[(~0x8000 & ptr)]; - if (u->le->type == UD_TAB__OPC_3DNOW) { - return decode_3dnow(u); - } - - switch (u->le->type) { - case UD_TAB__OPC_MOD: - /* !11 = 0, 11 = 1 */ - idx = (MODRM_MOD(modrm(u)) + 1) / 4; - break; - /* disassembly mode/operand size/address size based tables. - * 16 = 0,, 32 = 1, 64 = 2 - */ - case UD_TAB__OPC_MODE: - idx = u->dis_mode != 64 ? 0 : 1; - break; - case UD_TAB__OPC_OSIZE: - idx = eff_opr_mode(u->dis_mode, REX_W(u->pfx_rex), u->pfx_opr) / 32; - break; - case UD_TAB__OPC_ASIZE: - idx = eff_adr_mode(u->dis_mode, u->pfx_adr) / 32; - break; - case UD_TAB__OPC_X87: - idx = modrm(u) - 0xC0; - break; - case UD_TAB__OPC_VENDOR: - if (u->vendor == UD_VENDOR_ANY) { - /* choose a valid entry */ - idx = (u->le->table[idx] != 0) ? 0 : 1; - } else if (u->vendor == UD_VENDOR_AMD) { - idx = 0; - } else { - idx = 1; - } - break; - case UD_TAB__OPC_RM: - idx = MODRM_RM(modrm(u)); - break; - case UD_TAB__OPC_REG: - idx = MODRM_REG(modrm(u)); - break; - case UD_TAB__OPC_SSE: - return decode_ssepfx(u); - case UD_TAB__OPC_VEX: - return decode_vex(u); - case UD_TAB__OPC_VEX_W: - idx = vex_w(u); - break; - case UD_TAB__OPC_VEX_L: - idx = vex_l(u); - break; - case UD_TAB__OPC_TABLE: - inp_next(u); - return decode_opcode(u); - default: - UD_ASSERT(!"not reached"); - break; - } - - return decode_ext(u, u->le->table[idx]); -} - - -static int -decode_opcode(struct ud *u) -{ - uint16_t ptr; - UD_ASSERT(u->le->type == UD_TAB__OPC_TABLE); - UD_RETURN_ON_ERROR(u); - ptr = u->le->table[inp_curr(u)]; - return decode_ext(u, ptr); -} - - -/* ============================================================================= - * ud_decode() - Instruction decoder. Returns the number of bytes decoded. - * ============================================================================= - */ -unsigned int -ud_decode(struct ud *u) -{ - inp_start(u); - clear_insn(u); - u->le = &ud_lookup_table_list[0]; - u->error = decode_prefixes(u) == -1 || - decode_opcode(u) == -1 || - u->error; - /* Handle decode error. */ - if (u->error) { - /* clear out the decode data. */ - clear_insn(u); - /* mark the sequence of bytes as invalid. */ - u->itab_entry = &ud_itab[0]; /* entry 0 is invalid */ - u->mnemonic = u->itab_entry->mnemonic; - } - - /* maybe this stray segment override byte - * should be spewed out? - */ - if ( !P_SEG( u->itab_entry->prefix ) && - u->operand[0].type != UD_OP_MEM && - u->operand[1].type != UD_OP_MEM ) - u->pfx_seg = 0; - - u->insn_offset = u->pc; /* set offset of instruction */ - u->asm_buf_fill = 0; /* set translation buffer index to 0 */ - u->pc += u->inp_ctr; /* move program counter by bytes decoded */ - - /* return number of bytes disassembled. */ - return u->inp_ctr; -} - -/* -vim: set ts=2 sw=2 expandtab -*/ diff --git a/rpython/rlib/rvmprof/src/shared/libudis86/decode.h b/rpython/rlib/rvmprof/src/shared/libudis86/decode.h deleted file mode 100644 --- a/rpython/rlib/rvmprof/src/shared/libudis86/decode.h +++ /dev/null @@ -1,197 +0,0 @@ -/* udis86 - libudis86/decode.h - * - * Copyright (c) 2002-2009 Vivek Thampi - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef UD_DECODE_H -#define UD_DECODE_H - -#include "types.h" -#include "udint.h" -#include "itab.h" - -#define MAX_INSN_LENGTH 15 - -/* itab prefix bits */ -#define P_none ( 0 ) - -#define P_inv64 ( 1 << 0 ) -#define P_INV64(n) ( ( n >> 0 ) & 1 ) -#define P_def64 ( 1 << 1 ) -#define P_DEF64(n) ( ( n >> 1 ) & 1 ) - -#define P_oso ( 1 << 2 ) -#define P_OSO(n) ( ( n >> 2 ) & 1 ) -#define P_aso ( 1 << 3 ) -#define P_ASO(n) ( ( n >> 3 ) & 1 ) - -#define P_rexb ( 1 << 4 ) -#define P_REXB(n) ( ( n >> 4 ) & 1 ) -#define P_rexw ( 1 << 5 ) -#define P_REXW(n) ( ( n >> 5 ) & 1 ) -#define P_rexr ( 1 << 6 ) -#define P_REXR(n) ( ( n >> 6 ) & 1 ) -#define P_rexx ( 1 << 7 ) -#define P_REXX(n) ( ( n >> 7 ) & 1 ) - -#define P_seg ( 1 << 8 ) -#define P_SEG(n) ( ( n >> 8 ) & 1 ) - -#define P_vexl ( 1 << 9 ) -#define P_VEXL(n) ( ( n >> 9 ) & 1 ) -#define P_vexw ( 1 << 10 ) -#define P_VEXW(n) ( ( n >> 10 ) & 1 ) - -#define P_str ( 1 << 11 ) -#define P_STR(n) ( ( n >> 11 ) & 1 ) -#define P_strz ( 1 << 12 ) -#define P_STR_ZF(n) ( ( n >> 12 ) & 1 ) - -/* operand type constants -- order is important! */ - -enum ud_operand_code { - OP_NONE, - - OP_A, OP_E, OP_M, OP_G, - OP_I, OP_F, - - OP_R0, OP_R1, OP_R2, OP_R3, - OP_R4, OP_R5, OP_R6, OP_R7, - - OP_AL, OP_CL, OP_DL, - OP_AX, OP_CX, OP_DX, - OP_eAX, OP_eCX, OP_eDX, - OP_rAX, OP_rCX, OP_rDX, - - OP_ES, OP_CS, OP_SS, OP_DS, - OP_FS, OP_GS, - - OP_ST0, OP_ST1, OP_ST2, OP_ST3, - OP_ST4, OP_ST5, OP_ST6, OP_ST7, - - OP_J, OP_S, OP_O, - OP_I1, OP_I3, OP_sI, - - OP_V, OP_W, OP_Q, OP_P, - OP_U, OP_N, OP_MU, OP_H, - OP_L, - - OP_R, OP_C, OP_D, - - OP_MR -} UD_ATTR_PACKED; - - -/* - * Operand size constants - * - * Symbolic constants for various operand sizes. Some of these constants - * are given a value equal to the width of the data (SZ_B == 8), such - * that they maybe used interchangeably in the internals. Modifying them - * will most certainly break things! - */ -typedef uint16_t ud_operand_size_t; - -#define SZ_NA 0 -#define SZ_Z 1 -#define SZ_V 2 -#define SZ_Y 3 -#define SZ_X 4 -#define SZ_RDQ 7 -#define SZ_B 8 -#define SZ_W 16 -#define SZ_D 32 -#define SZ_Q 64 -#define SZ_T 80 -#define SZ_O 12 -#define SZ_DQ 128 /* double quad */ -#define SZ_QQ 256 /* quad quad */ - -/* - * Complex size types; that encode sizes for operands of type MR (memory or - * register); for internal use only. Id space above 256. - */ -#define SZ_BD ((SZ_B << 8) | SZ_D) -#define SZ_BV ((SZ_B << 8) | SZ_V) -#define SZ_WD ((SZ_W << 8) | SZ_D) -#define SZ_WV ((SZ_W << 8) | SZ_V) -#define SZ_WY ((SZ_W << 8) | SZ_Y) -#define SZ_DY ((SZ_D << 8) | SZ_Y) -#define SZ_WO ((SZ_W << 8) | SZ_O) -#define SZ_DO ((SZ_D << 8) | SZ_O) -#define SZ_QO ((SZ_Q << 8) | SZ_O) - - -/* resolve complex size type. - */ -static UD_INLINE ud_operand_size_t -Mx_mem_size(ud_operand_size_t size) -{ - return (size >> 8) & 0xff; -} - -static UD_INLINE ud_operand_size_t -Mx_reg_size(ud_operand_size_t size) -{ - return size & 0xff; -} - -/* A single operand of an entry in the instruction table. - * (internal use only) - */ -struct ud_itab_entry_operand -{ - enum ud_operand_code type; - ud_operand_size_t size; -}; - - -/* A single entry in an instruction table. - *(internal use only) - */ -struct ud_itab_entry -{ - enum ud_mnemonic_code mnemonic; - struct ud_itab_entry_operand operand1; - struct ud_itab_entry_operand operand2; - struct ud_itab_entry_operand operand3; - struct ud_itab_entry_operand operand4; - uint32_t prefix; -}; - -struct ud_lookup_table_list_entry { - const uint16_t *table; - enum ud_table_type type; - const char *meta; -}; - -extern struct ud_itab_entry ud_itab[]; -extern struct ud_lookup_table_list_entry ud_lookup_table_list[]; - -#endif /* UD_DECODE_H */ - -/* vim:cindent - * vim:expandtab - * vim:ts=4 - * vim:sw=4 - */ diff --git a/rpython/rlib/rvmprof/src/shared/libudis86/extern.h b/rpython/rlib/rvmprof/src/shared/libudis86/extern.h deleted file mode 100644 --- a/rpython/rlib/rvmprof/src/shared/libudis86/extern.h +++ /dev/null @@ -1,113 +0,0 @@ -/* udis86 - libudis86/extern.h - * - * Copyright (c) 2002-2009, 2013 Vivek Thampi - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef UD_EXTERN_H -#define UD_EXTERN_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "types.h" - -#if defined(_MSC_VER) && defined(_USRDLL) -# ifdef LIBUDIS86_EXPORTS -# define LIBUDIS86_DLLEXTERN __declspec(dllexport) -# else -# define LIBUDIS86_DLLEXTERN __declspec(dllimport) -# endif -#else -# define LIBUDIS86_DLLEXTERN -#endif - -/* ============================= PUBLIC API ================================= */ - -extern LIBUDIS86_DLLEXTERN void ud_init(struct ud*); - -extern LIBUDIS86_DLLEXTERN void ud_set_mode(struct ud*, uint8_t); - -extern LIBUDIS86_DLLEXTERN void ud_set_pc(struct ud*, uint64_t); - -extern LIBUDIS86_DLLEXTERN void ud_set_input_hook(struct ud*, int (*)(struct ud*)); - -extern LIBUDIS86_DLLEXTERN void ud_set_input_buffer(struct ud*, const uint8_t*, size_t); - -#ifndef __UD_STANDALONE__ -extern LIBUDIS86_DLLEXTERN void ud_set_input_file(struct ud*, FILE*); -#endif /* __UD_STANDALONE__ */ - -extern LIBUDIS86_DLLEXTERN void ud_set_vendor(struct ud*, unsigned); - -extern LIBUDIS86_DLLEXTERN void ud_set_syntax(struct ud*, void (*)(struct ud*)); - -extern LIBUDIS86_DLLEXTERN void ud_input_skip(struct ud*, size_t); - -extern LIBUDIS86_DLLEXTERN int ud_input_end(const struct ud*); - -extern LIBUDIS86_DLLEXTERN unsigned int ud_decode(struct ud*); - -extern LIBUDIS86_DLLEXTERN unsigned int ud_disassemble(struct ud*); - -extern LIBUDIS86_DLLEXTERN void ud_translate_intel(struct ud*); - -extern LIBUDIS86_DLLEXTERN void ud_translate_att(struct ud*); - -extern LIBUDIS86_DLLEXTERN const char* ud_insn_asm(const struct ud* u); - -extern LIBUDIS86_DLLEXTERN const uint8_t* ud_insn_ptr(const struct ud* u); - -extern LIBUDIS86_DLLEXTERN uint64_t ud_insn_off(const struct ud*); - -extern LIBUDIS86_DLLEXTERN const char* ud_insn_hex(struct ud*); - -extern LIBUDIS86_DLLEXTERN unsigned int ud_insn_len(const struct ud* u); - -extern LIBUDIS86_DLLEXTERN const struct ud_operand* ud_insn_opr(const struct ud *u, unsigned int n); - -extern LIBUDIS86_DLLEXTERN int ud_opr_is_sreg(const struct ud_operand *opr); - -extern LIBUDIS86_DLLEXTERN int ud_opr_is_gpr(const struct ud_operand *opr); - -extern LIBUDIS86_DLLEXTERN enum ud_mnemonic_code ud_insn_mnemonic(const struct ud *u); - -extern LIBUDIS86_DLLEXTERN const char* ud_lookup_mnemonic(enum ud_mnemonic_code c); - -extern LIBUDIS86_DLLEXTERN void ud_set_user_opaque_data(struct ud*, void*); - -extern LIBUDIS86_DLLEXTERN void* ud_get_user_opaque_data(const struct ud*); - -extern LIBUDIS86_DLLEXTERN void ud_set_asm_buffer(struct ud *u, char *buf, size_t size); - -extern LIBUDIS86_DLLEXTERN void ud_set_sym_resolver(struct ud *u, - const char* (*resolver)(struct ud*, - uint64_t addr, - int64_t *offset)); - -/* ========================================================================== */ - -#ifdef __cplusplus -} -#endif -#endif /* UD_EXTERN_H */ diff --git a/rpython/rlib/rvmprof/src/shared/libudis86/itab.c b/rpython/rlib/rvmprof/src/shared/libudis86/itab.c deleted file mode 100644 --- a/rpython/rlib/rvmprof/src/shared/libudis86/itab.c +++ /dev/null @@ -1,5946 +0,0 @@ -/* itab.c -- generated by udis86:scripts/ud_itab.py, do no edit */ -#include "decode.h" - -#define GROUP(n) (0x8000 | (n)) -#define INVALID 0 - - -const uint16_t ud_itab__0[] = { - /* 0 */ 15, 16, 17, 18, - /* 4 */ 19, 20, GROUP(1), GROUP(2), - /* 8 */ 964, 965, 966, 967, - /* c */ 968, 969, GROUP(3), GROUP(4), - /* 10 */ 5, 6, 7, 8, - /* 14 */ 9, 10, GROUP(284), GROUP(285), - /* 18 */ 1336, 1337, 1338, 1339, - /* 1c */ 1340, 1341, GROUP(286), GROUP(287), - /* 20 */ 49, 50, 51, 52, - /* 24 */ 53, 54, INVALID, GROUP(288), - /* 28 */ 1407, 1408, 1409, 1410, - /* 2c */ 1411, 1412, INVALID, GROUP(289), - /* 30 */ 1487, 1488, 1489, 1490, - /* 34 */ 1491, 1492, INVALID, GROUP(290), - /* 38 */ 100, 101, 102, 103, - /* 3c */ 104, 105, INVALID, GROUP(291), - /* 40 */ 699, 700, 701, 702, - /* 44 */ 703, 704, 705, 706, - /* 48 */ 175, 176, 177, 178, - /* 4c */ 179, 180, 181, 182, - /* 50 */ 1246, 1247, 1248, 1249, - /* 54 */ 1250, 1251, 1252, 1253, - /* 58 */ 1101, 1102, 1103, 1104, - /* 5c */ 1105, 1106, 1107, 1108, - /* 60 */ GROUP(292), GROUP(295), GROUP(298), GROUP(299), - /* 64 */ INVALID, INVALID, INVALID, INVALID, - /* 68 */ 1254, 697, 1256, 698, - /* 6c */ 709, GROUP(300), 982, GROUP(301), - /* 70 */ 726, 728, 730, 732, - /* 74 */ 734, 736, 738, 740, - /* 78 */ 742, 744, 746, 748, - /* 7c */ 750, 752, 754, 756, - /* 80 */ GROUP(302), GROUP(303), GROUP(304), GROUP(313), - /* 84 */ 1433, 1434, 1475, 1476, - /* 88 */ 828, 829, 830, 831, - /* 8c */ 832, 770, 833, GROUP(314), - /* 90 */ 1477, 1478, 1479, 1480, - /* 94 */ 1481, 1482, 1483, 1484, - /* 98 */ GROUP(315), GROUP(316), GROUP(317), 1470, - /* 9c */ GROUP(318), GROUP(322), 1310, 766, - /* a0 */ 834, 835, 836, 837, - /* a4 */ 922, GROUP(326), 114, GROUP(327), - /* a8 */ 1435, 1436, 1402, GROUP(328), - /* ac */ 790, GROUP(329), 1346, GROUP(330), - /* b0 */ 838, 839, 840, 841, - /* b4 */ 842, 843, 844, 845, - /* b8 */ 846, 847, 848, 849, - /* bc */ 850, 851, 852, 853, - /* c0 */ GROUP(331), GROUP(332), 1301, 1302, - /* c4 */ GROUP(333), GROUP(403), GROUP(405), GROUP(406), - /* c8 */ 200, 776, 1303, 1304, - /* cc */ 713, 714, GROUP(407), GROUP(408), - /* d0 */ GROUP(409), GROUP(410), GROUP(411), GROUP(412), - /* d4 */ GROUP(413), GROUP(414), GROUP(415), 1486, - /* d8 */ GROUP(416), GROUP(419), GROUP(422), GROUP(425), - /* dc */ GROUP(428), GROUP(431), GROUP(434), GROUP(437), - /* e0 */ 794, 795, 796, GROUP(440), - /* e4 */ 690, 691, 978, 979, - /* e8 */ 72, 763, GROUP(441), 765, - /* ec */ 692, 693, 980, 981, - /* f0 */ 789, 712, 1299, 1300, - /* f4 */ 687, 83, GROUP(442), GROUP(443), - /* f8 */ 77, 1395, 81, 1398, - /* fc */ 78, 1396, GROUP(444), GROUP(445), -}; - -static const uint16_t ud_itab__1[] = { - /* 0 */ 1240, INVALID, -}; - -static const uint16_t ud_itab__2[] = { - /* 0 */ 1096, INVALID, -}; - -static const uint16_t ud_itab__3[] = { - /* 0 */ 1241, INVALID, -}; - -static const uint16_t ud_itab__4[] = { - /* 0 */ GROUP(5), GROUP(6), 767, 797, - /* 4 */ INVALID, 1426, 82, 1431, - /* 8 */ 716, 1471, INVALID, 1444, - /* c */ INVALID, GROUP(27), 430, GROUP(28), - /* 10 */ GROUP(29), GROUP(30), GROUP(31), GROUP(34), - /* 14 */ GROUP(35), GROUP(36), GROUP(37), GROUP(40), - /* 18 */ GROUP(41), 955, 956, 957, - /* 1c */ 958, 959, 960, 961, - /* 20 */ 854, 855, 856, 857, - /* 24 */ INVALID, INVALID, INVALID, INVALID, - /* 28 */ GROUP(42), GROUP(43), GROUP(44), GROUP(45), - /* 2c */ GROUP(46), GROUP(47), GROUP(48), GROUP(49), - /* 30 */ 1472, 1297, 1295, 1296, - /* 34 */ GROUP(50), GROUP(52), INVALID, 1514, - /* 38 */ GROUP(54), INVALID, GROUP(116), INVALID, - /* 3c */ INVALID, INVALID, INVALID, INVALID, - /* 40 */ 84, 85, 86, 87, - /* 44 */ 88, 89, 90, 91, - /* 48 */ 92, 93, 94, 95, - /* 4c */ 96, 97, 98, 99, - /* 50 */ GROUP(143), GROUP(144), GROUP(145), GROUP(146), - /* 54 */ GROUP(147), GROUP(148), GROUP(149), GROUP(150), - /* 58 */ GROUP(151), GROUP(152), GROUP(153), GROUP(154), - /* 5c */ GROUP(155), GROUP(156), GROUP(157), GROUP(158), - /* 60 */ GROUP(159), GROUP(160), GROUP(161), GROUP(162), - /* 64 */ GROUP(163), GROUP(164), GROUP(165), GROUP(166), - /* 68 */ GROUP(167), GROUP(168), GROUP(169), GROUP(170), - /* 6c */ GROUP(171), GROUP(172), GROUP(173), GROUP(176), - /* 70 */ GROUP(177), GROUP(178), GROUP(182), GROUP(186), - /* 74 */ GROUP(191), GROUP(192), GROUP(193), 199, - /* 78 */ GROUP(194), GROUP(195), INVALID, INVALID, - /* 7c */ GROUP(196), GROUP(197), GROUP(198), GROUP(201), - /* 80 */ 727, 729, 731, 733, - /* 84 */ 735, 737, 739, 741, - /* 88 */ 743, 745, 747, 749, - /* 8c */ 751, 753, 755, 757, - /* 90 */ 1350, 1351, 1352, 1353, - /* 94 */ 1354, 1355, 1356, 1357, - /* 98 */ 1358, 1359, 1360, 1361, - /* 9c */ 1362, 1363, 1364, 1365, - /* a0 */ 1245, 1100, 131, 1670, - /* a4 */ 1375, 1376, GROUP(202), GROUP(207), - /* a8 */ 1244, 1099, 1305, 1675, - /* ac */ 1377, 1378, GROUP(215), 694, - /* b0 */ 122, 123, 775, 1673, - /* b4 */ 772, 773, 940, 941, - /* b8 */ GROUP(221), INVALID, GROUP(222), 1671, - /* bc */ 1659, 1660, 930, 931, - /* c0 */ 1473, 1474, GROUP(223), 904, - /* c4 */ GROUP(224), GROUP(225), GROUP(226), GROUP(227), - /* c8 */ 1661, 1662, 1663, 1664, - /* cc */ 1665, 1666, 1667, 1668, - /* d0 */ GROUP(236), GROUP(237), GROUP(238), GROUP(239), - /* d4 */ GROUP(240), GROUP(241), GROUP(242), GROUP(243), - /* d8 */ GROUP(244), GROUP(245), GROUP(246), GROUP(247), - /* dc */ GROUP(248), GROUP(249), GROUP(250), GROUP(251), - /* e0 */ GROUP(252), GROUP(253), GROUP(254), GROUP(255), - /* e4 */ GROUP(256), GROUP(257), GROUP(258), GROUP(259), - /* e8 */ GROUP(260), GROUP(261), GROUP(262), GROUP(263), - /* ec */ GROUP(264), GROUP(265), GROUP(266), GROUP(267), - /* f0 */ GROUP(268), GROUP(269), GROUP(270), GROUP(271), - /* f4 */ GROUP(272), GROUP(273), GROUP(274), GROUP(275), - /* f8 */ GROUP(277), GROUP(278), GROUP(279), GROUP(280), - /* fc */ GROUP(281), GROUP(282), GROUP(283), INVALID, -}; - -static const uint16_t ud_itab__5[] = { - /* 0 */ 1384, 1406, 786, 798, - /* 4 */ 1453, 1454, INVALID, INVALID, -}; - -static const uint16_t ud_itab__6[] = { - /* 0 */ GROUP(7), GROUP(8), -}; - -static const uint16_t ud_itab__7[] = { - /* 0 */ 1374, 1383, 785, 774, - /* 4 */ 1385, INVALID, 787, 719, -}; - -static const uint16_t ud_itab__8[] = { - /* 0 */ GROUP(9), GROUP(14), GROUP(15), GROUP(16), - /* 4 */ 1386, INVALID, 788, GROUP(25), -}; - -static const uint16_t ud_itab__9[] = { - /* 0 */ INVALID, GROUP(10), GROUP(11), GROUP(12), - /* 4 */ GROUP(13), INVALID, INVALID, INVALID, -}; - -static const uint16_t ud_itab__10[] = { - /* 0 */ INVALID, 1455, INVALID, -}; - -static const uint16_t ud_itab__11[] = { - /* 0 */ INVALID, 1461, INVALID, -}; - -static const uint16_t ud_itab__12[] = { - /* 0 */ INVALID, 1462, INVALID, -}; - -static const uint16_t ud_itab__13[] = { - /* 0 */ INVALID, 1463, INVALID, -}; - -static const uint16_t ud_itab__14[] = { - /* 0 */ 824, 952, INVALID, INVALID, - /* 4 */ INVALID, INVALID, INVALID, INVALID, -}; - -static const uint16_t ud_itab__15[] = { - /* 0 */ 1485, 1508, INVALID, INVALID, - /* 4 */ INVALID, INVALID, INVALID, INVALID, -}; - -static const uint16_t ud_itab__16[] = { - /* 0 */ GROUP(17), GROUP(18), GROUP(19), GROUP(20), - /* 4 */ GROUP(21), GROUP(22), GROUP(23), GROUP(24), -}; - -static const uint16_t ud_itab__17[] = { - /* 0 */ 1466, INVALID, INVALID, -}; - -static const uint16_t ud_itab__18[] = { - /* 0 */ 1467, INVALID, INVALID, -}; - -static const uint16_t ud_itab__19[] = { - /* 0 */ 1468, INVALID, INVALID, -}; - -static const uint16_t ud_itab__20[] = { - /* 0 */ 1469, INVALID, INVALID, -}; - -static const uint16_t ud_itab__21[] = { - /* 0 */ 1397, INVALID, INVALID, -}; - -static const uint16_t ud_itab__22[] = { - /* 0 */ 80, INVALID, INVALID, -}; - -static const uint16_t ud_itab__23[] = { - /* 0 */ 1399, INVALID, INVALID, -}; - -static const uint16_t ud_itab__24[] = { - /* 0 */ 720, INVALID, INVALID, -}; - -static const uint16_t ud_itab__25[] = { - /* 0 */ 1425, GROUP(26), INVALID, INVALID, - /* 4 */ INVALID, INVALID, INVALID, INVALID, -}; - -static const uint16_t ud_itab__26[] = { - /* 0 */ 1298, INVALID, INVALID, -}; - -static const uint16_t ud_itab__27[] = { - /* 0 */ 1119, 1120, 1121, 1122, - /* 4 */ 1123, 1124, 1125, 1126, -}; - -static const uint16_t ud_itab__28[] = { - /* 0 */ INVALID, INVALID, INVALID, INVALID, - /* 4 */ INVALID, INVALID, INVALID, INVALID, - /* 8 */ INVALID, INVALID, INVALID, INVALID, - /* c */ 1216, 1217, INVALID, INVALID, - /* 10 */ INVALID, INVALID, INVALID, INVALID, - /* 14 */ INVALID, INVALID, INVALID, INVALID, - /* 18 */ INVALID, INVALID, INVALID, INVALID, - /* 1c */ 1218, 1219, INVALID, INVALID, - /* 20 */ INVALID, INVALID, INVALID, INVALID, - /* 24 */ INVALID, INVALID, INVALID, INVALID, - /* 28 */ INVALID, INVALID, INVALID, INVALID, - /* 2c */ INVALID, INVALID, INVALID, INVALID, - /* 30 */ INVALID, INVALID, INVALID, INVALID, - /* 34 */ INVALID, INVALID, INVALID, INVALID, - /* 38 */ INVALID, INVALID, INVALID, INVALID, - /* 3c */ INVALID, INVALID, INVALID, INVALID, - /* 40 */ INVALID, INVALID, INVALID, INVALID, - /* 44 */ INVALID, INVALID, INVALID, INVALID, - /* 48 */ INVALID, INVALID, INVALID, INVALID, - /* 4c */ INVALID, INVALID, INVALID, INVALID, - /* 50 */ INVALID, INVALID, INVALID, INVALID, - /* 54 */ INVALID, INVALID, INVALID, INVALID, - /* 58 */ INVALID, INVALID, INVALID, INVALID, - /* 5c */ INVALID, INVALID, INVALID, INVALID, - /* 60 */ INVALID, INVALID, INVALID, INVALID, - /* 64 */ INVALID, INVALID, INVALID, INVALID, - /* 68 */ INVALID, INVALID, INVALID, INVALID, - /* 6c */ INVALID, INVALID, INVALID, INVALID, - /* 70 */ INVALID, INVALID, INVALID, INVALID, - /* 74 */ INVALID, INVALID, INVALID, INVALID, - /* 78 */ INVALID, INVALID, INVALID, INVALID, - /* 7c */ INVALID, INVALID, INVALID, INVALID, - /* 80 */ INVALID, INVALID, INVALID, INVALID, - /* 84 */ INVALID, INVALID, INVALID, INVALID, - /* 88 */ INVALID, INVALID, 1220, INVALID, - /* 8c */ INVALID, INVALID, 1221, INVALID, - /* 90 */ 1222, INVALID, INVALID, INVALID, - /* 94 */ 1223, INVALID, 1224, 1225, - /* 98 */ INVALID, INVALID, 1226, INVALID, - /* 9c */ INVALID, INVALID, 1227, INVALID, - /* a0 */ 1228, INVALID, INVALID, INVALID, - /* a4 */ 1229, INVALID, 1230, 1231, - /* a8 */ INVALID, INVALID, 1232, INVALID, - /* ac */ INVALID, INVALID, 1233, INVALID, - /* b0 */ 1234, INVALID, INVALID, INVALID, - /* b4 */ 1235, INVALID, 1236, 1237, - /* b8 */ INVALID, INVALID, INVALID, 1238, - /* bc */ INVALID, INVALID, INVALID, 1239, - /* c0 */ INVALID, INVALID, INVALID, INVALID, - /* c4 */ INVALID, INVALID, INVALID, INVALID, - /* c8 */ INVALID, INVALID, INVALID, INVALID, - /* cc */ INVALID, INVALID, INVALID, INVALID, - /* d0 */ INVALID, INVALID, INVALID, INVALID, - /* d4 */ INVALID, INVALID, INVALID, INVALID, - /* d8 */ INVALID, INVALID, INVALID, INVALID, - /* dc */ INVALID, INVALID, INVALID, INVALID, - /* e0 */ INVALID, INVALID, INVALID, INVALID, - /* e4 */ INVALID, INVALID, INVALID, INVALID, - /* e8 */ INVALID, INVALID, INVALID, INVALID, - /* ec */ INVALID, INVALID, INVALID, INVALID, - /* f0 */ INVALID, INVALID, INVALID, INVALID, - /* f4 */ INVALID, INVALID, INVALID, INVALID, - /* f8 */ INVALID, INVALID, INVALID, INVALID, - /* fc */ INVALID, INVALID, INVALID, INVALID, -}; - -static const uint16_t ud_itab__29[] = { - /* 0 */ 936, 925, 928, 932, -}; - -static const uint16_t ud_itab__30[] = { - /* 0 */ 938, 926, 929, 934, -}; - -static const uint16_t ud_itab__31[] = { - /* 0 */ GROUP(32), GROUP(33), -}; - -static const uint16_t ud_itab__32[] = { - /* 0 */ 892, 1563, 1571, 888, -}; From pypy.commits at gmail.com Wed Mar 15 12:22:52 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 15 Mar 2017 09:22:52 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: fix test, no debugging symbols could be found Message-ID: <58c96a5c.97e1190a.1ec0d.2bed@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90707:0f877a858b49 Date: 2017-03-15 17:22 +0100 http://bitbucket.org/pypy/pypy/changeset/0f877a858b49/ Log: fix test, no debugging symbols could be found diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -8,6 +8,7 @@ # include "structdef.h" # include "src/threadlocal.h" # include "rvmprof.h" +# include "forwarddecl.h" #endif @@ -23,7 +24,6 @@ #ifdef RPYTHON_LL2CTYPES int IS_VMPROF_EVAL(void * ptr) { return 0; } #else -extern void * __vmprof_eval_vmprof; int IS_VMPROF_EVAL(void * ptr) { return ptr == __vmprof_eval_vmprof; diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -25,6 +25,7 @@ static int (*unw_step)(unw_cursor_t*) = NULL; static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL; static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; +static int (*unw_get_proc_name)(unw_cursor_t *, char *, size_t, unw_word_t*) = NULL; static int (*unw_is_signal_frame)(unw_cursor_t *) = NULL; static int (*unw_getcontext)(unw_context_t *) = NULL; @@ -226,7 +227,7 @@ // printf("func_addr is 0, now %p\n", rip); //} -#ifdef RPYTHON_VMPROF +#ifdef PYPY_JIT_CODEMAP long start_addr = 0; unw_word_t rip = 0; if (unw_get_reg(&cursor, UNW_REG_IP, &rip) < 0) { @@ -237,7 +238,7 @@ if (IS_VMPROF_EVAL((void*)pip.start_ip)) { // yes we found one stack entry of the python frames! return vmp_walk_and_record_python_stack_only(frame, result, max_depth, depth, pc); -#ifdef RPYTHON_VMPROF +#ifdef PYPY_JIT_CODEMAP } else if (pypy_find_codemap_at_addr(rip, &start_addr) != NULL) { depth = vmprof_write_header_for_jit_addr(result, depth, pc, max_depth); return vmp_walk_and_record_python_stack_only(frame, result, max_depth, depth, pc); @@ -468,6 +469,8 @@ #define UL_PREFIX "" #endif +extern void * __vmprof_eval_vmprof_addr; + int vmp_native_enable(void) { void * libhandle; vmp_native_traces_enabled = 1; @@ -482,6 +485,9 @@ if (!(unw_get_proc_info = dlsym(libhandle, UL_PREFIX PREFIX "_get_proc_info"))){ goto bail_out; } + if (!(unw_get_proc_name = dlsym(libhandle, UL_PREFIX PREFIX "_get_proc_name"))){ + goto bail_out; + } if (!(unw_init_local = dlsym(libhandle, UL_PREFIX PREFIX "_init_local"))) { goto bail_out; } diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -151,18 +151,22 @@ os.unlink(tmpfilename) def test_native(): - eci = ExternalCompilationInfo(compile_extra=['-g','-O1'], + eci = ExternalCompilationInfo(compile_extra=['-g','-O0'], separate_module_sources=[""" - RPY_EXTERN int native_func(void) { + RPY_EXTERN int native_func(int d) { int j = 0; - for (int i = 0; i < 420000; i++) { - j += 1; + if (d > 0) { + return native_func(d-1); + } else { + for (int i = 0; i < 42000; i++) { + j += d; + } } return j; } """]) - native_func = rffi.llexternal("native_func", [], rffi.INT, + native_func = rffi.llexternal("native_func", [rffi.INT], rffi.INT, compilation_info=eci) class MyCode: @@ -180,7 +184,7 @@ if num > 0: return main(code, num-1) else: - return native_func() + return native_func(100) tmpfilename = str(udir.join('test_rvmprof')) @@ -195,8 +199,7 @@ period = 0.0001 rvmprof.enable(fd, period, native=1) for i in range(num): - res = main(code, 10) - #assert res == 499999500000 + res = main(code, 3) rvmprof.disable() os.close(fd) return 0 @@ -217,8 +220,7 @@ walk(child, symbols) symbols = [] walk(tree, symbols) - not_found = ['n:pypy_g_main', 'n:native_func', 'n:pypy_g_f', - 'n:pypy_g_main'] + not_found = ['n:native_func'] for sym in symbols: for i,name in enumerate(not_found): if sym.startswith(name): @@ -226,7 +228,7 @@ break assert not_found == [] - fn = compile(f, [], gcpolicy="minimark") + fn = compile(f, [], gcpolicy="incminimark", lldebug=True) assert fn() == 0 try: import vmprof From pypy.commits at gmail.com Wed Mar 15 12:45:10 2017 From: pypy.commits at gmail.com (plan_rich) Date: Wed, 15 Mar 2017 09:45:10 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: remove vmp_dynamic.c and do not use the API exposed of that c file Message-ID: <58c96f96.014b2e0a.98489.2c84@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90708:76c2ff962014 Date: 2017-03-15 17:43 +0100 http://bitbucket.org/pypy/pypy/changeset/76c2ff962014/ Log: remove vmp_dynamic.c and do not use the API exposed of that c file diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -53,7 +53,6 @@ SHARED.join('machine.c'), SHARED.join('symboltable.c'), SHARED.join('vmp_stack.c'), - SHARED.join('vmp_dynamic.c'), ] + separate_module_files, post_include_bits=[], compile_extra=compile_extra @@ -92,19 +91,6 @@ lltype.Signed, compilation_info=eci, _nowrapper=True) - vmp_dyn_register_jit_page = rffi.llexternal("vmp_dyn_register_jit_page", - [lltype.Signed, lltype.Signed, rffi.CCHARP], - rffi.INT, compilation_info=eci, - _nowrapper=True) - - vmp_dyn_cancel = rffi.llexternal("vmp_dyn_cancel", [rffi.INT], - lltype.Void, compilation_info=eci, - _nowrapper=True) - - vmp_dyn_teardown = rffi.llexternal("vmp_dyn_teardown", [lltype.Void], - rffi.INT, compilation_info=eci, - _nowrapper=True) - return CInterface(locals()) diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c deleted file mode 100644 --- a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.c +++ /dev/null @@ -1,133 +0,0 @@ -#include "vmp_dynamic.h" - -#include -#include -#include - -static int g_dyn_entry_count = 0; -static int g_has_holes = -1; -static int g_dyn_entry_count_max = 0; -static unw_dyn_info_t ** g_dyn_entries = 0; - -RPY_EXTERN -int vmp_dyn_teardown(void) -{ - int i; - for (i = 0; i < g_dyn_entry_count; i++) { - unw_dyn_info_t * u = g_dyn_entries[i]; - if (u != NULL) { - free(u); - g_dyn_entries[i] = NULL; - } - } - if (g_dyn_entries != NULL) { - free(g_dyn_entries); - } - g_dyn_entry_count = 0; - g_dyn_entry_count_max = 0; - g_has_holes = -1; - return 0; -} - -static void _vmp_dyn_resize(void) { - if (g_dyn_entry_count_max == 0) { - g_dyn_entry_count_max = 128; - g_dyn_entries = (unw_dyn_info_t**)calloc(sizeof(unw_dyn_info_t*), 128); - } - - if (g_dyn_entry_count + 1 >= g_dyn_entry_count_max) { - g_dyn_entry_count_max *= 2; - g_dyn_entries = (unw_dyn_info_t**)realloc(g_dyn_entries, sizeof(unw_dyn_info_t*) * g_dyn_entry_count_max); - memset(g_dyn_entries + g_dyn_entry_count, 0, - sizeof(unw_dyn_info_t*)*(g_dyn_entry_count_max - g_dyn_entry_count)); - } -} - -static unw_dyn_info_t * _vmp_alloc_dyn_info(int * reference) -{ - unw_dyn_info_t * u; - - u = (unw_dyn_info_t*)malloc(sizeof(unw_dyn_info_t)); - - int i = 0; - int ref = -1; - if (g_has_holes >= 0) { - i = g_has_holes; - while (i < g_dyn_entry_count) { - if (g_dyn_entries[i] == NULL) { - ref = i; - g_has_holes += 1; - } - } - if (i == g_dyn_entry_count) { - _vmp_dyn_resize(); - ref = g_dyn_entry_count; - g_dyn_entry_count++; - } - } else { - _vmp_dyn_resize(); - ref = g_dyn_entry_count; - g_dyn_entry_count++; - } - assert(ref != -1 && "ref position MUST be found"); - g_dyn_entries[ref] = u; - *reference = ref; - - return u; -} - -static void _vmp_free_dyn_info(unw_dyn_info_t * u) -{ - free(u); -} - -RPY_EXTERN -int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr, - const char * name) -{ - char * name_cpy = NULL; - int ref = -1; - unw_dyn_info_t * u = _vmp_alloc_dyn_info(&ref); - if (ref == -1) { - return -1; // fail, could not alloc - } - u->start_ip = (unw_word_t)addr; - u->end_ip = (unw_word_t)end_addr; - u->format = UNW_INFO_FORMAT_DYNAMIC; - if (name != NULL) { - name_cpy = strdup(name); - } - unw_dyn_proc_info_t * ip = (unw_dyn_proc_info_t*)&(u->u); - ip->name_ptr = (unw_word_t)name_cpy; - ip->handler = 0; - // the docs say, we cannot use this field. but looking at libunwind, it just copies - // the value over when unw_get_proc_info is called. This should be fine to identify - ip->flags = DYN_JIT_FLAG; - ip->regions = NULL; - - _U_dyn_register(u); - - return ref; -} - -RPY_EXTERN -int vmp_dyn_cancel(int ref) { - unw_dyn_info_t * u; - - if (ref >= g_dyn_entry_count || ref < 0) { - return 1; - } - - u = g_dyn_entries[ref]; - if (u != NULL) { - g_dyn_entries[ref] = NULL; - if (g_has_holes > ref) { - g_has_holes = ref; - } - - _U_dyn_cancel(u); - } - - _vmp_free_dyn_info(u); - return 0; -} diff --git a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h b/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h deleted file mode 100644 --- a/rpython/rlib/rvmprof/src/shared/vmp_dynamic.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include - -#include "rvmprof.h" - -RPY_EXTERN int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr, - const char * name); -RPY_EXTERN int vmp_dyn_cancel(int ref); -RPY_EXTERN int vmp_dyn_teardown(void); - From pypy.commits at gmail.com Wed Mar 15 12:48:02 2017 From: pypy.commits at gmail.com (rlamy) Date: Wed, 15 Mar 2017 09:48:02 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix test Message-ID: <58c97042.cdd8190a.f875f.2f7c@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90709:9709a3b66222 Date: 2017-03-15 16:47 +0000 http://bitbucket.org/pypy/pypy/changeset/9709a3b66222/ Log: fix test diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -158,6 +158,6 @@ assert module.is_TupleLike(a) == 1 assert isinstance(a, tuple) assert issubclass(type(a), tuple) - assert list(a) == range(100, 400, 100) - assert list(a) == range(100, 400, 100) - assert list(a) == range(100, 400, 100) + assert list(a) == list(range(100, 400, 100)) + assert list(a) == list(range(100, 400, 100)) + assert list(a) == list(range(100, 400, 100)) From pypy.commits at gmail.com Wed Mar 15 12:56:46 2017 From: pypy.commits at gmail.com (rlamy) Date: Wed, 15 Mar 2017 09:56:46 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix test Message-ID: <58c9724e.8f1f190a.4757b.2d23@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90710:7cd246019641 Date: 2017-03-15 16:56 +0000 http://bitbucket.org/pypy/pypy/changeset/7cd246019641/ Log: fix test diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -345,7 +345,7 @@ module = self.import_extension('foo', [ ("foo", "METH_O", """ - _PyTraceMalloc_Track(0, 0, PyInt_AsLong(args) - sizeof(long)); + _PyTraceMalloc_Track(0, 0, PyLong_AsLong(args) - sizeof(long)); Py_INCREF(Py_None); return Py_None; """)]) From pypy.commits at gmail.com Wed Mar 15 13:00:02 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 10:00:02 -0700 (PDT) Subject: [pypy-commit] pypy default: Un-export a few meaningless symbols from libpypy-c.so Message-ID: <58c97312.cf4d2e0a.8f819.2cb7@mx.google.com> Author: Armin Rigo Branch: Changeset: r90711:b9f4dc1c4cd1 Date: 2017-03-15 17:59 +0100 http://bitbucket.org/pypy/pypy/changeset/b9f4dc1c4cd1/ Log: Un-export a few meaningless symbols from libpypy-c.so diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h --- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h +++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h @@ -201,10 +201,13 @@ #define BEGIN_MAPPINGS_LIST /* empty */ #define MAPPING_ENCONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, NULL}; #define MAPPING_DECONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, NULL, (void*)enc##_decmap}; #define MAPPING_ENCDEC(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, \ (void*)enc##_decmap}; #define END_MAPPINGS_LIST /* empty */ @@ -294,7 +297,7 @@ #ifdef USING_IMPORTED_MAPS #define USING_IMPORTED_MAP(charset) \ - extern const struct dbcs_map pypy_cjkmap_##charset; + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##charset; #define IMPORT_MAP(locale, charset, encmap, decmap) \ importmap(&pypy_cjkmap_##charset, encmap, decmap) diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h --- a/rpython/rlib/rvmprof/src/vmprof_getpc.h +++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h @@ -131,7 +131,7 @@ // typedef int ucontext_t; // #endif -intptr_t GetPC(ucontext_t *signal_ucontext) { +static intptr_t GetPC(ucontext_t *signal_ucontext) { // RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n"); fprintf(stderr, "GetPC is not yet implemented on Windows\n"); return NULL; @@ -142,7 +142,7 @@ // the right value for your system, and add it to the list in // vmrpof_config.h #else -intptr_t GetPC(ucontext_t *signal_ucontext) { +static intptr_t GetPC(ucontext_t *signal_ucontext) { return signal_ucontext->PC_FROM_UCONTEXT; // defined in config.h } diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h --- a/rpython/rlib/rvmprof/src/vmprof_main.h +++ b/rpython/rlib/rvmprof/src/vmprof_main.h @@ -104,8 +104,8 @@ #include -volatile int spinlock; -jmp_buf restore_point; +static volatile int spinlock; +static jmp_buf restore_point; static void segfault_handler(int arg) { From pypy.commits at gmail.com Wed Mar 15 14:44:53 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 11:44:53 -0700 (PDT) Subject: [pypy-commit] pypy default: document that we're expecting a positive value here Message-ID: <58c98ba5.4d29190a.f3c4b.3324@mx.google.com> Author: Armin Rigo Branch: Changeset: r90712:2adedf0af060 Date: 2017-03-15 19:44 +0100 http://bitbucket.org/pypy/pypy/changeset/2adedf0af060/ Log: document that we're expecting a positive value here diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py --- a/rpython/rlib/test/test_rsiphash.py +++ b/rpython/rlib/test/test_rsiphash.py @@ -52,12 +52,12 @@ os.environ['PYTHONHASHSEED'] = '0' initialize_from_env() assert siphash24("foo") == 15988776847138518036 - # value checked with CPython 3.5 + # value checked with CPython 3.5 (turned positive by adding 2**64) os.environ['PYTHONHASHSEED'] = '4000000000' initialize_from_env() assert siphash24("foo") == 13829150778707464258 - # value checked with CPython 3.5 + # value checked with CPython 3.5 (turned positive by adding 2**64) for env in ['', 'random']: os.environ['PYTHONHASHSEED'] = env From pypy.commits at gmail.com Wed Mar 15 14:49:50 2017 From: pypy.commits at gmail.com (mattip) Date: Wed, 15 Mar 2017 11:49:50 -0700 (PDT) Subject: [pypy-commit] pypy default: highlight asmgcc and update doc Message-ID: <58c98cce.1459190a.ba955.3460@mx.google.com> Author: Matti Picus Branch: Changeset: r90713:031f96ce5f5a Date: 2017-03-15 20:48 +0200 http://bitbucket.org/pypy/pypy/changeset/031f96ce5f5a/ Log: highlight asmgcc and update doc diff --git a/pypy/doc/config/translation.gcrootfinder.txt b/pypy/doc/config/translation.gcrootfinder.txt --- a/pypy/doc/config/translation.gcrootfinder.txt +++ b/pypy/doc/config/translation.gcrootfinder.txt @@ -9,10 +9,8 @@ - ``--gcrootfinder=asmgcc``: use assembler hackery to find the roots directly from the normal stack. This is a bit faster, but platform specific. It works so far with GCC or MSVC, - on i386 and x86-64. It is tested only on Linux (where it is - the default) so other platforms (as well as MSVC) may need - various fixes before they can be used. + on i386 and x86-64. It is tested only on Linux + so other platforms (as well as MSVC) may need + various fixes before they can be used. Note asmgcc will be deprecated + at some future date, and does not work with clang. -You may have to force the use of the shadowstack root finder if -you are running into troubles or if you insist on translating -PyPy with other compilers like clang. diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -24,6 +24,10 @@ CFFI_ has been updated to 1.10, improving an already great package for interfacing with C. +We now use shadowstack as our default gcrootfinder_, asmgcc will be deprecated +at some future point. While about 3% slower, shadowstack is much more +easily maintained and debuggable. + As always, this release fixed many issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. @@ -47,6 +51,7 @@ .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: project-ideas.html .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html +.. _gcrootfinder: config/translation.gcrootfinder.html What is PyPy? ============= From pypy.commits at gmail.com Wed Mar 15 14:57:51 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 11:57:51 -0700 (PDT) Subject: [pypy-commit] pypy default: a few more details Message-ID: <58c98eaf.98e5190a.8ee6c.30b1@mx.google.com> Author: Armin Rigo Branch: Changeset: r90714:4b6b0c95ec49 Date: 2017-03-15 19:57 +0100 http://bitbucket.org/pypy/pypy/changeset/4b6b0c95ec49/ Log: a few more details diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -24,9 +24,11 @@ CFFI_ has been updated to 1.10, improving an already great package for interfacing with C. -We now use shadowstack as our default gcrootfinder_, asmgcc will be deprecated -at some future point. While about 3% slower, shadowstack is much more -easily maintained and debuggable. +We now use shadowstack as our default gcrootfinder_ even on Linux. The +alternative, asmgcc, will be deprecated at some future point. While about 3% +slower, shadowstack is much more easily maintained and debuggable. Also, +the performance of shadowstack has been improved in general: this should +close the speed gap between Linux and other platforms. As always, this release fixed many issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. From pypy.commits at gmail.com Wed Mar 15 16:17:40 2017 From: pypy.commits at gmail.com (mattip) Date: Wed, 15 Mar 2017 13:17:40 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix translation (arigato around) Message-ID: <58c9a164.85d5190a.82c29.3378@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r90716:e08e9c6ffbd8 Date: 2017-03-15 22:16 +0200 http://bitbucket.org/pypy/pypy/changeset/e08e9c6ffbd8/ Log: fix translation (arigato around) diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -23,9 +23,9 @@ Most uses of this function should be using PyModule_Create() instead; only use this if you are sure you need it.""" - modname = rffi.charp2str(module.c_m_name) + modname = rffi.charp2str(rffi.cast(rffi.CCHARP, module.c_m_name)) if module.c_m_doc: - doc = rffi.charp2str(module.c_m_doc) + doc = rffi.charp2str(rffi.cast(rffi.CCHARP, module.c_m_doc)) else: doc = None methods = module.c_m_methods From pypy.commits at gmail.com Wed Mar 15 16:23:02 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 13:23:02 -0700 (PDT) Subject: [pypy-commit] cffi default: Merged in realitix/cffi/binary_enum (pull request #78) Message-ID: <58c9a2a6.09532e0a.186ec.3515@mx.google.com> Author: Armin Rigo Branch: Changeset: r2913:663852865a03 Date: 2017-03-15 20:22 +0000 http://bitbucket.org/cffi/cffi/changeset/663852865a03/ Log: Merged in realitix/cffi/binary_enum (pull request #78) Add operation support in enum diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -803,6 +803,16 @@ "the actual array length in this context" % exprnode.coord.line) # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '+'): + return (self._parse_constant(exprnode.left) + + self._parse_constant(exprnode.right)) + # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '-'): + return (self._parse_constant(exprnode.left) - + self._parse_constant(exprnode.right)) + # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py --- a/testing/cffi0/test_parsing.py +++ b/testing/cffi0/test_parsing.py @@ -386,13 +386,14 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; """) C = ffi.dlopen(None) assert C.POS == 1 assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 + assert C.OP == 2 def test_stdcall(): ffi = FFI() From pypy.commits at gmail.com Wed Mar 15 16:23:08 2017 From: pypy.commits at gmail.com (realitix) Date: Wed, 15 Mar 2017 13:23:08 -0700 (PDT) Subject: [pypy-commit] cffi binary_enum: Add test for binary operation in enum definition Message-ID: <58c9a2ac.13542e0a.5f3c0.3413@mx.google.com> Author: Jean-Sebastien Bevilacqua Branch: binary_enum Changeset: r2912:330013c6d0c3 Date: 2017-03-15 21:10 +0100 http://bitbucket.org/cffi/cffi/changeset/330013c6d0c3/ Log: Add test for binary operation in enum definition diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py --- a/testing/cffi0/test_parsing.py +++ b/testing/cffi0/test_parsing.py @@ -386,13 +386,14 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; """) C = ffi.dlopen(None) assert C.POS == 1 assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 + assert C.OP == 2 def test_stdcall(): ffi = FFI() From pypy.commits at gmail.com Wed Mar 15 16:23:05 2017 From: pypy.commits at gmail.com (realitix) Date: Wed, 15 Mar 2017 13:23:05 -0700 (PDT) Subject: [pypy-commit] cffi binary_enum: Add operation support in enum Message-ID: <58c9a2a9.a215190a.c98de.3439@mx.google.com> Author: Jean-Sebastien Bevilacqua Branch: binary_enum Changeset: r2911:5e9d6a26b6b1 Date: 2017-03-15 16:48 +0100 http://bitbucket.org/cffi/cffi/changeset/5e9d6a26b6b1/ Log: Add operation support in enum With this patch, the following C code works now: ```c typedef enum VkPipelineCacheHeaderVersion { VK_PIPELINE_CACHE_HEADER_VERSION_ONE = 1, VK_PIPELINE_CACHE_HEADER_VERSION_RANGE_SIZE = (VK_PIPELINE_CACHE_HEADER_VERSION_ONE - VK_PIPELINE_CACHE_HEADER_VERSION_ONE + 1), } VkPipelineCacheHeaderVersion; ``` diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -803,6 +803,16 @@ "the actual array length in this context" % exprnode.coord.line) # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '+'): + return (self._parse_constant(exprnode.left) + + self._parse_constant(exprnode.right)) + # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '-'): + return (self._parse_constant(exprnode.left) - + self._parse_constant(exprnode.right)) + # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) From pypy.commits at gmail.com Wed Mar 15 16:25:02 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 13:25:02 -0700 (PDT) Subject: [pypy-commit] pypy default: import cffi/663852865a03 Message-ID: <58c9a31e.51512e0a.0834.2dca@mx.google.com> Author: Armin Rigo Branch: Changeset: r90717:ccf33d2353fa Date: 2017-03-15 21:24 +0100 http://bitbucket.org/pypy/pypy/changeset/ccf33d2353fa/ Log: import cffi/663852865a03 diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -803,6 +803,16 @@ "the actual array length in this context" % exprnode.coord.line) # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '+'): + return (self._parse_constant(exprnode.left) + + self._parse_constant(exprnode.right)) + # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '-'): + return (self._parse_constant(exprnode.left) - + self._parse_constant(exprnode.right)) + # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py @@ -387,13 +387,14 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; """) C = ffi.dlopen(None) assert C.POS == 1 assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 + assert C.OP == 2 def test_stdcall(): ffi = FFI() From pypy.commits at gmail.com Wed Mar 15 16:25:17 2017 From: pypy.commits at gmail.com (arigo) Date: Wed, 15 Mar 2017 13:25:17 -0700 (PDT) Subject: [pypy-commit] cffi release-1.10: hg merge default Message-ID: <58c9a32d.110f2e0a.c307.36f3@mx.google.com> Author: Armin Rigo Branch: release-1.10 Changeset: r2914:00667b1b0ab3 Date: 2017-03-15 21:25 +0100 http://bitbucket.org/cffi/cffi/changeset/00667b1b0ab3/ Log: hg merge default diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -803,6 +803,16 @@ "the actual array length in this context" % exprnode.coord.line) # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '+'): + return (self._parse_constant(exprnode.left) + + self._parse_constant(exprnode.right)) + # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '-'): + return (self._parse_constant(exprnode.left) - + self._parse_constant(exprnode.right)) + # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py --- a/testing/cffi0/test_parsing.py +++ b/testing/cffi0/test_parsing.py @@ -386,13 +386,14 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; """) C = ffi.dlopen(None) assert C.POS == 1 assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 + assert C.OP == 2 def test_stdcall(): ffi = FFI() From pypy.commits at gmail.com Wed Mar 15 17:53:25 2017 From: pypy.commits at gmail.com (mjacob) Date: Wed, 15 Mar 2017 14:53:25 -0700 (PDT) Subject: [pypy-commit] pypy default: Silence glibc warning. Message-ID: <58c9b7d5.46162e0a.800d.386f@mx.google.com> Author: Manuel Jacob Branch: Changeset: r90718:853bb72fa73c Date: 2017-03-15 22:51 +0100 http://bitbucket.org/pypy/pypy/changeset/853bb72fa73c/ Log: Silence glibc warning. diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -237,6 +237,8 @@ 'sys/resource.h', 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h', 'signal.h', 'sys/utsname.h', _ptyh] + if sys.platform.startswith('linux'): + includes.append('sys/sysmacros.h') if sys.platform.startswith('freebsd'): includes.append('sys/ttycom.h') libraries = ['util'] From pypy.commits at gmail.com Wed Mar 15 19:05:45 2017 From: pypy.commits at gmail.com (mjacob) Date: Wed, 15 Mar 2017 16:05:45 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix test_importlib.util.EXTENSIONS for lazily built _testcapi on PyPy. Message-ID: <58c9c8c9.41072e0a.4527a.38ec@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90719:cba0df095b3f Date: 2017-03-16 00:04 +0100 http://bitbucket.org/pypy/pypy/changeset/cba0df095b3f/ Log: Fix test_importlib.util.EXTENSIONS for lazily built _testcapi on PyPy. diff --git a/lib-python/3/test/test_importlib/util.py b/lib-python/3/test/test_importlib/util.py --- a/lib-python/3/test/test_importlib/util.py +++ b/lib-python/3/test/test_importlib/util.py @@ -30,7 +30,12 @@ def _extension_details(): global EXTENSIONS - for path in sys.path: + # we need this hack on PyPy because _testcapi is built lazily + import _testcapi + import _pypy_testcapi + lib_pypy_dir = os.path.dirname(_pypy_testcapi.__file__) + c_file = os.path.join(lib_pypy_dir, '_testcapimodule.c') + for path in [_pypy_testcapi.get_hashed_dir(c_file)]: for ext in machinery.EXTENSION_SUFFIXES: filename = EXTENSIONS.name + ext file_path = os.path.join(path, filename) From pypy.commits at gmail.com Thu Mar 16 03:48:18 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 16 Mar 2017 00:48:18 -0700 (PDT) Subject: [pypy-commit] pypy default: siphash24 changes for big endian machines, only byte swap when directly loading from memory. in any other case we already place the bytes into int64 integers Message-ID: <58ca4342.1e43190a.324a7.48b5@mx.google.com> Author: Richard Plangger Branch: Changeset: r90720:0555945f3314 Date: 2017-03-16 08:46 +0100 http://bitbucket.org/pypy/pypy/changeset/0555945f3314/ Log: siphash24 changes for big endian machines, only byte swap when directly loading from memory. in any other case we already place the bytes into int64 integers diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py --- a/rpython/rlib/rsiphash.py +++ b/rpython/rlib/rsiphash.py @@ -24,14 +24,11 @@ if sys.byteorder == 'little': def _le64toh(x): return x + def _le32toh(x): + return x else: _le64toh = rarithmetic.byteswap - - -class Seed: - k0l = k1l = r_uint64(0) -seed = Seed() - + _le32toh = rarithmetic.byteswap def _decode64(s): return (r_uint64(ord(s[0])) | @@ -43,6 +40,11 @@ r_uint64(ord(s[6])) << 48 | r_uint64(ord(s[7])) << 56) +class Seed: + k0l = k1l = r_uint64(0) +seed = Seed() + + def select_random_seed(s): """'s' is a string of length 16""" seed.k0l = _decode64(s) @@ -177,17 +179,13 @@ """For tests.""" global misaligned_is_fine old = seed.k0l, seed.k1l, misaligned_is_fine - seed.k0l = _le64toh(r_uint64(new_k0)) - seed.k1l = _le64toh(r_uint64(new_k1)) + seed.k0l = r_uint64(new_k0) + seed.k1l = r_uint64(new_k1) if test_misaligned_path: misaligned_is_fine = False yield seed.k0l, seed.k1l, misaligned_is_fine = old -def get_current_seed(): - return _le64toh(seed.k0l), _le64toh(seed.k1l) - - magic0 = r_uint64(0x736f6d6570736575) magic1 = r_uint64(0x646f72616e646f6d) magic2 = r_uint64(0x6c7967656e657261) @@ -271,7 +269,8 @@ size = 4 if size == 4: if direct: - t |= r_uint64(llop.raw_load(rffi.UINT, addr_in, index)) + v = _le32toh(r_uint32(llop.raw_load(rffi.UINT, addr_in, index))) + t |= r_uint64(v) size = 0 else: t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24 @@ -287,7 +286,7 @@ size = 0 assert size == 0 - b |= _le64toh(t) + b |= t v3 ^= b v0, v1, v2, v3 = _double_round(v0, v1, v2, v3) diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py --- a/rpython/rlib/test/test_rsiphash.py +++ b/rpython/rlib/test/test_rsiphash.py @@ -118,8 +118,9 @@ 123, 123, intmask(15988776847138518036), 456, 456, intmask(15988776847138518036), 789, 789] - assert s1[8] in [intmask(17593683438421985039), # ucs2 mode - intmask(94801584261658677)] # ucs4 mode + assert s1[8] in [intmask(17593683438421985039), # ucs2 mode little endian + intmask(94801584261658677), # ucs4 mode little endian + intmask(3849431280840015342),] # ucs4 mode big endian os.environ['PYTHONHASHSEED'] = '3987654321' s1 = getall() @@ -127,8 +128,9 @@ 123, 123, intmask(5890804383681474441), 456, 456, intmask(5890804383681474441), 789, 789] - assert s1[8] in [intmask(4192582507672183374), # ucs2 mode - intmask(7179255293164649778)] # ucs4 mode + assert s1[8] in [intmask(4192582507672183374), # ucs2 mode little endian + intmask(7179255293164649778), # ucs4 mode little endian + intmask(-3945781295304514711),] # ucs4 mode big endian for env in ['', 'random']: os.environ['PYTHONHASHSEED'] = env From pypy.commits at gmail.com Thu Mar 16 08:44:06 2017 From: pypy.commits at gmail.com (mattip) Date: Thu, 16 Mar 2017 05:44:06 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into py3.5-release Message-ID: <58ca8896.de002e0a.97182.050a@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-5.x Changeset: r90723:74c8ff71e7ae Date: 2017-03-16 14:43 +0200 http://bitbucket.org/pypy/pypy/changeset/74c8ff71e7ae/ Log: merge py3.5 into py3.5-release diff --git a/lib-python/3/test/test_importlib/util.py b/lib-python/3/test/test_importlib/util.py --- a/lib-python/3/test/test_importlib/util.py +++ b/lib-python/3/test/test_importlib/util.py @@ -30,7 +30,12 @@ def _extension_details(): global EXTENSIONS - for path in sys.path: + # we need this hack on PyPy because _testcapi is built lazily + import _testcapi + import _pypy_testcapi + lib_pypy_dir = os.path.dirname(_pypy_testcapi.__file__) + c_file = os.path.join(lib_pypy_dir, '_testcapimodule.c') + for path in [_pypy_testcapi.get_hashed_dir(c_file)]: for ext in machinery.EXTENSION_SUFFIXES: filename = EXTENSIONS.name + ext file_path = os.path.join(path, filename) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -803,6 +803,16 @@ "the actual array length in this context" % exprnode.coord.line) # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '+'): + return (self._parse_constant(exprnode.left) + + self._parse_constant(exprnode.right)) + # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '-'): + return (self._parse_constant(exprnode.left) - + self._parse_constant(exprnode.right)) + # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) diff --git a/pypy/doc/config/translation.gcrootfinder.txt b/pypy/doc/config/translation.gcrootfinder.txt --- a/pypy/doc/config/translation.gcrootfinder.txt +++ b/pypy/doc/config/translation.gcrootfinder.txt @@ -9,10 +9,8 @@ - ``--gcrootfinder=asmgcc``: use assembler hackery to find the roots directly from the normal stack. This is a bit faster, but platform specific. It works so far with GCC or MSVC, - on i386 and x86-64. It is tested only on Linux (where it is - the default) so other platforms (as well as MSVC) may need - various fixes before they can be used. + on i386 and x86-64. It is tested only on Linux + so other platforms (as well as MSVC) may need + various fixes before they can be used. Note asmgcc will be deprecated + at some future date, and does not work with clang. -You may have to force the use of the shadowstack root finder if -you are running into troubles or if you insist on translating -PyPy with other compilers like clang. diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -24,6 +24,12 @@ CFFI_ has been updated to 1.10, improving an already great package for interfacing with C. +We now use shadowstack as our default gcrootfinder_ even on Linux. The +alternative, asmgcc, will be deprecated at some future point. While about 3% +slower, shadowstack is much more easily maintained and debuggable. Also, +the performance of shadowstack has been improved in general: this should +close the speed gap between Linux and other platforms. + As always, this release fixed many issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. @@ -47,6 +53,7 @@ .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: project-ideas.html .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html +.. _gcrootfinder: config/translation.gcrootfinder.html What is PyPy? ============= diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h --- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h +++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h @@ -201,10 +201,13 @@ #define BEGIN_MAPPINGS_LIST /* empty */ #define MAPPING_ENCONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, NULL}; #define MAPPING_DECONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, NULL, (void*)enc##_decmap}; #define MAPPING_ENCDEC(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, \ (void*)enc##_decmap}; #define END_MAPPINGS_LIST /* empty */ @@ -294,7 +297,7 @@ #ifdef USING_IMPORTED_MAPS #define USING_IMPORTED_MAP(charset) \ - extern const struct dbcs_map pypy_cjkmap_##charset; + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##charset; #define IMPORT_MAP(locale, charset, encmap, decmap) \ importmap(&pypy_cjkmap_##charset, encmap, decmap) diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -23,9 +23,9 @@ Most uses of this function should be using PyModule_Create() instead; only use this if you are sure you need it.""" - modname = rffi.charp2str(module.c_m_name) + modname = rffi.charp2str(rffi.cast(rffi.CCHARP, module.c_m_name)) if module.c_m_doc: - doc = rffi.charp2str(module.c_m_doc) + doc = rffi.charp2str(rffi.cast(rffi.CCHARP, module.c_m_doc)) else: doc = None methods = module.c_m_methods diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py @@ -387,13 +387,14 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; """) C = ffi.dlopen(None) assert C.POS == 1 assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 + assert C.OP == 2 def test_stdcall(): ffi = FFI() diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -237,6 +237,8 @@ 'sys/resource.h', 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h', 'signal.h', 'sys/utsname.h', _ptyh] + if sys.platform.startswith('linux'): + includes.append('sys/sysmacros.h') if sys.platform.startswith('freebsd'): includes.append('sys/ttycom.h') libraries = ['util'] diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py --- a/rpython/rlib/rsiphash.py +++ b/rpython/rlib/rsiphash.py @@ -24,14 +24,11 @@ if sys.byteorder == 'little': def _le64toh(x): return x + def _le32toh(x): + return x else: _le64toh = rarithmetic.byteswap - - -class Seed: - k0l = k1l = r_uint64(0) -seed = Seed() - + _le32toh = rarithmetic.byteswap def _decode64(s): return (r_uint64(ord(s[0])) | @@ -43,6 +40,11 @@ r_uint64(ord(s[6])) << 48 | r_uint64(ord(s[7])) << 56) +class Seed: + k0l = k1l = r_uint64(0) +seed = Seed() + + def select_random_seed(s): """'s' is a string of length 16""" seed.k0l = _decode64(s) @@ -177,17 +179,13 @@ """For tests.""" global misaligned_is_fine old = seed.k0l, seed.k1l, misaligned_is_fine - seed.k0l = _le64toh(r_uint64(new_k0)) - seed.k1l = _le64toh(r_uint64(new_k1)) + seed.k0l = r_uint64(new_k0) + seed.k1l = r_uint64(new_k1) if test_misaligned_path: misaligned_is_fine = False yield seed.k0l, seed.k1l, misaligned_is_fine = old -def get_current_seed(): - return _le64toh(seed.k0l), _le64toh(seed.k1l) - - magic0 = r_uint64(0x736f6d6570736575) magic1 = r_uint64(0x646f72616e646f6d) magic2 = r_uint64(0x6c7967656e657261) @@ -271,7 +269,8 @@ size = 4 if size == 4: if direct: - t |= r_uint64(llop.raw_load(rffi.UINT, addr_in, index)) + v = _le32toh(r_uint32(llop.raw_load(rffi.UINT, addr_in, index))) + t |= r_uint64(v) size = 0 else: t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24 @@ -287,7 +286,7 @@ size = 0 assert size == 0 - b |= _le64toh(t) + b |= t v3 ^= b v0, v1, v2, v3 = _double_round(v0, v1, v2, v3) diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h --- a/rpython/rlib/rvmprof/src/vmprof_getpc.h +++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h @@ -131,7 +131,7 @@ // typedef int ucontext_t; // #endif -intptr_t GetPC(ucontext_t *signal_ucontext) { +static intptr_t GetPC(ucontext_t *signal_ucontext) { // RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n"); fprintf(stderr, "GetPC is not yet implemented on Windows\n"); return NULL; @@ -142,7 +142,7 @@ // the right value for your system, and add it to the list in // vmrpof_config.h #else -intptr_t GetPC(ucontext_t *signal_ucontext) { +static intptr_t GetPC(ucontext_t *signal_ucontext) { return signal_ucontext->PC_FROM_UCONTEXT; // defined in config.h } diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h --- a/rpython/rlib/rvmprof/src/vmprof_main.h +++ b/rpython/rlib/rvmprof/src/vmprof_main.h @@ -104,8 +104,8 @@ #include -volatile int spinlock; -jmp_buf restore_point; +static volatile int spinlock; +static jmp_buf restore_point; static void segfault_handler(int arg) { diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py --- a/rpython/rlib/test/test_rsiphash.py +++ b/rpython/rlib/test/test_rsiphash.py @@ -52,12 +52,12 @@ os.environ['PYTHONHASHSEED'] = '0' initialize_from_env() assert siphash24("foo") == 15988776847138518036 - # value checked with CPython 3.5 + # value checked with CPython 3.5 (turned positive by adding 2**64) os.environ['PYTHONHASHSEED'] = '4000000000' initialize_from_env() assert siphash24("foo") == 13829150778707464258 - # value checked with CPython 3.5 + # value checked with CPython 3.5 (turned positive by adding 2**64) for env in ['', 'random']: os.environ['PYTHONHASHSEED'] = env @@ -118,8 +118,9 @@ 123, 123, intmask(15988776847138518036), 456, 456, intmask(15988776847138518036), 789, 789] - assert s1[8] in [intmask(17593683438421985039), # ucs2 mode - intmask(94801584261658677)] # ucs4 mode + assert s1[8] in [intmask(17593683438421985039), # ucs2 mode little endian + intmask(94801584261658677), # ucs4 mode little endian + intmask(3849431280840015342),] # ucs4 mode big endian os.environ['PYTHONHASHSEED'] = '3987654321' s1 = getall() @@ -127,8 +128,9 @@ 123, 123, intmask(5890804383681474441), 456, 456, intmask(5890804383681474441), 789, 789] - assert s1[8] in [intmask(4192582507672183374), # ucs2 mode - intmask(7179255293164649778)] # ucs4 mode + assert s1[8] in [intmask(4192582507672183374), # ucs2 mode little endian + intmask(7179255293164649778), # ucs4 mode little endian + intmask(-3945781295304514711),] # ucs4 mode big endian for env in ['', 'random']: os.environ['PYTHONHASHSEED'] = env From pypy.commits at gmail.com Thu Mar 16 08:44:01 2017 From: pypy.commits at gmail.com (mattip) Date: Thu, 16 Mar 2017 05:44:01 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default into branch Message-ID: <58ca8891.1ce5190a.10480.5591@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-5.x Changeset: r90721:a7294dae4050 Date: 2017-03-16 14:41 +0200 http://bitbucket.org/pypy/pypy/changeset/a7294dae4050/ Log: merge default into branch diff too long, truncating to 2000 out of 2146 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,5 @@ +#encoding utf-8 + License ======= @@ -42,9 +44,9 @@ Antonio Cuni Samuele Pedroni Matti Picus + Ronan Lamy Alex Gaynor Philip Jenvey - Ronan Lamy Brian Kearns Richard Plangger Michael Hudson @@ -55,12 +57,12 @@ Hakan Ardo Benjamin Peterson Anders Chrigstrom + Wim Lavrijsen Eric van Riet Paap - Wim Lavrijsen Richard Emslie Alexander Schremmer + Remi Meier Dan Villiom Podlaski Christiansen - Remi Meier Lukas Diekmann Sven Hager Anders Lehmann @@ -83,8 +85,8 @@ Lawrence Oluyede Bartosz Skowron Daniel Roberts + Adrien Di Mascio Niko Matsakis - Adrien Di Mascio Alexander Hesse Ludovic Aubry Jacob Hallen @@ -100,8 +102,8 @@ Michael Foord Stephan Diehl Stefan Schwarzer + Tomek Meka Valentino Volonghi - Tomek Meka Stefano Rivera Patrick Maupin Devin Jeanpierre @@ -109,268 +111,273 @@ Bruno Gola David Malcolm Jean-Paul Calderone - Timo Paulssen Edd Barrett Squeaky + Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton + Nicolas Truessel Martin Matusiak - Nicolas Truessel + Wenzhu Man Konstantin Lopuhin - Wenzhu Man John Witulski Laurence Tratt + Greg Price Ivan Sichmann Freitas - Greg Price Dario Bertini + Jeremy Thurgood Mark Pearse Simon Cross - Jeremy Thurgood + Tobias Pape Andreas Stührk - Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski + William Leslie + marky1991 + Ilya Osadchiy + Tobias Oberstein Paul deGrandis - Ilya Osadchiy - marky1991 - Tobias Oberstein + Boris Feigin + Taavi Burns Adrian Kuhn - Boris Feigin tav - Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp + Mike Blume Gerald Klix - Mike Blume Oscar Nierstrasz + Rami Chowdhury Stefan H. Muller - Rami Chowdhury + Joannah Nanjekye Eugene Oden + Tim Felgentreff + Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton - Jeff Terrace - Tim Felgentreff Dusty Phillips Lukas Renggli Guenter Jantzen - William Leslie Ned Batchelder + Amit Regmi Anton Gulenko - Amit Regmi - Ben Young + Sergey Matyunin Jasper Schulz + Andrew Chambers Nicolas Chauvat Andrew Durdin - Andrew Chambers - Sergey Matyunin + Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu + Michael Twomey Rocco Moretti Gintautas Miliauskas - Michael Twomey Lucian Branescu Mihaila anatoly techtonik + Karl Bartel Gabriel Lavoie + Jared Grubb Olivier Dormond - Jared Grubb - Karl Bartel Wouter van Heyst + Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina - Sebastian Pawluś - Stuart Williams - Daniel Patrick Aaron Iles Toby Watson + Daniel Patrick + Stuart Williams Antoine Pitrou Christian Hudon + Justas Sadzevicius + Neil Shepperd Michael Cheng - Justas Sadzevicius + Mikael Schönenberg + Stanislaw Halik + Berkin Ilbeyi Gasper Zejn - Neil Shepperd - Stanislaw Halik - Mikael Schönenberg - Berkin Ilbeyi Faye Zhao Elmo Mäntynen - Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani + Jonathan David Riehl Beatrice During Alex Perry + p_zieschang at yahoo.de + Robert Zaremba + Alan McIntyre + Alexander Sedov Vaibhav Sood - Alan McIntyre Reuben Cummings - Alexander Sedov - p_zieschang at yahoo.de Attila Gobi Christopher Pope - Aaron Gallagher + Tristan Arthur + Christian Tismer + Dan Stromberg + Carl Meyer Florin Papa - Christian Tismer - Marc Abramowitz - Dan Stromberg - Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Marc Abramowitz + Arjun Naik + Aaron Gallagher Alexis Daboville - Jens-Uwe Mager - Carl Meyer + Pieter Zieschang Karl Ramm - Pieter Zieschang - Gabriel Lukas Vacek - Kunal Grover - Andrew Dalke + Omer Katz + Jacek Generowicz Sylvain Thenault Jakub Stasiak + Stefan Beyer + Andrew Dalke + Alejandro J. Cura + Vladimir Kryachko + Gabriel + Mark Williams + Kunal Grover Nathan Taylor - Vladimir Kryachko - Omer Katz - Mark Williams - Jacek Generowicz - Alejandro J. Cura + Travis Francis Athougies + Yasir Suhail + Sergey Kishchenko + Martin Blais + Lutz Paelike + Ian Foote + Philipp Rustemeuer + Catalin Gabriel Manciu Jacob Oscarson - Travis Francis Athougies Ryan Gonzalez - Ian Foote Kristjan Valur Jonsson + Lucio Torre + Richard Lancaster + Dan Buch + Lene Wagner + Tomo Cocoa + Alecsandru Patrascu David Lievens Neil Blakey-Milner - Lutz Paelike - Lucio Torre + Henrik Vendelbo Lars Wassermann - Philipp Rustemeuer - Henrik Vendelbo - Richard Lancaster - Yasir Suhail - Dan Buch + Ignas Mikalajunas + Christoph Gerum Miguel de Val Borro Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Alecsandru Patrascu - Christoph Gerum - Martin Blais - Lene Wagner - Catalin Gabriel Manciu - Tomo Cocoa - Kim Jin Su - rafalgalczynski at gmail.com Toni Mattis - Amber Brown + Laurens Van Houtven + Bobby Impollonia + Roberto De Ioris + Jeong YunWon + Christopher Armstrong + Aaron Tubbs + Vasantha Ganesh K + Jason Michalski + Markus Holtermann + Andrew Thompson + Yusei Tahara + Ruochen Huang + Fabio Niephaus + Akira Li + Gustavo Niemeyer + Rafał Gałczyński + Logan Chien Lucas Stadler - Julian Berman - Markus Holtermann roberto at goyle + Matt Bogosian Yury V. Zaytsev - Anna Katrina Dominguez - Bobby Impollonia - Vasantha Ganesh K - Andrew Thompson florinpapa - Yusei Tahara - Aaron Tubbs - Ben Darnell - Roberto De Ioris - Logan Chien - Juan Francisco Cantero Hurtado - Ruochen Huang - Jeong YunWon - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle Anders Sigfridsson Nikolay Zinov - Jason Michalski + rafalgalczynski at gmail.com + Joshua Gilbert + Anna Katrina Dominguez + Kim Jin Su + Amber Brown + Ben Darnell + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Julian Berman + Michael Hudson-Doyle Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer Stephan Busemann - Rafał Gałczyński - Matt Bogosian + Dan Colish timo - Christian Muirhead - Berker Peksag - James Lan Volodymyr Vladymyrov - shoma hosaka - Ben Mather - Niclas Olofsson - Matthew Miller - Rodrigo Araújo + Daniel Neuhäuser + Flavio Percoco halgari - Boglarka Vezer - Chris Pressey - Buck Golemon - Diana Popa - Konrad Delong - Dinu Gherman + Jim Baker Chris Lambacher coolbutuseless at gmail.com + Mike Bayer + Rodrigo Araújo Daniil Yarancev - Jim Baker + OlivierBlanvillain + Jonas Pfannschmidt + Zearin + Andrey Churin Dan Crosta - Nikolaos-Digenis Karagiannis - James Robert - Armin Ronacher - Brett Cannon - Donald Stufft - yrttyr - aliceinwire - OlivierBlanvillain - Dan Sanders - Zooko Wilcox-O Hearn + reubano at gmail.com + Julien Phalip + Roman Podoliaka + Eli Stevens + Boglarka Vezer + PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio - jiaaro - Mads Kiilerich Antony Lee - Jason Madden - Daniel Neuh�user - reubano at gmail.com - Yaroslav Fedevych Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - squeaky - soareschen - Jonas Pfannschmidt - Kurt Griffiths - Mike Bayer - Stefan Marr - Flavio Percoco - Kristoffer Kleine + shoma hosaka + Buck Golemon + JohnDoe + yrttyr Michael Chermside Anna Ravencroft + remarkablerocket + Berker Peksag + Christian Muirhead + soareschen + Matthew Miller + Konrad Delong + Dinu Gherman pizi - remarkablerocket - Andrey Churin - Zearin - Eli Stevens - Tobias Diaz - Julien Phalip - Roman Podoliaka + James Robert + Armin Ronacher + Diana Popa + Mads Kiilerich + Brett Cannon + aliceinwire + Zooko Wilcox-O Hearn + James Lan + jiaaro + Markus Unterwaditzer + Kristoffer Kleine + Graham Markall Dan Loewenherz werat + Niclas Olofsson + Chris Pressey + Tobias Diaz + Nikolaos-Digenis Karagiannis + Kurt Griffiths + Ben Mather + Donald Stufft + Dan Sanders + Jason Madden + Yaroslav Fedevych + Even Wiik Thomassen + Stefan Marr Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -803,6 +803,16 @@ "the actual array length in this context" % exprnode.coord.line) # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '+'): + return (self._parse_constant(exprnode.left) + + self._parse_constant(exprnode.right)) + # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '-'): + return (self._parse_constant(exprnode.left) - + self._parse_constant(exprnode.right)) + # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) diff --git a/pypy/doc/config/translation.gcrootfinder.txt b/pypy/doc/config/translation.gcrootfinder.txt --- a/pypy/doc/config/translation.gcrootfinder.txt +++ b/pypy/doc/config/translation.gcrootfinder.txt @@ -9,10 +9,8 @@ - ``--gcrootfinder=asmgcc``: use assembler hackery to find the roots directly from the normal stack. This is a bit faster, but platform specific. It works so far with GCC or MSVC, - on i386 and x86-64. It is tested only on Linux (where it is - the default) so other platforms (as well as MSVC) may need - various fixes before they can be used. + on i386 and x86-64. It is tested only on Linux + so other platforms (as well as MSVC) may need + various fixes before they can be used. Note asmgcc will be deprecated + at some future date, and does not work with clang. -You may have to force the use of the shadowstack root finder if -you are running into troubles or if you insist on translating -PyPy with other compilers like clang. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -11,9 +11,9 @@ Antonio Cuni Samuele Pedroni Matti Picus + Ronan Lamy Alex Gaynor Philip Jenvey - Ronan Lamy Brian Kearns Richard Plangger Michael Hudson @@ -24,12 +24,12 @@ Hakan Ardo Benjamin Peterson Anders Chrigstrom + Wim Lavrijsen Eric van Riet Paap - Wim Lavrijsen Richard Emslie Alexander Schremmer + Remi Meier Dan Villiom Podlaski Christiansen - Remi Meier Lukas Diekmann Sven Hager Anders Lehmann @@ -52,8 +52,8 @@ Lawrence Oluyede Bartosz Skowron Daniel Roberts + Adrien Di Mascio Niko Matsakis - Adrien Di Mascio Alexander Hesse Ludovic Aubry Jacob Hallen @@ -69,8 +69,8 @@ Michael Foord Stephan Diehl Stefan Schwarzer + Tomek Meka Valentino Volonghi - Tomek Meka Stefano Rivera Patrick Maupin Devin Jeanpierre @@ -78,267 +78,270 @@ Bruno Gola David Malcolm Jean-Paul Calderone - Timo Paulssen Edd Barrett Squeaky + Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton + Nicolas Truessel Martin Matusiak - Nicolas Truessel + Wenzhu Man Konstantin Lopuhin - Wenzhu Man John Witulski Laurence Tratt + Greg Price Ivan Sichmann Freitas - Greg Price Dario Bertini + Jeremy Thurgood Mark Pearse Simon Cross - Jeremy Thurgood + Tobias Pape Andreas Stührk - Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski + William Leslie + marky1991 + Ilya Osadchiy + Tobias Oberstein Paul deGrandis - Ilya Osadchiy - marky1991 - Tobias Oberstein + Boris Feigin + Taavi Burns Adrian Kuhn - Boris Feigin tav - Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp + Mike Blume Gerald Klix - Mike Blume Oscar Nierstrasz + Rami Chowdhury Stefan H. Muller - Rami Chowdhury + Joannah Nanjekye Eugene Oden + Tim Felgentreff + Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton - Jeff Terrace - Tim Felgentreff Dusty Phillips Lukas Renggli Guenter Jantzen - William Leslie Ned Batchelder + Amit Regmi Anton Gulenko - Amit Regmi - Ben Young + Sergey Matyunin Jasper Schulz + Andrew Chambers Nicolas Chauvat Andrew Durdin - Andrew Chambers - Sergey Matyunin + Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu + Michael Twomey Rocco Moretti Gintautas Miliauskas - Michael Twomey Lucian Branescu Mihaila anatoly techtonik + Karl Bartel Gabriel Lavoie + Jared Grubb Olivier Dormond - Jared Grubb - Karl Bartel Wouter van Heyst + Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina - Sebastian Pawluś - Stuart Williams - Daniel Patrick Aaron Iles Toby Watson + Daniel Patrick + Stuart Williams Antoine Pitrou Christian Hudon + Justas Sadzevicius + Neil Shepperd Michael Cheng - Justas Sadzevicius + Mikael Schönenberg + Stanislaw Halik + Berkin Ilbeyi Gasper Zejn - Neil Shepperd - Stanislaw Halik - Mikael Schönenberg - Berkin Ilbeyi Faye Zhao Elmo Mäntynen - Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani + Jonathan David Riehl Beatrice During Alex Perry + p_zieschang at yahoo.de + Robert Zaremba + Alan McIntyre + Alexander Sedov Vaibhav Sood - Alan McIntyre Reuben Cummings - Alexander Sedov - p_zieschang at yahoo.de Attila Gobi Christopher Pope - Aaron Gallagher + Tristan Arthur + Christian Tismer + Dan Stromberg + Carl Meyer Florin Papa - Christian Tismer - Marc Abramowitz - Dan Stromberg - Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Marc Abramowitz + Arjun Naik + Aaron Gallagher Alexis Daboville - Jens-Uwe Mager - Carl Meyer + Pieter Zieschang Karl Ramm - Pieter Zieschang - Gabriel Lukas Vacek - Kunal Grover - Andrew Dalke + Omer Katz + Jacek Generowicz Sylvain Thenault Jakub Stasiak + Stefan Beyer + Andrew Dalke + Alejandro J. Cura + Vladimir Kryachko + Gabriel + Mark Williams + Kunal Grover Nathan Taylor - Vladimir Kryachko - Omer Katz - Mark Williams - Jacek Generowicz - Alejandro J. Cura + Travis Francis Athougies + Yasir Suhail + Sergey Kishchenko + Martin Blais + Lutz Paelike + Ian Foote + Philipp Rustemeuer + Catalin Gabriel Manciu Jacob Oscarson - Travis Francis Athougies Ryan Gonzalez - Ian Foote Kristjan Valur Jonsson + Lucio Torre + Richard Lancaster + Dan Buch + Lene Wagner + Tomo Cocoa + Alecsandru Patrascu David Lievens Neil Blakey-Milner - Lutz Paelike - Lucio Torre + Henrik Vendelbo Lars Wassermann - Philipp Rustemeuer - Henrik Vendelbo - Richard Lancaster - Yasir Suhail - Dan Buch + Ignas Mikalajunas + Christoph Gerum Miguel de Val Borro Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Alecsandru Patrascu - Christoph Gerum - Martin Blais - Lene Wagner - Catalin Gabriel Manciu - Tomo Cocoa - Kim Jin Su - rafalgalczynski at gmail.com Toni Mattis - Amber Brown + Laurens Van Houtven + Bobby Impollonia + Roberto De Ioris + Jeong YunWon + Christopher Armstrong + Aaron Tubbs + Vasantha Ganesh K + Jason Michalski + Markus Holtermann + Andrew Thompson + Yusei Tahara + Ruochen Huang + Fabio Niephaus + Akira Li + Gustavo Niemeyer + Rafał Gałczyński + Logan Chien Lucas Stadler - Julian Berman - Markus Holtermann roberto at goyle + Matt Bogosian Yury V. Zaytsev - Anna Katrina Dominguez - Bobby Impollonia - Vasantha Ganesh K - Andrew Thompson florinpapa - Yusei Tahara - Aaron Tubbs - Ben Darnell - Roberto De Ioris - Logan Chien - Juan Francisco Cantero Hurtado - Ruochen Huang - Jeong YunWon - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle Anders Sigfridsson Nikolay Zinov - Jason Michalski + rafalgalczynski at gmail.com + Joshua Gilbert + Anna Katrina Dominguez + Kim Jin Su + Amber Brown + Ben Darnell + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Julian Berman + Michael Hudson-Doyle Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer Stephan Busemann - Rafał Gałczyński - Matt Bogosian + Dan Colish timo - Christian Muirhead - Berker Peksag - James Lan Volodymyr Vladymyrov - shoma hosaka - Ben Mather - Niclas Olofsson - Matthew Miller - Rodrigo Araújo + Daniel Neuhäuser + Flavio Percoco halgari - Boglarka Vezer - Chris Pressey - Buck Golemon - Diana Popa - Konrad Delong - Dinu Gherman + Jim Baker Chris Lambacher coolbutuseless at gmail.com + Mike Bayer + Rodrigo Araújo Daniil Yarancev - Jim Baker + OlivierBlanvillain + Jonas Pfannschmidt + Zearin + Andrey Churin Dan Crosta - Nikolaos-Digenis Karagiannis - James Robert - Armin Ronacher - Brett Cannon - Donald Stufft - yrttyr - aliceinwire - OlivierBlanvillain - Dan Sanders - Zooko Wilcox-O Hearn + reubano at gmail.com + Julien Phalip + Roman Podoliaka + Eli Stevens + Boglarka Vezer + PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio - jiaaro - Mads Kiilerich Antony Lee - Jason Madden - Daniel Neuhäuser - reubano at gmail.com - Yaroslav Fedevych Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - squeaky - soareschen - Jonas Pfannschmidt - Kurt Griffiths - Mike Bayer - Stefan Marr - Flavio Percoco - Kristoffer Kleine + shoma hosaka + Buck Golemon + JohnDoe + yrttyr Michael Chermside Anna Ravencroft + remarkablerocket + Berker Peksag + Christian Muirhead + soareschen + Matthew Miller + Konrad Delong + Dinu Gherman pizi - remarkablerocket - Andrey Churin - Zearin - Eli Stevens - Tobias Diaz - Julien Phalip - Roman Podoliaka + James Robert + Armin Ronacher + Diana Popa + Mads Kiilerich + Brett Cannon + aliceinwire + Zooko Wilcox-O Hearn + James Lan + jiaaro + Markus Unterwaditzer + Kristoffer Kleine + Graham Markall Dan Loewenherz werat - - + Niclas Olofsson + Chris Pressey + Tobias Diaz + Nikolaos-Digenis Karagiannis + Kurt Griffiths + Ben Mather + Donald Stufft + Dan Sanders + Jason Madden + Yaroslav Fedevych + Even Wiik Thomassen + Stefan Marr diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v5.7.0.rst release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -53,6 +54,12 @@ release-0.7.0.rst release-0.6 +CPython 3.5 compatible versions +------------------------------- + +.. toctree:: + + release-v5.7.0.rst CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.7.0.rst whatsnew-pypy2-5.6.0.rst whatsnew-pypy2-5.4.0.rst whatsnew-pypy2-5.3.1.rst diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.7.0.rst @@ -0,0 +1,190 @@ +============================================= +PyPy2.7 and PyPy3.5 v5.7 - two in one release +============================================= + +We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7. +This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and +PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version +3.5.3. + +We continue to make incremental improvements to our C-API +compatibility layer (cpyext). PyPy2 can now import and run many c-extension +packages, among the most notable are numpy, cython, and pandas. Performance may +be slower than CPython, especially for frequently-called short C functions. +Please let us know if your use case is slow, we have ideas how to make things +faster but need real-world examples (not micro-benchmarks) of problematic code. + +Work proceeds at a good pace on the PyPy3.5 +version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta +release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and +as `these benchmarks show`_ it already gives a nice speed bump. +We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise +"PyPy3.5" supports the Python 3.5 language). + +CFFI_ has been updated to 1.10, improving an already great package for +interfacing with C. + +We now use shadowstack as our default gcrootfinder_ even on Linux. The +alternative, asmgcc, will be deprecated at some future point. While about 3% +slower, shadowstack is much more easily maintained and debuggable. Also, +the performance of shadowstack has been improved in general: this should +close the speed gap between Linux and other platforms. + +As always, this release fixed many issues and bugs raised by the +growing community of PyPy users. We strongly recommend updating. + +You can download the v5.7 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html +.. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html +.. _gcrootfinder: config/translation.gcrootfinder.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) +============================================================================================= + +See also issues that were resolved_ + +* New features and cleanups + + * update the format of the PYPYLOG file and improvements to vmprof + * improve the consistency of RPython annotation unions + * emit more sysconfig values for downstream cextension packages + * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*, + _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork, + * add translation option --keepgoing to continue after the first AnnotationError + * detect and raise on recreation of a PyPy object from a PyObject during + tp_dealloc + * refactor and clean up poor handling of unicode exposed in work on py3.5 + * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) + * add translation time --disable_entrypoints option for embedding PyPy together + with another RPython VM + * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14 + * support translations with cpyext and the Boehm GC (for special cases like + revdb + * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is + now possible to obtain the address of any readonly object without pinning it + * refactor the initialization code in translating cpyext + * fix ``"".replace("", "x", num)`` to give the same result as CPython + * use a cffi-style C parser to create rffi objects in cpyext, now the + translating python must have cffi available + * add a rpython implementation of siphash24, allow choosing hash algorithm + randomizing the seed + * make ``attach_gdb`` work on Windows (with Visual Studio Debugger) + * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make + available as ``__pypy__.move_to_end`` and, on py3.5, + ``OrderedDict.move_to_end()`` + * remove completely RPython ``space.wrap`` in a major cleanup, differentiate + between ``space.newtext`` and ``space.newbytes`` on py3.5 + * improve shadowstack to where it is now the default in place of asmgcc + +* Bug Fixes + + * any uncaught RPython exception in the interpreter is turned into a + SystemError (rather than a segfault) + * create log files without the executable bit + * disable clock_gettime() on OS/X, since we support 10.11 and it was only + added in 10.12 + * support HAVE_FSTATVFS which was unintentionally always false + * fix user-created C-API heaptype, issue #2434 + * fix PyDict_Update is not actually the same as dict.update + * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute + issue #2446 + * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``, + ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` + * improve support for creating c-extension objects from app-level classes, + filling more slots especially ``tp_new`` and ``tp_dealloc`` + * add rstack.stack_almost_full() and use it to avoid stack overflow due to + the JIT where possible + * fix for ctypes.c_bool returning bool restype issue #2475 + * fix in corner cases with the GIL and C-API functions + +* Performance improvements: + + * clean-ups in the jit optimizeopt + * optimize ``if x is not None: return x`` or ``if x != 0: return x`` + * add ``jit.conditional_call_elidable()``, a way to tell the JIT + "conditonally call this function" returning a result + * try harder to propagate ``can_be_None=False`` information + * add ``rarithmetic.ovfcheck_int32_add/sub/mul`` + * add and use ``rgc.may_ignore_finalizer()``: an optimization hint that makes + the GC stop tracking the object + * replace malloc+memset with a single calloc, useful for large allocations? + * linux: try to implement os.urandom() as the syscall getrandom() if available + * propagate ``debug.ll_assert_not_none()`` through the JIT to reduce number of + guards + * improve the performance of ``PyDict_Next`` + * improve ``dict.pop()`` + * improve the optimization of branchy Python code by retaining more + information across failing guards + * add optimized "zero-copy" path for ``io.FileIO.readinto`` + +Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) +========================================================= + +Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo + +* New features + + * this first PyPy3.5 release implements much, but not all, of Python 3.5.3 + * PEP 456 allowing secure and interchangable hash algorithms + * use cryptography_'s cffi backend for SSL + +* Bug Fixes + + * implement fixes for some CPython issues that arose since the last release + * solve deadlocks in thread locking mechanism + +* Performance improvements: + + * do not create a list whenever descr_new of a bytesobject is called + * + * + * + +.. _resolved: whatsnew-pypy2-5.7.0.html +.. _cryptography: https://cryptography.io +.. _cppyy: cppyy.html + +Please update, and continue to help us make PyPy better. + +Cheers diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -75,9 +75,10 @@ 'Spenser Bauman':['Spenser Andrew Bauman'], 'Raffael Tfirst':['raffael.tfirst at gmail.com'], 'timo':['timo at eistee.fritz.box'], - 'Jasper Schulz':['Jasper.Schulz'], + 'Jasper Schulz':['Jasper.Schulz', 'jbs'], 'Aaron Gallagher':['"Aaron Gallagher'], 'Yasir Suhail':['yasirs'], + 'Squeaky', ['squeaky'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,191 +1,8 @@ ========================== -What's new in PyPy2.7 5.6+ +What's new in PyPy2.7 5.8+ ========================== -.. this is a revision shortly after release-pypy2.7-v5.6 -.. startrev: 7e9787939641 +.. this is a revision shortly after release-pypy2.7-v5.7 +.. startrev: 44f31f6dd39f -Since a while now, PyPy preserves the order of dictionaries and sets. -However, the set literal syntax ``{x, y, z}`` would by mistake build a -set with the opposite order: ``set([z, y, x])``. This has been fixed. -Note that CPython is inconsistent too: in 2.7.12, ``{5, 5.0}`` would be -``set([5.0])``, but in 2.7.trunk it is ``set([5])``. PyPy's behavior -changed in exactly the same way because of this fix. - - -.. branch: rpython-error-to-systemerror - -Any uncaught RPython exception (from a PyPy bug) is turned into an -app-level SystemError. This should improve the lot of users hitting an -uncaught RPython error. - -.. branch: union-side-effects-2 - -Try to improve the consistency of RPython annotation unions. - -.. branch: pytest-2.9.2 - -.. branch: clean-exported-state - -Clean-ups in the jit optimizeopt - -.. branch: conditional_call_value_4 - -Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally -call this function" returning a result. - -.. branch: desc-specialize - -Refactor FunctionDesc.specialize() and related code (RPython annotator). - -.. branch: raw-calloc - -.. branch: issue2446 - -Assign ``tp_doc`` to the new TypeObject's type dictionary ``__doc__`` key -so it will be picked up by app-level objects of that type - -.. branch: cling-support - -Module cppyy now uses cling as its backend (Reflex has been removed). The -user-facing interface and main developer tools (genreflex, selection files, -class loader, etc.) remain the same. A libcppyy_backend.so library is still -needed but is now available through PyPI with pip: PyPy-cppyy-backend. - -The Cling-backend brings support for modern C++ (11, 14, etc.), dynamic -template instantations, and improved integration with CFFI for better -performance. It also provides interactive C++ (and bindings to that). - -.. branch: better-PyDict_Next - -Improve the performance of ``PyDict_Next``. When trying ``PyDict_Next`` on a -typedef dict, the test exposed a problem converting a ``GetSetProperty`` to a -``PyGetSetDescrObject``. The other direction seem to be fully implemented. -This branch made a minimal effort to convert the basic fields to avoid -segfaults, but trying to use the ``PyGetSetDescrObject`` will probably fail. - -.. branch: stdlib-2.7.13 - -Updated the implementation to match CPython 2.7.13 instead of 2.7.13. - -.. branch: issue2444 - -Fix ``PyObject_GetBuffer`` and ``PyMemoryView_GET_BUFFER``, which leaked -memory and held references. Add a finalizer to CPyBuffer, add a -PyMemoryViewObject with a PyBuffer attached so that the call to -``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory. -Properly call ``bf_releasebuffer`` when not ``NULL``. - -.. branch: boehm-rawrefcount - -Support translations of cpyext with the Boehm GC (for special cases like -revdb). - -.. branch: strbuf-as-buffer - -Implement StringBuffer.get_raw_address (missing feature for the buffer protocol). -More generally it is now possible to obtain the address of any object (if it -is readonly) without pinning it. - -.. branch: cpyext-cleanup -.. branch: api_func-refactor - -Refactor cpyext initialisation. - -.. branch: cpyext-from2 - -Fix a test failure introduced by strbuf-as-buffer - -.. branch: cpyext-FromBuffer - -Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to -the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak. - -.. branch: issue2464 - -Give (almost?) all GetSetProperties a valid __objclass__. - -.. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419 -.. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033 - -.. branch: missing-tp_new - -Improve mixing app-level classes in c-extensions, especially if the app-level -class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect -all the method slots to be filled with a function pointer, where app-level will -search up the mro for an appropriate function at runtime. With this branch we -now fill many more slots in the c-extenion type objects. -Also fix for c-extension type that calls ``tp_hash`` during initialization -(str, unicode types), and fix instantiating c-extension types from built-in -classes by enforcing an order of instaniation. - -.. branch: rffi-parser-2 - -rffi structures in cpyext can now be created by parsing simple C headers. -Additionally, the cts object that holds the parsed information can act like -cffi's ffi objects, with the methods cts.cast() and cts.gettype(). - -.. branch: rpython-hash - -Don't freeze hashes in the translated pypy. In practice, that means -that we can now translate PyPy with the option --hash=siphash24 and get -the same hashes as CPython 3.5, which can be randomized (in a -crypographically good way). It is the default in PyPy3. The default of -PyPy2 remains unchanged: there are user programs out there that depend -on constant hashes (or even sometimes on specific hash results). - -.. branch: dict-move-to-end - -Our dicts, which are always ordered, now have an extra "method" for -Python 3.x which moves an item to first or last position. In PyPy 3.5 -it is the standard ``OrderedDict.move_to_end()`` method, but the -behavior is also available on Python 2.x or for the ``dict`` type by -calling ``__pypy__.move_to_end(dict, key, last=True)``. - - -.. branch optinfo-into-bridges-3 - -Improve the optimization of branchy Python code by retaining more information -across failing guards. - - -.. branch: space-newtext - -Internal refactoring of ``space.wrap()``, which is now replaced with -explicitly-typed methods. Notably, there are now ``space.newbytes()`` -and ``space.newtext()``: these two methods are identical on PyPy 2.7 but -not on PyPy 3.x. The latter is used to get an app-level unicode string -by decoding the RPython string, assumed to be utf-8. - -.. branch: space-wrap - -.. branch: fix_bool_restype - -Fix for ``ctypes.c_bool``-returning ctypes functions - -.. branch: fix-cpyext-releasebuffer - -Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews -keeping objects alive forever (missing decref), and make sure that -bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. - -.. branch: fix-global - -Fix bug (bad reported info) when asked to translate SyntaxWarning to -SyntaxError. - -.. branch: optinfo-into-bridges-3 - -Improve the optimization of branchy Python code by retaining more -information across failing guards. This is done by appending some -carefully encoded extra information into the resume code. - -.. branch: shadowstack-perf-2 - -Two changes that together bring the performance of shadowstack close to -asmgcc---close enough that we can now make shadowstack the default even -on Linux. This should remove a whole class of rare bugs introduced by -asmgcc. - -.. branch: fniephaus/fix-typo-1488123166752 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.7.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-5.7.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-5.7.0.rst @@ -1,6 +1,6 @@ -========================== -What's new in PyPy2.7 5.6+ -========================== +========================= +What's new in PyPy2.7 5.7 +========================= .. this is a revision shortly after release-pypy2.7-v5.6 .. startrev: 7e9787939641 diff --git a/pypy/doc/whatsnew-pypy3-5.7.0.rst b/pypy/doc/whatsnew-pypy3-5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.7.0.rst @@ -0,0 +1,14 @@ +========================= +What's new in PyPy3 5.7.0 +========================= + +.. this is the revision after release-pypy3.3-5.5.x was branched +.. startrev: c5fb5db3c8ee + +.. branch: py3.5-time + +.. branch: py3.5-ssl + +.. branch: PEP393 + +Implement some level of compatibility with PEP 393 APIs. diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -0,0 +1,7 @@ +========================= +What's new in PyPy3 5.7+ +========================= + +.. this is the revision after release-pypy3.3-5.7.x was branched +.. startrev: afbf09453369 + diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h --- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h +++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h @@ -201,10 +201,13 @@ #define BEGIN_MAPPINGS_LIST /* empty */ #define MAPPING_ENCONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, NULL}; #define MAPPING_DECONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, NULL, (void*)enc##_decmap}; #define MAPPING_ENCDEC(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, \ (void*)enc##_decmap}; #define END_MAPPINGS_LIST /* empty */ @@ -294,7 +297,7 @@ #ifdef USING_IMPORTED_MAPS #define USING_IMPORTED_MAP(charset) \ - extern const struct dbcs_map pypy_cjkmap_##charset; + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##charset; #define IMPORT_MAP(locale, charset, encmap, decmap) \ importmap(&pypy_cjkmap_##charset, encmap, decmap) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -609,6 +609,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', ] TYPES = {} FORWARD_DECLS = [] diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1221,3 +1221,25 @@ pass bases = module.foo(C) assert bases == (A, B) + + def test_multiple_inheritance_old_style_base(self): + module = self.import_extension('foo', [ + ("foo", "METH_O", + ''' + PyTypeObject *tp; + tp = (PyTypeObject*)args; + Py_INCREF(tp->tp_bases); + return tp->tp_bases; + ''' + )]) + # used to segfault after some iterations + for i in range(11): + print i + class A(object): + pass + class B: + pass + class C(A, B): + pass + bases = module.foo(C) + assert bases == (A, B) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -931,7 +931,10 @@ if base: inherit_special(space, pto, base) for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): - inherit_slots(space, pto, w_base) + if isinstance(w_base, W_TypeObject): + inherit_slots(space, pto, w_base) + #else: + # w_base is a W_ClassObject, ignore it if not pto.c_tp_setattro: from pypy.module.cpyext.object import PyObject_GenericSetAttr diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py @@ -387,13 +387,14 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; """) C = ffi.dlopen(None) assert C.POS == 1 assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 + assert C.OP == 2 def test_stdcall(): ffi = FFI() diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -349,15 +349,12 @@ # mc.MOV(eax, heap(self.cpu.pos_exception())) mc.TEST_rr(eax.value, eax.value) - mc.J_il8(rx86.Conditions['NZ'], 0) - jnz_location = mc.get_relative_pos() + jnz_location = mc.emit_forward_jump('NZ') # mc.RET() # # patch the JNZ above - offset = mc.get_relative_pos() - jnz_location - assert 0 < offset <= 127 - mc.overwrite(jnz_location-1, chr(offset)) + mc.patch_forward_jump(jnz_location) # From now on this function is basically "merged" with # its caller and so contains DEFAULT_FRAME_BYTES bytes # plus my own return address, which we'll ignore next @@ -834,16 +831,13 @@ ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) mc.CMP_bi(ofs, 0xffffff) # force writing 32 bit stack_check_cmp_ofs = mc.get_relative_pos() - 4 - mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = mc.get_relative_pos() + jg_location = mc.emit_forward_jump('GE') mc.MOV_si(WORD, 0xffffff) # force writing 32 bit ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, store=True) mc.CALL(imm(self._frame_realloc_slowpath)) # patch the JG above - offset = mc.get_relative_pos() - jg_location - assert 0 < offset <= 127 - mc.overwrite(jg_location-1, chr(offset)) + mc.patch_forward_jump(jg_location) self.frame_depth_to_patch.append(stack_check_cmp_ofs) self.frame_depth_to_patch.append(ofs2) @@ -857,16 +851,13 @@ ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) mc.CMP_bi(ofs, 0xffffff) stack_check_cmp_ofs = mc.get_relative_pos() - 4 - mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = mc.get_relative_pos() + jg_location = mc.emit_forward_jump('GE') mc.MOV_rr(edi.value, ebp.value) mc.MOV_ri(esi.value, 0xffffff) ofs2 = mc.get_relative_pos() - 4 mc.CALL(imm(self.cpu.realloc_frame_crash)) # patch the JG above - offset = mc.get_relative_pos() - jg_location - assert 0 < offset <= 127 - mc.overwrite(jg_location-1, chr(offset)) + mc.patch_forward_jump(jg_location) self.frame_depth_to_patch.append(stack_check_cmp_ofs) self.frame_depth_to_patch.append(ofs2) @@ -1002,13 +993,10 @@ self.mc.MOV(eax, heap(endaddr)) # MOV eax, [start] self.mc.SUB(eax, esp) # SUB eax, current self.mc.CMP(eax, heap(lengthaddr)) # CMP eax, [length] - self.mc.J_il8(rx86.Conditions['BE'], 0) # JBE .skip - jb_location = self.mc.get_relative_pos() + jb_location = self.mc.emit_forward_jump('BE')#JBE .skip self.mc.CALL(imm(self.stack_check_slowpath))# CALL slowpath # patch the JB above # .skip: - offset = self.mc.get_relative_pos() - jb_location - assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, chr(offset)) + self.mc.patch_forward_jump(jb_location) # def _call_footer(self): @@ -1242,15 +1230,12 @@ return genop_cmp def _if_parity_clear_zero_and_carry(self): - self.mc.J_il8(rx86.Conditions['NP'], 0) - jnp_location = self.mc.get_relative_pos() + jnp_location = self.mc.emit_forward_jump('NP') # CMP EBP, 0: as EBP cannot be null here, that operation should # always clear zero and carry self.mc.CMP_ri(ebp.value, 0) # patch the JNP above - offset = self.mc.get_relative_pos() - jnp_location - assert 0 < offset <= 127 - self.mc.overwrite(jnp_location-1, chr(offset)) + self.mc.patch_forward_jump(jnp_location) def _cmpop_float(cond, rev_cond): is_ne = cond == 'NE' @@ -1728,10 +1713,11 @@ # jump to jump over this GUARD_NO_EXCEPTION as well, if we can if self._find_nearby_operation(-1).getopnum() in ( rop.COND_CALL, rop.COND_CALL_VALUE_I, rop.COND_CALL_VALUE_R): - jmp_adr = self.previous_cond_call_jcond - offset = self.mc.get_relative_pos() - jmp_adr - if offset <= 127: - self.mc.overwrite(jmp_adr-1, chr(offset)) + j_location = self.previous_cond_call_jcond + try: + self.mc.patch_forward_jump(j_location) + except codebuf.ShortJumpTooFar: + pass # ignore this case def genop_guard_guard_not_invalidated(self, guard_op, guard_token, locs, ign): @@ -1842,13 +1828,10 @@ def genop_guard_guard_nonnull_class(self, guard_op, guard_token, locs, ign): self.mc.CMP(locs[0], imm1) # Patched below - self.mc.J_il8(rx86.Conditions['B'], 0) - jb_location = self.mc.get_relative_pos() + jb_location = self.mc.emit_forward_jump('B') self._cmp_guard_class(locs) # patch the JB above - offset = self.mc.get_relative_pos() - jb_location - assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, chr(offset)) + self.mc.patch_forward_jump(jb_location) # self.guard_success_cc = rx86.Conditions['E'] self.implement_guard(guard_token) @@ -2223,19 +2206,15 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.CMP(mem(eax, ofs), imm(value)) # patched later - self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' - return self.mc.get_relative_pos() + return self.mc.emit_forward_jump('E') # goto B if we get 'done_with_this_frame' def _call_assembler_patch_je(self, result_loc, je_location): if (IS_X86_32 and isinstance(result_loc, FrameLoc) and result_loc.type == FLOAT): self.mc.FSTPL_b(result_loc.value) - self.mc.JMP_l8(0) # jump to done, patched later - jmp_location = self.mc.get_relative_pos() + jmp_location = self.mc.emit_forward_jump_uncond() # jump to the end # - offset = jmp_location - je_location - assert 0 < offset <= 127 - self.mc.overwrite(je_location - 1, chr(offset)) + self.mc.patch_forward_jump(je_location) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) # return jmp_location @@ -2255,9 +2234,7 @@ self.mc.MOV_rm(eax.value, (eax.value, ofs)) def _call_assembler_patch_jmp(self, jmp_location): - offset = self.mc.get_relative_pos() - jmp_location - assert 0 <= offset <= 127 - self.mc.overwrite(jmp_location - 1, chr(offset)) + self.mc.patch_forward_jump(jmp_location) # ------------------- END CALL ASSEMBLER ----------------------- @@ -2288,16 +2265,14 @@ else: loc = addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs) mc.TEST8(loc, imm(mask)) - mc.J_il8(rx86.Conditions['Z'], 0) # patched later - jz_location = mc.get_relative_pos() + jz_location = mc.emit_forward_jump('Z') # patched later # for cond_call_gc_wb_array, also add another fast path: # if GCFLAG_CARDS_SET, then we can just set one bit and be done if card_marking: # GCFLAG_CARDS_SET is in this byte at 0x80, so this fact can - # been checked by the status flags of the previous TEST8 - mc.J_il8(rx86.Conditions['S'], 0) # patched later - js_location = mc.get_relative_pos() + # been checked by the sign flags of the previous TEST8 + js_location = mc.emit_forward_jump('S') # patched later else: js_location = 0 @@ -2326,13 +2301,10 @@ # The helper ends again with a check of the flag in the object. # So here, we can simply write again a 'JNS', which will be # taken if GCFLAG_CARDS_SET is still not set. - mc.J_il8(rx86.Conditions['NS'], 0) # patched later - jns_location = mc.get_relative_pos() + jns_location = mc.emit_forward_jump('NS') # patched later # # patch the JS above - offset = mc.get_relative_pos() - js_location - assert 0 < offset <= 127 - mc.overwrite(js_location-1, chr(offset)) + mc.patch_forward_jump(js_location) # # case GCFLAG_CARDS_SET: emit a few instructions to do # directly the card flag setting @@ -2367,14 +2339,10 @@ raise AssertionError("index is neither RegLoc nor ImmedLoc") # # patch the JNS above - offset = mc.get_relative_pos() - jns_location - assert 0 < offset <= 127 - mc.overwrite(jns_location-1, chr(offset)) + mc.patch_forward_jump(jns_location) # patch the JZ above - offset = mc.get_relative_pos() - jz_location - assert 0 < offset <= 127 - mc.overwrite(jz_location-1, chr(offset)) + mc.patch_forward_jump(jz_location) def genop_discard_cond_call_gc_wb(self, op, arglocs): self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs) @@ -2407,9 +2375,8 @@ def cond_call(self, gcmap, imm_func, arglocs, resloc=None): assert self.guard_success_cc >= 0 - self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0) - # patched later - jmp_adr = self.mc.get_relative_pos() + j_location = self.mc.emit_forward_jump_cond( + rx86.invert_condition(self.guard_success_cc)) self.guard_success_cc = rx86.cond_none # self.push_gcmap(self.mc, gcmap, store=True) @@ -2459,26 +2426,21 @@ v = gpr_reg_mgr_cls.all_reg_indexes[eax.value] self.mc.MOV_rb(eax.value, v * WORD + base_ofs) # - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(j_location) # might be overridden again to skip over the following # guard_no_exception too - self.previous_cond_call_jcond = jmp_adr + self.previous_cond_call_jcond = j_location def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): assert size & (WORD-1) == 0 # must be correctly aligned self.mc.MOV(ecx, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (ecx.value, size)) self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(jna_location) self.mc.MOV(heap(nursery_free_adr), edx) def malloc_cond_varsize_frame(self, nursery_free_adr, nursery_top_adr, @@ -2492,14 +2454,11 @@ else: self.mc.LEA_ra(edx.value, (ecx.value, sizeloc.value, 0, 0)) self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) - offset = self.mc.get_relative_pos() - jmp_adr - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.patch_forward_jump(jna_location) self.mc.MOV(heap(nursery_free_adr), edx) def malloc_cond_varsize(self, kind, nursery_free_adr, nursery_top_adr, @@ -2517,8 +2476,7 @@ varsizeloc = edx self.mc.CMP(varsizeloc, imm(maxlength)) - self.mc.J_il8(rx86.Conditions['A'], 0) # patched later - jmp_adr0 = self.mc.get_relative_pos() + ja_location = self.mc.emit_forward_jump('A') # patched later self.mc.MOV(ecx, heap(nursery_free_adr)) if valid_addressing_size(itemsize): @@ -2542,12 +2500,9 @@ # now edx contains the total size in bytes, rounded up to a multiple # of WORD, plus nursery_free_adr self.mc.CMP(edx, heap(nursery_top_adr)) - self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later - jmp_adr1 = self.mc.get_relative_pos() + jna_location = self.mc.emit_forward_jump('NA') # patched later # - offset = self.mc.get_relative_pos() - jmp_adr0 - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr0-1, chr(offset)) + self.mc.patch_forward_jump(ja_location) # save the gcmap self.push_gcmap(self.mc, gcmap, store=True) if kind == rewrite.FLAG_ARRAY: @@ -2563,21 +2518,16 @@ addr = self.malloc_slowpath_unicode self.mc.MOV(edx, lengthloc) self.mc.CALL(imm(follow_jump(addr))) - self.mc.JMP_l8(0) # jump to done, patched later - jmp_location = self.mc.get_relative_pos() + jmp_location = self.mc.emit_forward_jump_uncond() # jump to later # - offset = self.mc.get_relative_pos() - jmp_adr1 - assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr1-1, chr(offset)) + self.mc.patch_forward_jump(jna_location) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) # write down the tid, but not if it's the result of the CALL self.mc.MOV(mem(ecx, 0), imm(arraydescr.tid)) # while we're at it, this line is not needed if we've done the CALL self.mc.MOV(heap(nursery_free_adr), edx) # - offset = self.mc.get_relative_pos() - jmp_location - assert 0 < offset <= 127 - self.mc.overwrite(jmp_location - 1, chr(offset)) + self.mc.patch_forward_jump(jmp_location) def store_force_descr(self, op, fail_locs, frame_depth): guard_token = self.implement_guard_recovery(op.opnum, diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -341,25 +341,20 @@ # thread. So here we check if the shadowstack pointer # is still the same as before we released the GIL (saved # in 'ebx'), and if not, we fall back to 'reacqgil_addr'. - mc.J_il8(rx86.Conditions['NE'], 0) - jne_location = mc.get_relative_pos() + jne_location = mc.emit_forward_jump('NE') # here, ecx (=old_value) is zero (so rpy_fastgil was in 'released' # state before the XCHG, but the XCHG acquired it by writing 1) rst = gcrootmap.get_root_stack_top_addr() mc = self.mc mc.CMP(ebx, heap(rst)) - mc.J_il8(rx86.Conditions['E'], 0) - je_location = mc.get_relative_pos() + je_location = mc.emit_forward_jump('E') # revert the rpy_fastgil acquired above, so that the # general 'reacqgil_addr' below can acquire it again... mc.MOV(heap(fastgil), ecx) # patch the JNE above - offset = mc.get_relative_pos() - jne_location - assert 0 < offset <= 127 - mc.overwrite(jne_location-1, chr(offset)) + mc.patch_forward_jump(jne_location) else: - mc.J_il8(rx86.Conditions['E'], 0) - je_location = mc.get_relative_pos() + je_location = mc.emit_forward_jump('E') # # Yes, we need to call the reacqgil() function if not self.result_value_saved_early: @@ -374,9 +369,7 @@ self.restore_result_value(save_edx=False) # # patch the JE above - offset = mc.get_relative_pos() - je_location - assert 0 < offset <= 127 - mc.overwrite(je_location-1, chr(offset)) + mc.patch_forward_jump(je_location) # if restore_edx: mc.MOV_rs(edx.value, 12) # restore this diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py --- a/rpython/jit/backend/x86/codebuf.py +++ b/rpython/jit/backend/x86/codebuf.py @@ -1,12 +1,13 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import intmask +from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.debug import have_debug_prints from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from rpython.jit.backend.x86.rx86 import X86_32_CodeBuilder, X86_64_CodeBuilder from rpython.jit.backend.x86.regloc import LocationCodeBuilder from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64, WORD -from rpython.jit.backend.x86 import valgrind +from rpython.jit.backend.x86 import rx86, valgrind # XXX: Seems nasty to change the superclass of MachineCodeBlockWrapper # like this @@ -17,6 +18,9 @@ codebuilder_cls = X86_64_CodeBuilder backend_name = 'x86_64' +class ShortJumpTooFar(Exception): + pass + class MachineCodeBlockWrapper(BlockBuilderMixin, LocationCodeBuilder, @@ -53,3 +57,22 @@ adr[0] = rffi.cast(rffi.INT, intmask(adr[0]) - p) valgrind.discard_translations(addr, self.get_relative_pos()) self._dump(addr, "jit-backend-dump", backend_name) + + @specialize.arg(1) + def emit_forward_jump(self, condition_string): + return self.emit_forward_jump_cond(rx86.Conditions[condition_string]) + + def emit_forward_jump_cond(self, cond): + self.J_il8(cond, 0) + return self.get_relative_pos() + + def emit_forward_jump_uncond(self): + self.JMP_l8(0) + return self.get_relative_pos() + + def patch_forward_jump(self, jcond_location): + offset = self.get_relative_pos() - jcond_location + assert offset >= 0 + if offset > 127: + raise ShortJumpTooFar + self.overwrite(jcond_location-1, chr(offset)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -237,6 +237,8 @@ 'sys/resource.h', 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h', 'signal.h', 'sys/utsname.h', _ptyh] + if sys.platform.startswith('linux'): + includes.append('sys/sysmacros.h') if sys.platform.startswith('freebsd'): includes.append('sys/ttycom.h') libraries = ['util'] diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py --- a/rpython/rlib/rsiphash.py +++ b/rpython/rlib/rsiphash.py From pypy.commits at gmail.com Thu Mar 16 08:44:04 2017 From: pypy.commits at gmail.com (mattip) Date: Thu, 16 Mar 2017 05:44:04 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <58ca8894.da12190a.c8972.5705@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r90722:702d82ab8269 Date: 2017-03-16 14:42 +0200 http://bitbucket.org/pypy/pypy/changeset/702d82ab8269/ Log: merge default into py3.5 diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -803,6 +803,16 @@ "the actual array length in this context" % exprnode.coord.line) # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '+'): + return (self._parse_constant(exprnode.left) + + self._parse_constant(exprnode.right)) + # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '-'): + return (self._parse_constant(exprnode.left) - + self._parse_constant(exprnode.right)) + # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) diff --git a/pypy/doc/config/translation.gcrootfinder.txt b/pypy/doc/config/translation.gcrootfinder.txt --- a/pypy/doc/config/translation.gcrootfinder.txt +++ b/pypy/doc/config/translation.gcrootfinder.txt @@ -9,10 +9,8 @@ - ``--gcrootfinder=asmgcc``: use assembler hackery to find the roots directly from the normal stack. This is a bit faster, but platform specific. It works so far with GCC or MSVC, - on i386 and x86-64. It is tested only on Linux (where it is - the default) so other platforms (as well as MSVC) may need - various fixes before they can be used. + on i386 and x86-64. It is tested only on Linux + so other platforms (as well as MSVC) may need + various fixes before they can be used. Note asmgcc will be deprecated + at some future date, and does not work with clang. -You may have to force the use of the shadowstack root finder if -you are running into troubles or if you insist on translating -PyPy with other compilers like clang. diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -24,6 +24,12 @@ CFFI_ has been updated to 1.10, improving an already great package for interfacing with C. +We now use shadowstack as our default gcrootfinder_ even on Linux. The +alternative, asmgcc, will be deprecated at some future point. While about 3% +slower, shadowstack is much more easily maintained and debuggable. Also, +the performance of shadowstack has been improved in general: this should +close the speed gap between Linux and other platforms. + As always, this release fixed many issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. @@ -47,6 +53,7 @@ .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: project-ideas.html .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html +.. _gcrootfinder: config/translation.gcrootfinder.html What is PyPy? ============= diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h --- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h +++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h @@ -201,10 +201,13 @@ #define BEGIN_MAPPINGS_LIST /* empty */ #define MAPPING_ENCONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, NULL}; #define MAPPING_DECONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, NULL, (void*)enc##_decmap}; #define MAPPING_ENCDEC(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, \ (void*)enc##_decmap}; #define END_MAPPINGS_LIST /* empty */ @@ -294,7 +297,7 @@ #ifdef USING_IMPORTED_MAPS #define USING_IMPORTED_MAP(charset) \ - extern const struct dbcs_map pypy_cjkmap_##charset; + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##charset; #define IMPORT_MAP(locale, charset, encmap, decmap) \ importmap(&pypy_cjkmap_##charset, encmap, decmap) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py @@ -387,13 +387,14 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; """) C = ffi.dlopen(None) assert C.POS == 1 assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 + assert C.OP == 2 def test_stdcall(): ffi = FFI() diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -237,6 +237,8 @@ 'sys/resource.h', 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h', 'signal.h', 'sys/utsname.h', _ptyh] + if sys.platform.startswith('linux'): + includes.append('sys/sysmacros.h') if sys.platform.startswith('freebsd'): includes.append('sys/ttycom.h') libraries = ['util'] diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py --- a/rpython/rlib/rsiphash.py +++ b/rpython/rlib/rsiphash.py @@ -24,14 +24,11 @@ if sys.byteorder == 'little': def _le64toh(x): return x + def _le32toh(x): + return x else: _le64toh = rarithmetic.byteswap - - -class Seed: - k0l = k1l = r_uint64(0) -seed = Seed() - + _le32toh = rarithmetic.byteswap def _decode64(s): return (r_uint64(ord(s[0])) | @@ -43,6 +40,11 @@ r_uint64(ord(s[6])) << 48 | r_uint64(ord(s[7])) << 56) +class Seed: + k0l = k1l = r_uint64(0) +seed = Seed() + + def select_random_seed(s): """'s' is a string of length 16""" seed.k0l = _decode64(s) @@ -177,17 +179,13 @@ """For tests.""" global misaligned_is_fine old = seed.k0l, seed.k1l, misaligned_is_fine - seed.k0l = _le64toh(r_uint64(new_k0)) - seed.k1l = _le64toh(r_uint64(new_k1)) + seed.k0l = r_uint64(new_k0) + seed.k1l = r_uint64(new_k1) if test_misaligned_path: misaligned_is_fine = False yield seed.k0l, seed.k1l, misaligned_is_fine = old -def get_current_seed(): - return _le64toh(seed.k0l), _le64toh(seed.k1l) - - magic0 = r_uint64(0x736f6d6570736575) magic1 = r_uint64(0x646f72616e646f6d) magic2 = r_uint64(0x6c7967656e657261) @@ -271,7 +269,8 @@ size = 4 if size == 4: if direct: - t |= r_uint64(llop.raw_load(rffi.UINT, addr_in, index)) + v = _le32toh(r_uint32(llop.raw_load(rffi.UINT, addr_in, index))) + t |= r_uint64(v) size = 0 else: t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24 @@ -287,7 +286,7 @@ size = 0 assert size == 0 - b |= _le64toh(t) + b |= t v3 ^= b v0, v1, v2, v3 = _double_round(v0, v1, v2, v3) diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h --- a/rpython/rlib/rvmprof/src/vmprof_getpc.h +++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h @@ -131,7 +131,7 @@ // typedef int ucontext_t; // #endif -intptr_t GetPC(ucontext_t *signal_ucontext) { +static intptr_t GetPC(ucontext_t *signal_ucontext) { // RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n"); fprintf(stderr, "GetPC is not yet implemented on Windows\n"); return NULL; @@ -142,7 +142,7 @@ // the right value for your system, and add it to the list in // vmrpof_config.h #else -intptr_t GetPC(ucontext_t *signal_ucontext) { +static intptr_t GetPC(ucontext_t *signal_ucontext) { return signal_ucontext->PC_FROM_UCONTEXT; // defined in config.h } diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h --- a/rpython/rlib/rvmprof/src/vmprof_main.h +++ b/rpython/rlib/rvmprof/src/vmprof_main.h @@ -104,8 +104,8 @@ #include -volatile int spinlock; -jmp_buf restore_point; +static volatile int spinlock; +static jmp_buf restore_point; static void segfault_handler(int arg) { diff --git a/rpython/rlib/test/test_rsiphash.py b/rpython/rlib/test/test_rsiphash.py --- a/rpython/rlib/test/test_rsiphash.py +++ b/rpython/rlib/test/test_rsiphash.py @@ -52,12 +52,12 @@ os.environ['PYTHONHASHSEED'] = '0' initialize_from_env() assert siphash24("foo") == 15988776847138518036 - # value checked with CPython 3.5 + # value checked with CPython 3.5 (turned positive by adding 2**64) os.environ['PYTHONHASHSEED'] = '4000000000' initialize_from_env() assert siphash24("foo") == 13829150778707464258 - # value checked with CPython 3.5 + # value checked with CPython 3.5 (turned positive by adding 2**64) for env in ['', 'random']: os.environ['PYTHONHASHSEED'] = env @@ -118,8 +118,9 @@ 123, 123, intmask(15988776847138518036), 456, 456, intmask(15988776847138518036), 789, 789] - assert s1[8] in [intmask(17593683438421985039), # ucs2 mode - intmask(94801584261658677)] # ucs4 mode + assert s1[8] in [intmask(17593683438421985039), # ucs2 mode little endian + intmask(94801584261658677), # ucs4 mode little endian + intmask(3849431280840015342),] # ucs4 mode big endian os.environ['PYTHONHASHSEED'] = '3987654321' s1 = getall() @@ -127,8 +128,9 @@ 123, 123, intmask(5890804383681474441), 456, 456, intmask(5890804383681474441), 789, 789] - assert s1[8] in [intmask(4192582507672183374), # ucs2 mode - intmask(7179255293164649778)] # ucs4 mode + assert s1[8] in [intmask(4192582507672183374), # ucs2 mode little endian + intmask(7179255293164649778), # ucs4 mode little endian + intmask(-3945781295304514711),] # ucs4 mode big endian for env in ['', 'random']: os.environ['PYTHONHASHSEED'] = env From pypy.commits at gmail.com Thu Mar 16 09:25:34 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 16 Mar 2017 06:25:34 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: merge default Message-ID: <58ca924e.4920190a.115c.57b8@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90724:4a1cac3da487 Date: 2017-03-16 10:19 +0100 http://bitbucket.org/pypy/pypy/changeset/4a1cac3da487/ Log: merge default diff too long, truncating to 2000 out of 2513 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,5 @@ +#encoding utf-8 + License ======= @@ -42,9 +44,9 @@ Antonio Cuni Samuele Pedroni Matti Picus + Ronan Lamy Alex Gaynor Philip Jenvey - Ronan Lamy Brian Kearns Richard Plangger Michael Hudson @@ -55,12 +57,12 @@ Hakan Ardo Benjamin Peterson Anders Chrigstrom + Wim Lavrijsen Eric van Riet Paap - Wim Lavrijsen Richard Emslie Alexander Schremmer + Remi Meier Dan Villiom Podlaski Christiansen - Remi Meier Lukas Diekmann Sven Hager Anders Lehmann @@ -83,8 +85,8 @@ Lawrence Oluyede Bartosz Skowron Daniel Roberts + Adrien Di Mascio Niko Matsakis - Adrien Di Mascio Alexander Hesse Ludovic Aubry Jacob Hallen @@ -100,8 +102,8 @@ Michael Foord Stephan Diehl Stefan Schwarzer + Tomek Meka Valentino Volonghi - Tomek Meka Stefano Rivera Patrick Maupin Devin Jeanpierre @@ -109,268 +111,273 @@ Bruno Gola David Malcolm Jean-Paul Calderone - Timo Paulssen Edd Barrett Squeaky + Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton + Nicolas Truessel Martin Matusiak - Nicolas Truessel + Wenzhu Man Konstantin Lopuhin - Wenzhu Man John Witulski Laurence Tratt + Greg Price Ivan Sichmann Freitas - Greg Price Dario Bertini + Jeremy Thurgood Mark Pearse Simon Cross - Jeremy Thurgood + Tobias Pape Andreas Stührk - Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski + William Leslie + marky1991 + Ilya Osadchiy + Tobias Oberstein Paul deGrandis - Ilya Osadchiy - marky1991 - Tobias Oberstein + Boris Feigin + Taavi Burns Adrian Kuhn - Boris Feigin tav - Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp + Mike Blume Gerald Klix - Mike Blume Oscar Nierstrasz + Rami Chowdhury Stefan H. Muller - Rami Chowdhury + Joannah Nanjekye Eugene Oden + Tim Felgentreff + Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton - Jeff Terrace - Tim Felgentreff Dusty Phillips Lukas Renggli Guenter Jantzen - William Leslie Ned Batchelder + Amit Regmi Anton Gulenko - Amit Regmi - Ben Young + Sergey Matyunin Jasper Schulz + Andrew Chambers Nicolas Chauvat Andrew Durdin - Andrew Chambers - Sergey Matyunin + Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu + Michael Twomey Rocco Moretti Gintautas Miliauskas - Michael Twomey Lucian Branescu Mihaila anatoly techtonik + Karl Bartel Gabriel Lavoie + Jared Grubb Olivier Dormond - Jared Grubb - Karl Bartel Wouter van Heyst + Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina - Sebastian Pawluś - Stuart Williams - Daniel Patrick Aaron Iles Toby Watson + Daniel Patrick + Stuart Williams Antoine Pitrou Christian Hudon + Justas Sadzevicius + Neil Shepperd Michael Cheng - Justas Sadzevicius + Mikael Schönenberg + Stanislaw Halik + Berkin Ilbeyi Gasper Zejn - Neil Shepperd - Stanislaw Halik - Mikael Schönenberg - Berkin Ilbeyi Faye Zhao Elmo Mäntynen - Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani + Jonathan David Riehl Beatrice During Alex Perry + p_zieschang at yahoo.de + Robert Zaremba + Alan McIntyre + Alexander Sedov Vaibhav Sood - Alan McIntyre Reuben Cummings - Alexander Sedov - p_zieschang at yahoo.de Attila Gobi Christopher Pope - Aaron Gallagher + Tristan Arthur + Christian Tismer + Dan Stromberg + Carl Meyer Florin Papa - Christian Tismer - Marc Abramowitz - Dan Stromberg - Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Marc Abramowitz + Arjun Naik + Aaron Gallagher Alexis Daboville - Jens-Uwe Mager - Carl Meyer + Pieter Zieschang Karl Ramm - Pieter Zieschang - Gabriel Lukas Vacek - Kunal Grover - Andrew Dalke + Omer Katz + Jacek Generowicz Sylvain Thenault Jakub Stasiak + Stefan Beyer + Andrew Dalke + Alejandro J. Cura + Vladimir Kryachko + Gabriel + Mark Williams + Kunal Grover Nathan Taylor - Vladimir Kryachko - Omer Katz - Mark Williams - Jacek Generowicz - Alejandro J. Cura + Travis Francis Athougies + Yasir Suhail + Sergey Kishchenko + Martin Blais + Lutz Paelike + Ian Foote + Philipp Rustemeuer + Catalin Gabriel Manciu Jacob Oscarson - Travis Francis Athougies Ryan Gonzalez - Ian Foote Kristjan Valur Jonsson + Lucio Torre + Richard Lancaster + Dan Buch + Lene Wagner + Tomo Cocoa + Alecsandru Patrascu David Lievens Neil Blakey-Milner - Lutz Paelike - Lucio Torre + Henrik Vendelbo Lars Wassermann - Philipp Rustemeuer - Henrik Vendelbo - Richard Lancaster - Yasir Suhail - Dan Buch + Ignas Mikalajunas + Christoph Gerum Miguel de Val Borro Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Alecsandru Patrascu - Christoph Gerum - Martin Blais - Lene Wagner - Catalin Gabriel Manciu - Tomo Cocoa - Kim Jin Su - rafalgalczynski at gmail.com Toni Mattis - Amber Brown + Laurens Van Houtven + Bobby Impollonia + Roberto De Ioris + Jeong YunWon + Christopher Armstrong + Aaron Tubbs + Vasantha Ganesh K + Jason Michalski + Markus Holtermann + Andrew Thompson + Yusei Tahara + Ruochen Huang + Fabio Niephaus + Akira Li + Gustavo Niemeyer + Rafał Gałczyński + Logan Chien Lucas Stadler - Julian Berman - Markus Holtermann roberto at goyle + Matt Bogosian Yury V. Zaytsev - Anna Katrina Dominguez - Bobby Impollonia - Vasantha Ganesh K - Andrew Thompson florinpapa - Yusei Tahara - Aaron Tubbs - Ben Darnell - Roberto De Ioris - Logan Chien - Juan Francisco Cantero Hurtado - Ruochen Huang - Jeong YunWon - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle Anders Sigfridsson Nikolay Zinov - Jason Michalski + rafalgalczynski at gmail.com + Joshua Gilbert + Anna Katrina Dominguez + Kim Jin Su + Amber Brown + Ben Darnell + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Julian Berman + Michael Hudson-Doyle Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer Stephan Busemann - Rafał Gałczyński - Matt Bogosian + Dan Colish timo - Christian Muirhead - Berker Peksag - James Lan Volodymyr Vladymyrov - shoma hosaka - Ben Mather - Niclas Olofsson - Matthew Miller - Rodrigo Araújo + Daniel Neuhäuser + Flavio Percoco halgari - Boglarka Vezer - Chris Pressey - Buck Golemon - Diana Popa - Konrad Delong - Dinu Gherman + Jim Baker Chris Lambacher coolbutuseless at gmail.com + Mike Bayer + Rodrigo Araújo Daniil Yarancev - Jim Baker + OlivierBlanvillain + Jonas Pfannschmidt + Zearin + Andrey Churin Dan Crosta - Nikolaos-Digenis Karagiannis - James Robert - Armin Ronacher - Brett Cannon - Donald Stufft - yrttyr - aliceinwire - OlivierBlanvillain - Dan Sanders - Zooko Wilcox-O Hearn + reubano at gmail.com + Julien Phalip + Roman Podoliaka + Eli Stevens + Boglarka Vezer + PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio - jiaaro - Mads Kiilerich Antony Lee - Jason Madden - Daniel Neuh�user - reubano at gmail.com - Yaroslav Fedevych Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - squeaky - soareschen - Jonas Pfannschmidt - Kurt Griffiths - Mike Bayer - Stefan Marr - Flavio Percoco - Kristoffer Kleine + shoma hosaka + Buck Golemon + JohnDoe + yrttyr Michael Chermside Anna Ravencroft + remarkablerocket + Berker Peksag + Christian Muirhead + soareschen + Matthew Miller + Konrad Delong + Dinu Gherman pizi - remarkablerocket - Andrey Churin - Zearin - Eli Stevens - Tobias Diaz - Julien Phalip - Roman Podoliaka + James Robert + Armin Ronacher + Diana Popa + Mads Kiilerich + Brett Cannon + aliceinwire + Zooko Wilcox-O Hearn + James Lan + jiaaro + Markus Unterwaditzer + Kristoffer Kleine + Graham Markall Dan Loewenherz werat + Niclas Olofsson + Chris Pressey + Tobias Diaz + Nikolaos-Digenis Karagiannis + Kurt Griffiths + Ben Mather + Donald Stufft + Dan Sanders + Jason Madden + Yaroslav Fedevych + Even Wiik Thomassen + Stefan Marr Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -803,6 +803,16 @@ "the actual array length in this context" % exprnode.coord.line) # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '+'): + return (self._parse_constant(exprnode.left) + + self._parse_constant(exprnode.right)) + # + if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and + exprnode.op == '-'): + return (self._parse_constant(exprnode.left) - + self._parse_constant(exprnode.right)) + # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) diff --git a/pypy/doc/config/translation.gcrootfinder.txt b/pypy/doc/config/translation.gcrootfinder.txt --- a/pypy/doc/config/translation.gcrootfinder.txt +++ b/pypy/doc/config/translation.gcrootfinder.txt @@ -9,10 +9,8 @@ - ``--gcrootfinder=asmgcc``: use assembler hackery to find the roots directly from the normal stack. This is a bit faster, but platform specific. It works so far with GCC or MSVC, - on i386 and x86-64. It is tested only on Linux (where it is - the default) so other platforms (as well as MSVC) may need - various fixes before they can be used. + on i386 and x86-64. It is tested only on Linux + so other platforms (as well as MSVC) may need + various fixes before they can be used. Note asmgcc will be deprecated + at some future date, and does not work with clang. -You may have to force the use of the shadowstack root finder if -you are running into troubles or if you insist on translating -PyPy with other compilers like clang. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -11,9 +11,9 @@ Antonio Cuni Samuele Pedroni Matti Picus + Ronan Lamy Alex Gaynor Philip Jenvey - Ronan Lamy Brian Kearns Richard Plangger Michael Hudson @@ -24,12 +24,12 @@ Hakan Ardo Benjamin Peterson Anders Chrigstrom + Wim Lavrijsen Eric van Riet Paap - Wim Lavrijsen Richard Emslie Alexander Schremmer + Remi Meier Dan Villiom Podlaski Christiansen - Remi Meier Lukas Diekmann Sven Hager Anders Lehmann @@ -52,8 +52,8 @@ Lawrence Oluyede Bartosz Skowron Daniel Roberts + Adrien Di Mascio Niko Matsakis - Adrien Di Mascio Alexander Hesse Ludovic Aubry Jacob Hallen @@ -69,8 +69,8 @@ Michael Foord Stephan Diehl Stefan Schwarzer + Tomek Meka Valentino Volonghi - Tomek Meka Stefano Rivera Patrick Maupin Devin Jeanpierre @@ -78,267 +78,270 @@ Bruno Gola David Malcolm Jean-Paul Calderone - Timo Paulssen Edd Barrett Squeaky + Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton + Nicolas Truessel Martin Matusiak - Nicolas Truessel + Wenzhu Man Konstantin Lopuhin - Wenzhu Man John Witulski Laurence Tratt + Greg Price Ivan Sichmann Freitas - Greg Price Dario Bertini + Jeremy Thurgood Mark Pearse Simon Cross - Jeremy Thurgood + Tobias Pape Andreas Stührk - Tobias Pape Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski + William Leslie + marky1991 + Ilya Osadchiy + Tobias Oberstein Paul deGrandis - Ilya Osadchiy - marky1991 - Tobias Oberstein + Boris Feigin + Taavi Burns Adrian Kuhn - Boris Feigin tav - Taavi Burns Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp + Mike Blume Gerald Klix - Mike Blume Oscar Nierstrasz + Rami Chowdhury Stefan H. Muller - Rami Chowdhury + Joannah Nanjekye Eugene Oden + Tim Felgentreff + Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton - Jeff Terrace - Tim Felgentreff Dusty Phillips Lukas Renggli Guenter Jantzen - William Leslie Ned Batchelder + Amit Regmi Anton Gulenko - Amit Regmi - Ben Young + Sergey Matyunin Jasper Schulz + Andrew Chambers Nicolas Chauvat Andrew Durdin - Andrew Chambers - Sergey Matyunin + Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu + Michael Twomey Rocco Moretti Gintautas Miliauskas - Michael Twomey Lucian Branescu Mihaila anatoly techtonik + Karl Bartel Gabriel Lavoie + Jared Grubb Olivier Dormond - Jared Grubb - Karl Bartel Wouter van Heyst + Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina - Sebastian Pawluś - Stuart Williams - Daniel Patrick Aaron Iles Toby Watson + Daniel Patrick + Stuart Williams Antoine Pitrou Christian Hudon + Justas Sadzevicius + Neil Shepperd Michael Cheng - Justas Sadzevicius + Mikael Schönenberg + Stanislaw Halik + Berkin Ilbeyi Gasper Zejn - Neil Shepperd - Stanislaw Halik - Mikael Schönenberg - Berkin Ilbeyi Faye Zhao Elmo Mäntynen - Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani + Jonathan David Riehl Beatrice During Alex Perry + p_zieschang at yahoo.de + Robert Zaremba + Alan McIntyre + Alexander Sedov Vaibhav Sood - Alan McIntyre Reuben Cummings - Alexander Sedov - p_zieschang at yahoo.de Attila Gobi Christopher Pope - Aaron Gallagher + Tristan Arthur + Christian Tismer + Dan Stromberg + Carl Meyer Florin Papa - Christian Tismer - Marc Abramowitz - Dan Stromberg - Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan touilleMan + Marc Abramowitz + Arjun Naik + Aaron Gallagher Alexis Daboville - Jens-Uwe Mager - Carl Meyer + Pieter Zieschang Karl Ramm - Pieter Zieschang - Gabriel Lukas Vacek - Kunal Grover - Andrew Dalke + Omer Katz + Jacek Generowicz Sylvain Thenault Jakub Stasiak + Stefan Beyer + Andrew Dalke + Alejandro J. Cura + Vladimir Kryachko + Gabriel + Mark Williams + Kunal Grover Nathan Taylor - Vladimir Kryachko - Omer Katz - Mark Williams - Jacek Generowicz - Alejandro J. Cura + Travis Francis Athougies + Yasir Suhail + Sergey Kishchenko + Martin Blais + Lutz Paelike + Ian Foote + Philipp Rustemeuer + Catalin Gabriel Manciu Jacob Oscarson - Travis Francis Athougies Ryan Gonzalez - Ian Foote Kristjan Valur Jonsson + Lucio Torre + Richard Lancaster + Dan Buch + Lene Wagner + Tomo Cocoa + Alecsandru Patrascu David Lievens Neil Blakey-Milner - Lutz Paelike - Lucio Torre + Henrik Vendelbo Lars Wassermann - Philipp Rustemeuer - Henrik Vendelbo - Richard Lancaster - Yasir Suhail - Dan Buch + Ignas Mikalajunas + Christoph Gerum Miguel de Val Borro Artur Lisiecki - Sergey Kishchenko - Ignas Mikalajunas - Alecsandru Patrascu - Christoph Gerum - Martin Blais - Lene Wagner - Catalin Gabriel Manciu - Tomo Cocoa - Kim Jin Su - rafalgalczynski at gmail.com Toni Mattis - Amber Brown + Laurens Van Houtven + Bobby Impollonia + Roberto De Ioris + Jeong YunWon + Christopher Armstrong + Aaron Tubbs + Vasantha Ganesh K + Jason Michalski + Markus Holtermann + Andrew Thompson + Yusei Tahara + Ruochen Huang + Fabio Niephaus + Akira Li + Gustavo Niemeyer + Rafał Gałczyński + Logan Chien Lucas Stadler - Julian Berman - Markus Holtermann roberto at goyle + Matt Bogosian Yury V. Zaytsev - Anna Katrina Dominguez - Bobby Impollonia - Vasantha Ganesh K - Andrew Thompson florinpapa - Yusei Tahara - Aaron Tubbs - Ben Darnell - Roberto De Ioris - Logan Chien - Juan Francisco Cantero Hurtado - Ruochen Huang - Jeong YunWon - Godefroid Chappelle - Joshua Gilbert - Dan Colish - Christopher Armstrong - Michael Hudson-Doyle Anders Sigfridsson Nikolay Zinov - Jason Michalski + rafalgalczynski at gmail.com + Joshua Gilbert + Anna Katrina Dominguez + Kim Jin Su + Amber Brown + Ben Darnell + Juan Francisco Cantero Hurtado + Godefroid Chappelle + Julian Berman + Michael Hudson-Doyle Floris Bruynooghe - Laurens Van Houtven - Akira Li - Gustavo Niemeyer Stephan Busemann - Rafał Gałczyński - Matt Bogosian + Dan Colish timo - Christian Muirhead - Berker Peksag - James Lan Volodymyr Vladymyrov - shoma hosaka - Ben Mather - Niclas Olofsson - Matthew Miller - Rodrigo Araújo + Daniel Neuhäuser + Flavio Percoco halgari - Boglarka Vezer - Chris Pressey - Buck Golemon - Diana Popa - Konrad Delong - Dinu Gherman + Jim Baker Chris Lambacher coolbutuseless at gmail.com + Mike Bayer + Rodrigo Araújo Daniil Yarancev - Jim Baker + OlivierBlanvillain + Jonas Pfannschmidt + Zearin + Andrey Churin Dan Crosta - Nikolaos-Digenis Karagiannis - James Robert - Armin Ronacher - Brett Cannon - Donald Stufft - yrttyr - aliceinwire - OlivierBlanvillain - Dan Sanders - Zooko Wilcox-O Hearn + reubano at gmail.com + Julien Phalip + Roman Podoliaka + Eli Stevens + Boglarka Vezer + PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio - jiaaro - Mads Kiilerich Antony Lee - Jason Madden - Daniel Neuhäuser - reubano at gmail.com - Yaroslav Fedevych Jim Hunziker - Markus Unterwaditzer - Even Wiik Thomassen - jbs - squeaky - soareschen - Jonas Pfannschmidt - Kurt Griffiths - Mike Bayer - Stefan Marr - Flavio Percoco - Kristoffer Kleine + shoma hosaka + Buck Golemon + JohnDoe + yrttyr Michael Chermside Anna Ravencroft + remarkablerocket + Berker Peksag + Christian Muirhead + soareschen + Matthew Miller + Konrad Delong + Dinu Gherman pizi - remarkablerocket - Andrey Churin - Zearin - Eli Stevens - Tobias Diaz - Julien Phalip - Roman Podoliaka + James Robert + Armin Ronacher + Diana Popa + Mads Kiilerich + Brett Cannon + aliceinwire + Zooko Wilcox-O Hearn + James Lan + jiaaro + Markus Unterwaditzer + Kristoffer Kleine + Graham Markall Dan Loewenherz werat - - + Niclas Olofsson + Chris Pressey + Tobias Diaz + Nikolaos-Digenis Karagiannis + Kurt Griffiths + Ben Mather + Donald Stufft + Dan Sanders + Jason Madden + Yaroslav Fedevych + Even Wiik Thomassen + Stefan Marr diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v5.7.0.rst release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -53,6 +54,12 @@ release-0.7.0.rst release-0.6 +CPython 3.5 compatible versions +------------------------------- + +.. toctree:: + + release-v5.7.0.rst CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.7.0.rst whatsnew-pypy2-5.6.0.rst whatsnew-pypy2-5.4.0.rst whatsnew-pypy2-5.3.1.rst diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.7.0.rst @@ -0,0 +1,190 @@ +============================================= +PyPy2.7 and PyPy3.5 v5.7 - two in one release +============================================= + +We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7. +This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and +PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version +3.5.3. + +We continue to make incremental improvements to our C-API +compatibility layer (cpyext). PyPy2 can now import and run many c-extension +packages, among the most notable are numpy, cython, and pandas. Performance may +be slower than CPython, especially for frequently-called short C functions. +Please let us know if your use case is slow, we have ideas how to make things +faster but need real-world examples (not micro-benchmarks) of problematic code. + +Work proceeds at a good pace on the PyPy3.5 +version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta +release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and +as `these benchmarks show`_ it already gives a nice speed bump. +We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise +"PyPy3.5" supports the Python 3.5 language). + +CFFI_ has been updated to 1.10, improving an already great package for +interfacing with C. + +We now use shadowstack as our default gcrootfinder_ even on Linux. The +alternative, asmgcc, will be deprecated at some future point. While about 3% +slower, shadowstack is much more easily maintained and debuggable. Also, +the performance of shadowstack has been improved in general: this should +close the speed gap between Linux and other platforms. + +As always, this release fixed many issues and bugs raised by the +growing community of PyPy users. We strongly recommend updating. + +You can download the v5.7 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html +.. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html +.. _gcrootfinder: config/translation.gcrootfinder.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) +============================================================================================= + +See also issues that were resolved_ + +* New features and cleanups + + * update the format of the PYPYLOG file and improvements to vmprof + * improve the consistency of RPython annotation unions + * emit more sysconfig values for downstream cextension packages + * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*, + _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork, + * add translation option --keepgoing to continue after the first AnnotationError + * detect and raise on recreation of a PyPy object from a PyObject during + tp_dealloc + * refactor and clean up poor handling of unicode exposed in work on py3.5 + * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) + * add translation time --disable_entrypoints option for embedding PyPy together + with another RPython VM + * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14 + * support translations with cpyext and the Boehm GC (for special cases like + revdb + * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is + now possible to obtain the address of any readonly object without pinning it + * refactor the initialization code in translating cpyext + * fix ``"".replace("", "x", num)`` to give the same result as CPython + * use a cffi-style C parser to create rffi objects in cpyext, now the + translating python must have cffi available + * add a rpython implementation of siphash24, allow choosing hash algorithm + randomizing the seed + * make ``attach_gdb`` work on Windows (with Visual Studio Debugger) + * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make + available as ``__pypy__.move_to_end`` and, on py3.5, + ``OrderedDict.move_to_end()`` + * remove completely RPython ``space.wrap`` in a major cleanup, differentiate + between ``space.newtext`` and ``space.newbytes`` on py3.5 + * improve shadowstack to where it is now the default in place of asmgcc + +* Bug Fixes + + * any uncaught RPython exception in the interpreter is turned into a + SystemError (rather than a segfault) + * create log files without the executable bit + * disable clock_gettime() on OS/X, since we support 10.11 and it was only + added in 10.12 + * support HAVE_FSTATVFS which was unintentionally always false + * fix user-created C-API heaptype, issue #2434 + * fix PyDict_Update is not actually the same as dict.update + * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute + issue #2446 + * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``, + ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` + * improve support for creating c-extension objects from app-level classes, + filling more slots especially ``tp_new`` and ``tp_dealloc`` + * add rstack.stack_almost_full() and use it to avoid stack overflow due to + the JIT where possible + * fix for ctypes.c_bool returning bool restype issue #2475 + * fix in corner cases with the GIL and C-API functions + +* Performance improvements: + + * clean-ups in the jit optimizeopt + * optimize ``if x is not None: return x`` or ``if x != 0: return x`` + * add ``jit.conditional_call_elidable()``, a way to tell the JIT + "conditonally call this function" returning a result + * try harder to propagate ``can_be_None=False`` information + * add ``rarithmetic.ovfcheck_int32_add/sub/mul`` + * add and use ``rgc.may_ignore_finalizer()``: an optimization hint that makes + the GC stop tracking the object + * replace malloc+memset with a single calloc, useful for large allocations? + * linux: try to implement os.urandom() as the syscall getrandom() if available + * propagate ``debug.ll_assert_not_none()`` through the JIT to reduce number of + guards + * improve the performance of ``PyDict_Next`` + * improve ``dict.pop()`` + * improve the optimization of branchy Python code by retaining more + information across failing guards + * add optimized "zero-copy" path for ``io.FileIO.readinto`` + +Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) +========================================================= + +Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo + +* New features + + * this first PyPy3.5 release implements much, but not all, of Python 3.5.3 + * PEP 456 allowing secure and interchangable hash algorithms + * use cryptography_'s cffi backend for SSL + +* Bug Fixes + + * implement fixes for some CPython issues that arose since the last release + * solve deadlocks in thread locking mechanism + +* Performance improvements: + + * do not create a list whenever descr_new of a bytesobject is called + * + * + * + +.. _resolved: whatsnew-pypy2-5.7.0.html +.. _cryptography: https://cryptography.io +.. _cppyy: cppyy.html + +Please update, and continue to help us make PyPy better. + +Cheers diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -75,9 +75,10 @@ 'Spenser Bauman':['Spenser Andrew Bauman'], 'Raffael Tfirst':['raffael.tfirst at gmail.com'], 'timo':['timo at eistee.fritz.box'], - 'Jasper Schulz':['Jasper.Schulz'], + 'Jasper Schulz':['Jasper.Schulz', 'jbs'], 'Aaron Gallagher':['"Aaron Gallagher'], 'Yasir Suhail':['yasirs'], + 'Squeaky', ['squeaky'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,191 +1,8 @@ ========================== -What's new in PyPy2.7 5.6+ +What's new in PyPy2.7 5.8+ ========================== -.. this is a revision shortly after release-pypy2.7-v5.6 -.. startrev: 7e9787939641 +.. this is a revision shortly after release-pypy2.7-v5.7 +.. startrev: 44f31f6dd39f -Since a while now, PyPy preserves the order of dictionaries and sets. -However, the set literal syntax ``{x, y, z}`` would by mistake build a -set with the opposite order: ``set([z, y, x])``. This has been fixed. -Note that CPython is inconsistent too: in 2.7.12, ``{5, 5.0}`` would be -``set([5.0])``, but in 2.7.trunk it is ``set([5])``. PyPy's behavior -changed in exactly the same way because of this fix. - - -.. branch: rpython-error-to-systemerror - -Any uncaught RPython exception (from a PyPy bug) is turned into an -app-level SystemError. This should improve the lot of users hitting an -uncaught RPython error. - -.. branch: union-side-effects-2 - -Try to improve the consistency of RPython annotation unions. - -.. branch: pytest-2.9.2 - -.. branch: clean-exported-state - -Clean-ups in the jit optimizeopt - -.. branch: conditional_call_value_4 - -Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally -call this function" returning a result. - -.. branch: desc-specialize - -Refactor FunctionDesc.specialize() and related code (RPython annotator). - -.. branch: raw-calloc - -.. branch: issue2446 - -Assign ``tp_doc`` to the new TypeObject's type dictionary ``__doc__`` key -so it will be picked up by app-level objects of that type - -.. branch: cling-support - -Module cppyy now uses cling as its backend (Reflex has been removed). The -user-facing interface and main developer tools (genreflex, selection files, -class loader, etc.) remain the same. A libcppyy_backend.so library is still -needed but is now available through PyPI with pip: PyPy-cppyy-backend. - -The Cling-backend brings support for modern C++ (11, 14, etc.), dynamic -template instantations, and improved integration with CFFI for better -performance. It also provides interactive C++ (and bindings to that). - -.. branch: better-PyDict_Next - -Improve the performance of ``PyDict_Next``. When trying ``PyDict_Next`` on a -typedef dict, the test exposed a problem converting a ``GetSetProperty`` to a -``PyGetSetDescrObject``. The other direction seem to be fully implemented. -This branch made a minimal effort to convert the basic fields to avoid -segfaults, but trying to use the ``PyGetSetDescrObject`` will probably fail. - -.. branch: stdlib-2.7.13 - -Updated the implementation to match CPython 2.7.13 instead of 2.7.13. - -.. branch: issue2444 - -Fix ``PyObject_GetBuffer`` and ``PyMemoryView_GET_BUFFER``, which leaked -memory and held references. Add a finalizer to CPyBuffer, add a -PyMemoryViewObject with a PyBuffer attached so that the call to -``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory. -Properly call ``bf_releasebuffer`` when not ``NULL``. - -.. branch: boehm-rawrefcount - -Support translations of cpyext with the Boehm GC (for special cases like -revdb). - -.. branch: strbuf-as-buffer - -Implement StringBuffer.get_raw_address (missing feature for the buffer protocol). -More generally it is now possible to obtain the address of any object (if it -is readonly) without pinning it. - -.. branch: cpyext-cleanup -.. branch: api_func-refactor - -Refactor cpyext initialisation. - -.. branch: cpyext-from2 - -Fix a test failure introduced by strbuf-as-buffer - -.. branch: cpyext-FromBuffer - -Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to -the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak. - -.. branch: issue2464 - -Give (almost?) all GetSetProperties a valid __objclass__. - -.. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419 -.. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033 - -.. branch: missing-tp_new - -Improve mixing app-level classes in c-extensions, especially if the app-level -class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect -all the method slots to be filled with a function pointer, where app-level will -search up the mro for an appropriate function at runtime. With this branch we -now fill many more slots in the c-extenion type objects. -Also fix for c-extension type that calls ``tp_hash`` during initialization -(str, unicode types), and fix instantiating c-extension types from built-in -classes by enforcing an order of instaniation. - -.. branch: rffi-parser-2 - -rffi structures in cpyext can now be created by parsing simple C headers. -Additionally, the cts object that holds the parsed information can act like -cffi's ffi objects, with the methods cts.cast() and cts.gettype(). - -.. branch: rpython-hash - -Don't freeze hashes in the translated pypy. In practice, that means -that we can now translate PyPy with the option --hash=siphash24 and get -the same hashes as CPython 3.5, which can be randomized (in a -crypographically good way). It is the default in PyPy3. The default of -PyPy2 remains unchanged: there are user programs out there that depend -on constant hashes (or even sometimes on specific hash results). - -.. branch: dict-move-to-end - -Our dicts, which are always ordered, now have an extra "method" for -Python 3.x which moves an item to first or last position. In PyPy 3.5 -it is the standard ``OrderedDict.move_to_end()`` method, but the -behavior is also available on Python 2.x or for the ``dict`` type by -calling ``__pypy__.move_to_end(dict, key, last=True)``. - - -.. branch optinfo-into-bridges-3 - -Improve the optimization of branchy Python code by retaining more information -across failing guards. - - -.. branch: space-newtext - -Internal refactoring of ``space.wrap()``, which is now replaced with -explicitly-typed methods. Notably, there are now ``space.newbytes()`` -and ``space.newtext()``: these two methods are identical on PyPy 2.7 but -not on PyPy 3.x. The latter is used to get an app-level unicode string -by decoding the RPython string, assumed to be utf-8. - -.. branch: space-wrap - -.. branch: fix_bool_restype - -Fix for ``ctypes.c_bool``-returning ctypes functions - -.. branch: fix-cpyext-releasebuffer - -Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews -keeping objects alive forever (missing decref), and make sure that -bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. - -.. branch: fix-global - -Fix bug (bad reported info) when asked to translate SyntaxWarning to -SyntaxError. - -.. branch: optinfo-into-bridges-3 - -Improve the optimization of branchy Python code by retaining more -information across failing guards. This is done by appending some -carefully encoded extra information into the resume code. - -.. branch: shadowstack-perf-2 - -Two changes that together bring the performance of shadowstack close to -asmgcc---close enough that we can now make shadowstack the default even -on Linux. This should remove a whole class of rare bugs introduced by -asmgcc. - -.. branch: fniephaus/fix-typo-1488123166752 diff --git a/pypy/doc/whatsnew-pypy2-5.7.0.rst b/pypy/doc/whatsnew-pypy2-5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.7.0.rst @@ -0,0 +1,191 @@ +========================= +What's new in PyPy2.7 5.7 +========================= + +.. this is a revision shortly after release-pypy2.7-v5.6 +.. startrev: 7e9787939641 + + +Since a while now, PyPy preserves the order of dictionaries and sets. +However, the set literal syntax ``{x, y, z}`` would by mistake build a +set with the opposite order: ``set([z, y, x])``. This has been fixed. +Note that CPython is inconsistent too: in 2.7.12, ``{5, 5.0}`` would be +``set([5.0])``, but in 2.7.trunk it is ``set([5])``. PyPy's behavior +changed in exactly the same way because of this fix. + + +.. branch: rpython-error-to-systemerror + +Any uncaught RPython exception (from a PyPy bug) is turned into an +app-level SystemError. This should improve the lot of users hitting an +uncaught RPython error. + +.. branch: union-side-effects-2 + +Try to improve the consistency of RPython annotation unions. + +.. branch: pytest-2.9.2 + +.. branch: clean-exported-state + +Clean-ups in the jit optimizeopt + +.. branch: conditional_call_value_4 + +Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally +call this function" returning a result. + +.. branch: desc-specialize + +Refactor FunctionDesc.specialize() and related code (RPython annotator). + +.. branch: raw-calloc + +.. branch: issue2446 + +Assign ``tp_doc`` to the new TypeObject's type dictionary ``__doc__`` key +so it will be picked up by app-level objects of that type + +.. branch: cling-support + +Module cppyy now uses cling as its backend (Reflex has been removed). The +user-facing interface and main developer tools (genreflex, selection files, +class loader, etc.) remain the same. A libcppyy_backend.so library is still +needed but is now available through PyPI with pip: PyPy-cppyy-backend. + +The Cling-backend brings support for modern C++ (11, 14, etc.), dynamic +template instantations, and improved integration with CFFI for better +performance. It also provides interactive C++ (and bindings to that). + +.. branch: better-PyDict_Next + +Improve the performance of ``PyDict_Next``. When trying ``PyDict_Next`` on a +typedef dict, the test exposed a problem converting a ``GetSetProperty`` to a +``PyGetSetDescrObject``. The other direction seem to be fully implemented. +This branch made a minimal effort to convert the basic fields to avoid +segfaults, but trying to use the ``PyGetSetDescrObject`` will probably fail. + +.. branch: stdlib-2.7.13 + +Updated the implementation to match CPython 2.7.13 instead of 2.7.13. + +.. branch: issue2444 + +Fix ``PyObject_GetBuffer`` and ``PyMemoryView_GET_BUFFER``, which leaked +memory and held references. Add a finalizer to CPyBuffer, add a +PyMemoryViewObject with a PyBuffer attached so that the call to +``PyMemoryView_GET_BUFFER`` does not leak a PyBuffer-sized piece of memory. +Properly call ``bf_releasebuffer`` when not ``NULL``. + +.. branch: boehm-rawrefcount + +Support translations of cpyext with the Boehm GC (for special cases like +revdb). + +.. branch: strbuf-as-buffer + +Implement StringBuffer.get_raw_address (missing feature for the buffer protocol). +More generally it is now possible to obtain the address of any object (if it +is readonly) without pinning it. + +.. branch: cpyext-cleanup +.. branch: api_func-refactor + +Refactor cpyext initialisation. + +.. branch: cpyext-from2 + +Fix a test failure introduced by strbuf-as-buffer + +.. branch: cpyext-FromBuffer + +Do not recreate the object in PyMemoryView_FromBuffer, rather pass it to +the returned PyMemoryViewObject, to take ownership of it. Fixes a ref leak. + +.. branch: issue2464 + +Give (almost?) all GetSetProperties a valid __objclass__. + +.. branch: TreeStain/fixed-typo-line-29-mostly-to-most-1484469416419 +.. branch: TreeStain/main-lines-changed-in-l77-l83-made-para-1484471558033 + +.. branch: missing-tp_new + +Improve mixing app-level classes in c-extensions, especially if the app-level +class has a ``tp_new`` or ``tp_dealloc``. The issue is that c-extensions expect +all the method slots to be filled with a function pointer, where app-level will +search up the mro for an appropriate function at runtime. With this branch we +now fill many more slots in the c-extenion type objects. +Also fix for c-extension type that calls ``tp_hash`` during initialization +(str, unicode types), and fix instantiating c-extension types from built-in +classes by enforcing an order of instaniation. + +.. branch: rffi-parser-2 + +rffi structures in cpyext can now be created by parsing simple C headers. +Additionally, the cts object that holds the parsed information can act like +cffi's ffi objects, with the methods cts.cast() and cts.gettype(). + +.. branch: rpython-hash + +Don't freeze hashes in the translated pypy. In practice, that means +that we can now translate PyPy with the option --hash=siphash24 and get +the same hashes as CPython 3.5, which can be randomized (in a +crypographically good way). It is the default in PyPy3. The default of +PyPy2 remains unchanged: there are user programs out there that depend +on constant hashes (or even sometimes on specific hash results). + +.. branch: dict-move-to-end + +Our dicts, which are always ordered, now have an extra "method" for +Python 3.x which moves an item to first or last position. In PyPy 3.5 +it is the standard ``OrderedDict.move_to_end()`` method, but the +behavior is also available on Python 2.x or for the ``dict`` type by +calling ``__pypy__.move_to_end(dict, key, last=True)``. + + +.. branch optinfo-into-bridges-3 + +Improve the optimization of branchy Python code by retaining more information +across failing guards. + + +.. branch: space-newtext + +Internal refactoring of ``space.wrap()``, which is now replaced with +explicitly-typed methods. Notably, there are now ``space.newbytes()`` +and ``space.newtext()``: these two methods are identical on PyPy 2.7 but +not on PyPy 3.x. The latter is used to get an app-level unicode string +by decoding the RPython string, assumed to be utf-8. + +.. branch: space-wrap + +.. branch: fix_bool_restype + +Fix for ``ctypes.c_bool``-returning ctypes functions + +.. branch: fix-cpyext-releasebuffer + +Improve handling of the Py3-style buffer slots in cpyext: fix memoryviews +keeping objects alive forever (missing decref), and make sure that +bf_releasebuffer is called when it should, e.g. from PyBuffer_Release. + +.. branch: fix-global + +Fix bug (bad reported info) when asked to translate SyntaxWarning to +SyntaxError. + +.. branch: optinfo-into-bridges-3 + +Improve the optimization of branchy Python code by retaining more +information across failing guards. This is done by appending some +carefully encoded extra information into the resume code. + +.. branch: shadowstack-perf-2 + +Two changes that together bring the performance of shadowstack close to +asmgcc---close enough that we can now make shadowstack the default even +on Linux. This should remove a whole class of rare bugs introduced by +asmgcc. + +.. branch: fniephaus/fix-typo-1488123166752 diff --git a/pypy/doc/whatsnew-pypy3-5.7.0.rst b/pypy/doc/whatsnew-pypy3-5.7.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.7.0.rst @@ -0,0 +1,14 @@ +========================= +What's new in PyPy3 5.7.0 +========================= + +.. this is the revision after release-pypy3.3-5.5.x was branched +.. startrev: c5fb5db3c8ee + +.. branch: py3.5-time + +.. branch: py3.5-ssl + +.. branch: PEP393 + +Implement some level of compatibility with PEP 393 APIs. diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -0,0 +1,7 @@ +========================= +What's new in PyPy3 5.7+ +========================= + +.. this is the revision after release-pypy3.3-5.7.x was branched +.. startrev: afbf09453369 + diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -1866,7 +1866,7 @@ def test_call_with_nested_anonymous_struct(self): import sys if sys.platform == 'win32': - py.test.skip("needs a GCC extension") + skip("needs a GCC extension") ffi, lib = self.prepare(""" struct foo { int a; union { int b, c; }; }; struct foo f(void); @@ -1914,6 +1914,9 @@ "set_source() and not taking a final '...' argument)") def test_call_with_zero_length_field(self): + import sys + if sys.platform == 'win32': + skip("zero-length field not supported by MSVC") ffi, lib = self.prepare(""" struct foo { int a; int x[0]; }; struct foo f(void); @@ -1959,7 +1962,7 @@ def test_call_with_packed_struct(self): import sys if sys.platform == 'win32': - py.test.skip("needs a GCC extension") + skip("needs a GCC extension") ffi, lib = self.prepare(""" struct foo { char y; int x; }; struct foo f(void); diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h --- a/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h +++ b/pypy/module/_multibytecodec/src/cjkcodecs/cjkcodecs.h @@ -201,10 +201,13 @@ #define BEGIN_MAPPINGS_LIST /* empty */ #define MAPPING_ENCONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, NULL}; #define MAPPING_DECONLY(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, NULL, (void*)enc##_decmap}; #define MAPPING_ENCDEC(enc) \ + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##enc; \ const struct dbcs_map pypy_cjkmap_##enc = {#enc, (void*)enc##_encmap, \ (void*)enc##_decmap}; #define END_MAPPINGS_LIST /* empty */ @@ -294,7 +297,7 @@ #ifdef USING_IMPORTED_MAPS #define USING_IMPORTED_MAP(charset) \ - extern const struct dbcs_map pypy_cjkmap_##charset; + RPY_EXTERN const struct dbcs_map pypy_cjkmap_##charset; #define IMPORT_MAP(locale, charset, encmap, decmap) \ importmap(&pypy_cjkmap_##charset, encmap, decmap) diff --git a/pypy/module/_rawffi/alt/test/test_funcptr.py b/pypy/module/_rawffi/alt/test/test_funcptr.py --- a/pypy/module/_rawffi/alt/test/test_funcptr.py +++ b/pypy/module/_rawffi/alt/test/test_funcptr.py @@ -32,7 +32,10 @@ # c_file.write(py.code.Source('\n'.join(snippets))) eci = ExternalCompilationInfo(include_dirs=[cdir]) - return str(platform.compile([c_file], eci, 'x', standalone=False)) + # Windows note: can't reuse the same file name 'x.dll', because + # the previous one is likely still opened + return str(platform.compile([c_file], eci, 'x' + cls.__name__, + standalone=False)) def setup_class(cls): space = cls.space diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -609,6 +609,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', ] TYPES = {} FORWARD_DECLS = [] diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.13" /* PyPy version as a string */ -#define PYPY_VERSION "5.7.0-alpha0" -#define PYPY_VERSION_NUM 0x05070000 +#define PYPY_VERSION "5.8.0-alpha0" +#define PYPY_VERSION_NUM 0x05080000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py --- a/pypy/module/cpyext/test/test_pystate.py +++ b/pypy/module/cpyext/test/test_pystate.py @@ -73,6 +73,7 @@ ("dance", "METH_NOARGS", """ PyThreadState *old_tstate, *new_tstate; + PyObject *d; PyEval_InitThreads(); @@ -81,7 +82,7 @@ return PyLong_FromLong(0); } - PyObject* d = PyThreadState_GetDict(); /* fails on cpython */ + d = PyThreadState_GetDict(); /* fails on cpython */ if (d != NULL) { return PyLong_FromLong(1); } @@ -144,6 +145,9 @@ ("bounce", "METH_NOARGS", """ PyThreadState * tstate; + PyObject *dict; + PyGILState_STATE gilstate; + if (PyEval_ThreadsInitialized() == 0) { PyEval_InitThreads(); @@ -152,11 +156,11 @@ if (tstate == NULL) { return PyLong_FromLong(0); } - PyObject* dict = PyThreadState_GetDict(); + dict = PyThreadState_GetDict(); if (dict != NULL) { return PyLong_FromLong(1); } - PyGILState_STATE gilstate = PyGILState_Ensure(); + gilstate = PyGILState_Ensure(); dict = PyThreadState_GetDict(); if (dict == NULL) { return PyLong_FromLong(2); diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -154,5 +154,10 @@ def test_tuple_subclass(self): module = self.import_module(name='foo') - a = module.TupleLike([1, 2, 3]) + a = module.TupleLike(range(100, 400, 100)) assert module.is_TupleLike(a) == 1 + assert isinstance(a, tuple) + assert issubclass(type(a), tuple) + assert list(a) == range(100, 400, 100) + assert list(a) == range(100, 400, 100) + assert list(a) == range(100, 400, 100) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1221,3 +1221,25 @@ pass bases = module.foo(C) assert bases == (A, B) + + def test_multiple_inheritance_old_style_base(self): + module = self.import_extension('foo', [ + ("foo", "METH_O", + ''' + PyTypeObject *tp; + tp = (PyTypeObject*)args; + Py_INCREF(tp->tp_bases); + return tp->tp_bases; + ''' + )]) + # used to segfault after some iterations + for i in range(11): + print i + class A(object): + pass + class B: + pass + class C(A, B): + pass + bases = module.foo(C) + assert bases == (A, B) diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py --- a/pypy/module/cpyext/test/test_userslots.py +++ b/pypy/module/cpyext/test/test_userslots.py @@ -160,8 +160,9 @@ }; ''', more_init=''' PyObject * mod = PyImport_ImportModule("datetime"); + PyObject * dt; if (mod == NULL) INITERROR; - PyObject * dt = PyString_FromString("datetime"); + dt = PyString_FromString("datetime"); datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod, dt); if (datetime_cls == NULL) INITERROR; _Timestamp.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -418,8 +418,8 @@ def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way - # (minimally, if tp_basicsize is zero we copy it from the base) - if not pto.c_tp_basicsize: + # (minimally, if tp_basicsize is zero or too low, we copy it from the base) + if pto.c_tp_basicsize < base_pto.c_tp_basicsize: pto.c_tp_basicsize = base_pto.c_tp_basicsize if pto.c_tp_itemsize < base_pto.c_tp_itemsize: pto.c_tp_itemsize = base_pto.c_tp_itemsize @@ -931,7 +931,10 @@ if base: inherit_special(space, pto, base) for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)): - inherit_slots(space, pto, w_base) + if isinstance(w_base, W_TypeObject): + inherit_slots(space, pto, w_base) + #else: + # w_base is a W_ClassObject, ignore it if not pto.c_tp_setattro: from pypy.module.cpyext.object import PyObject_GenericSetAttr diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 7, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 8, 0, "alpha", 0) #XXX # sync patchlevel.h import pypy diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py @@ -387,13 +387,14 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1}; + enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; """) C = ffi.dlopen(None) assert C.POS == 1 assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 + assert C.OP == 2 def test_stdcall(): ffi = FFI() diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -349,15 +349,12 @@ # mc.MOV(eax, heap(self.cpu.pos_exception())) mc.TEST_rr(eax.value, eax.value) - mc.J_il8(rx86.Conditions['NZ'], 0) - jnz_location = mc.get_relative_pos() + jnz_location = mc.emit_forward_jump('NZ') # mc.RET() # # patch the JNZ above - offset = mc.get_relative_pos() - jnz_location - assert 0 < offset <= 127 - mc.overwrite(jnz_location-1, chr(offset)) + mc.patch_forward_jump(jnz_location) # From now on this function is basically "merged" with # its caller and so contains DEFAULT_FRAME_BYTES bytes # plus my own return address, which we'll ignore next @@ -834,16 +831,13 @@ ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) mc.CMP_bi(ofs, 0xffffff) # force writing 32 bit stack_check_cmp_ofs = mc.get_relative_pos() - 4 - mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = mc.get_relative_pos() + jg_location = mc.emit_forward_jump('GE') mc.MOV_si(WORD, 0xffffff) # force writing 32 bit ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, store=True) mc.CALL(imm(self._frame_realloc_slowpath)) # patch the JG above - offset = mc.get_relative_pos() - jg_location - assert 0 < offset <= 127 - mc.overwrite(jg_location-1, chr(offset)) + mc.patch_forward_jump(jg_location) self.frame_depth_to_patch.append(stack_check_cmp_ofs) self.frame_depth_to_patch.append(ofs2) @@ -857,16 +851,13 @@ ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) mc.CMP_bi(ofs, 0xffffff) stack_check_cmp_ofs = mc.get_relative_pos() - 4 - mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = mc.get_relative_pos() + jg_location = mc.emit_forward_jump('GE') mc.MOV_rr(edi.value, ebp.value) mc.MOV_ri(esi.value, 0xffffff) ofs2 = mc.get_relative_pos() - 4 mc.CALL(imm(self.cpu.realloc_frame_crash)) # patch the JG above - offset = mc.get_relative_pos() - jg_location - assert 0 < offset <= 127 - mc.overwrite(jg_location-1, chr(offset)) + mc.patch_forward_jump(jg_location) self.frame_depth_to_patch.append(stack_check_cmp_ofs) self.frame_depth_to_patch.append(ofs2) @@ -1002,13 +993,10 @@ self.mc.MOV(eax, heap(endaddr)) # MOV eax, [start] self.mc.SUB(eax, esp) # SUB eax, current self.mc.CMP(eax, heap(lengthaddr)) # CMP eax, [length] - self.mc.J_il8(rx86.Conditions['BE'], 0) # JBE .skip - jb_location = self.mc.get_relative_pos() + jb_location = self.mc.emit_forward_jump('BE')#JBE .skip self.mc.CALL(imm(self.stack_check_slowpath))# CALL slowpath # patch the JB above # .skip: - offset = self.mc.get_relative_pos() - jb_location - assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, chr(offset)) + self.mc.patch_forward_jump(jb_location) # def _call_footer(self): @@ -1242,15 +1230,12 @@ return genop_cmp def _if_parity_clear_zero_and_carry(self): - self.mc.J_il8(rx86.Conditions['NP'], 0) - jnp_location = self.mc.get_relative_pos() + jnp_location = self.mc.emit_forward_jump('NP') # CMP EBP, 0: as EBP cannot be null here, that operation should # always clear zero and carry self.mc.CMP_ri(ebp.value, 0) # patch the JNP above - offset = self.mc.get_relative_pos() - jnp_location - assert 0 < offset <= 127 - self.mc.overwrite(jnp_location-1, chr(offset)) + self.mc.patch_forward_jump(jnp_location) def _cmpop_float(cond, rev_cond): is_ne = cond == 'NE' @@ -1728,10 +1713,11 @@ # jump to jump over this GUARD_NO_EXCEPTION as well, if we can From pypy.commits at gmail.com Thu Mar 16 10:30:09 2017 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 16 Mar 2017 07:30:09 -0700 (PDT) Subject: [pypy-commit] pypy default: two tweaks Message-ID: <58caa171.0163190a.24436.59f6@mx.google.com> Author: Carl Friedrich Bolz Branch: Changeset: r90725:711b307e45d7 Date: 2017-03-16 15:29 +0100 http://bitbucket.org/pypy/pypy/changeset/711b307e45d7/ Log: two tweaks diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -8,15 +8,15 @@ 3.5.3. We continue to make incremental improvements to our C-API -compatibility layer (cpyext). PyPy2 can now import and run many c-extension -packages, among the most notable are numpy, cython, and pandas. Performance may +compatibility layer (cpyext). PyPy2 can now import and run many C-extension +packages, among the most notable are Numpy, Cython, and Pandas. Performance may be slower than CPython, especially for frequently-called short C functions. Please let us know if your use case is slow, we have ideas how to make things faster but need real-world examples (not micro-benchmarks) of problematic code. Work proceeds at a good pace on the PyPy3.5 version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta -release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and +release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and as `these benchmarks show`_ it already gives a nice speed bump. We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise "PyPy3.5" supports the Python 3.5 language). From pypy.commits at gmail.com Thu Mar 16 11:37:03 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 16 Mar 2017 08:37:03 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: do not link to libunwind, it is now dynamically loaded Message-ID: <58cab11f.0da5190a.fdfa1.5cc4@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90726:92ab51dfa540 Date: 2017-03-16 16:36 +0100 http://bitbucket.org/pypy/pypy/changeset/92ab51dfa540/ Log: do not link to libunwind, it is now dynamically loaded diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -17,7 +17,7 @@ SHARED = SRC.join('shared') BACKTRACE = SHARED.join('libbacktrace') -compile_extra = ['-DRPYTHON_VMPROF', '-g', '-O1'] +compile_extra = ['-DRPYTHON_VMPROF', '-O3'] if sys.platform.startswith('linux'): separate_module_files = [ BACKTRACE.join('backtrace.c'), @@ -30,7 +30,7 @@ BACKTRACE.join('posix.c'), BACKTRACE.join('sort.c'), ] - _libs = ['dl', 'unwind'] + _libs = ['dl'] compile_extra += ['-DVMPROF_UNIX'] compile_extra += ['-DVMPROF_LINUX'] elif sys.platform == 'darwin': From pypy.commits at gmail.com Thu Mar 16 12:01:10 2017 From: pypy.commits at gmail.com (mattip) Date: Thu, 16 Mar 2017 09:01:10 -0700 (PDT) Subject: [pypy-commit] pypy default: emphasize that pypy3.5 release is linux-64 only Message-ID: <58cab6c6.5e502e0a.1eee0.3fd9@mx.google.com> Author: Matti Picus Branch: Changeset: r90727:02b4b72d54e6 Date: 2017-03-16 18:00 +0200 http://bitbucket.org/pypy/pypy/changeset/02b4b72d54e6/ Log: emphasize that pypy3.5 release is linux-64 only diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -2,7 +2,7 @@ PyPy2.7 and PyPy3.5 v5.7 - two in one release ============================================= -We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7. +We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7 for 64 bit linux. This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version 3.5.3. @@ -65,7 +65,7 @@ We also welcome developers of other `dynamic languages`_ to see what RPython can do for them. -This release supports: +The PyPy 2.7 release supports: * **x86** machines on most common operating systems (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) @@ -80,7 +80,7 @@ .. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) -============================================================================================= +===================================================================================== See also issues that were resolved_ From pypy.commits at gmail.com Thu Mar 16 13:12:55 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 16 Mar 2017 10:12:55 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: remove test dynamic, code was disabled and removed from rvmprof Message-ID: <58cac797.84202e0a.5c9f.63dc@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90728:562ce8f6798e Date: 2017-03-16 18:12 +0100 http://bitbucket.org/pypy/pypy/changeset/562ce8f6798e/ Log: remove test dynamic, code was disabled and removed from rvmprof diff --git a/rpython/rlib/rvmprof/test/test_dynamic.py b/rpython/rlib/rvmprof/test/test_dynamic.py deleted file mode 100644 --- a/rpython/rlib/rvmprof/test/test_dynamic.py +++ /dev/null @@ -1,72 +0,0 @@ -import py -import sys -try: - import cffi -except ImportError: - py.test.skip('cffi required') - -from rpython.rlib import rvmprof -srcdir = py.path.local(rvmprof.__file__).join("..", "src") - - - at py.test.mark.skipif("sys.platform == 'win32'") -class TestDirect(object): - def setup_class(clz): - ffi = cffi.FFI() - ffi.cdef(""" - int vmp_dyn_register_jit_page(intptr_t addr, intptr_t end_addr, const char * name); - int vmp_dyn_cancel(int ref); - int vmp_dyn_teardown(void); - """) - - with open(str(srcdir.join("shared/vmp_dynamic.c"))) as fd: - ffi.set_source("rpython.rlib.rvmprof.test._test_dynamic", fd.read(), - include_dirs=[str(srcdir), str(srcdir.join('shared'))], - libraries=['unwind']) - - ffi.compile(verbose=True) - - from rpython.rlib.rvmprof.test import _test_dynamic - clz.lib = _test_dynamic.lib - clz.ffi = _test_dynamic.ffi - - def test_register_dynamic_code(self): - lib = self.lib - ffi = self.ffi - - assert 1 == lib.vmp_dyn_cancel(100) - assert 1 == lib.vmp_dyn_cancel(0) - assert 1 == lib.vmp_dyn_cancel(-1) - - s = ffi.new("char[]", "hello jit compiler") - assert 0 == lib.vmp_dyn_register_jit_page(0x100, 0x200, ffi.NULL) - assert 1 == lib.vmp_dyn_register_jit_page(0x200, 0x300, s) - - lib.vmp_dyn_cancel(0) - lib.vmp_dyn_cancel(1) - - def test_register_dynamic_code_many(self): - lib = self.lib - ffi = self.ffi - - refs = [] - for i in range(5000): - ref = lib.vmp_dyn_register_jit_page(0x100*i, 0x200*i, ffi.NULL) - refs.append(ref) - - for i in range(1000, 4000): - ref = refs[i] - lib.vmp_dyn_cancel(ref) - - refs = refs[:1000] + refs[4000:] - - for i in range(5000): - ref = lib.vmp_dyn_register_jit_page(0x100*i, 0x200*i, ffi.NULL) - refs.append(ref) - - while refs: - ref = refs.pop() - lib.vmp_dyn_cancel(ref) - - lib.vmp_dyn_teardown() - From pypy.commits at gmail.com Thu Mar 16 13:18:29 2017 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 16 Mar 2017 10:18:29 -0700 (PDT) Subject: [pypy-commit] pypy default: add some links, improve formatting, and factor out the RPython changes (since Message-ID: <58cac8e5.5ed7190a.4612.6842@mx.google.com> Author: Carl Friedrich Bolz Branch: Changeset: r90729:3ecf4b9db0f5 Date: 2017-03-16 18:10 +0100 http://bitbucket.org/pypy/pypy/changeset/3ecf4b9db0f5/ Log: add some links, improve formatting, and factor out the RPython changes (since they are often not so relevant to PyPy users) diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -87,57 +87,52 @@ * New features and cleanups * update the format of the PYPYLOG file and improvements to vmprof - * improve the consistency of RPython annotation unions * emit more sysconfig values for downstream cextension packages - * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*, - _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork, - * add translation option --keepgoing to continue after the first AnnotationError + * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``, + ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``, * detect and raise on recreation of a PyPy object from a PyObject during tp_dealloc * refactor and clean up poor handling of unicode exposed in work on py3.5 - * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) - * add translation time --disable_entrypoints option for embedding PyPy together - with another RPython VM - * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14 + * builtin module cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) + * adapt ``weakref`` according to CPython issue #19542_, will be in CPython 2.7.14 * support translations with cpyext and the Boehm GC (for special cases like - revdb + RevDB_ * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is now possible to obtain the address of any readonly object without pinning it * refactor the initialization code in translating cpyext - * fix ``"".replace("", "x", num)`` to give the same result as CPython - * use a cffi-style C parser to create rffi objects in cpyext, now the - translating python must have cffi available - * add a rpython implementation of siphash24, allow choosing hash algorithm - randomizing the seed - * make ``attach_gdb`` work on Windows (with Visual Studio Debugger) + * make ``__pypy__.attach_gdb`` work on Windows (with Visual Studio Debugger) * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make - available as ``__pypy__.move_to_end`` and, on py3.5, + available as ``__pypy__.move_to_end`` and, on Py3.5, ``OrderedDict.move_to_end()`` * remove completely RPython ``space.wrap`` in a major cleanup, differentiate between ``space.newtext`` and ``space.newbytes`` on py3.5 - * improve shadowstack to where it is now the default in place of asmgcc + * any uncaught RPython exception in the interpreter is turned into a + SystemError (rather than a segfault) + * add translation time --disable_entrypoints option for embedding PyPy together + with another RPython VM + * Bug Fixes - * any uncaught RPython exception in the interpreter is turned into a - SystemError (rather than a segfault) + * fix ``"".replace("", "x", num)`` to give the same result as CPython * create log files without the executable bit - * disable clock_gettime() on OS/X, since we support 10.11 and it was only + * disable ``clock_gettime()`` on OS/X, since we support 10.11 and it was only added in 10.12 - * support HAVE_FSTATVFS which was unintentionally always false - * fix user-created C-API heaptype, issue #2434 - * fix PyDict_Update is not actually the same as dict.update - * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute - issue #2446 + * support ``HAVE_FSTATVFS`` which was unintentionally always false + * fix user-created C-API heaptype, issue #2434_ + * fix ``PyDict_Update`` is not actually the same as ``dict.update`` + * assign ``tp_doc`` on ``PyTypeObject`` and tie it to the app-level ``__doc__`` attribute + issue #2446_ * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``, ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` - * improve support for creating c-extension objects from app-level classes, - filling more slots especially ``tp_new`` and ``tp_dealloc`` + * improve support for creating C-extension objects from app-level classes, + filling more slots, especially ``tp_new`` and ``tp_dealloc`` * add rstack.stack_almost_full() and use it to avoid stack overflow due to the JIT where possible - * fix for ctypes.c_bool returning bool restype issue #2475 + * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_ * fix in corner cases with the GIL and C-API functions + * Performance improvements: * clean-ups in the jit optimizeopt @@ -158,10 +153,20 @@ information across failing guards * add optimized "zero-copy" path for ``io.FileIO.readinto`` +* RPython improvements + + * improve the consistency of RPython annotation unions + * add translation option --keepgoing to continue after the first AnnotationError + * use a cffi-style C parser to create rffi objects in cpyext, now the + translating Python must have cffi available + * improve shadowstack to where it is now the default in place of asmgcc + * add a rpython implementation of siphash24, allow choosing hash algorithm + randomizing the seed + Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) -========================================================= +========================================================================== -Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo +Development moved from the py3k branch to the py3.5 branch in the PyPy bitbucket repo * New features @@ -176,12 +181,17 @@ * Performance improvements: - * do not create a list whenever descr_new of a bytesobject is called + * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called * * * .. _resolved: whatsnew-pypy2-5.7.0.html +.. _19542: https://bugs.python.org/issue19542 +.. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys +.. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on +.. _2475: https://bitbucket.org/pypy/pypy/issues/2475 +.. _RevDB: https://bitbucket.org/pypy/revdb .. _cryptography: https://cryptography.io .. _cppyy: cppyy.html From pypy.commits at gmail.com Thu Mar 16 13:30:49 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 16 Mar 2017 10:30:49 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: missing folder for include Message-ID: <58cacbc9.421a190a.5e06a.65e6@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90730:a21fcad99e6d Date: 2017-03-16 18:30 +0100 http://bitbucket.org/pypy/pypy/changeset/a21fcad99e6d/ Log: missing folder for include diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,6 +1,6 @@ #pragma once -#include "vmprof.h" +#include "shared/vmprof.h" #define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int)) From pypy.commits at gmail.com Thu Mar 16 13:35:45 2017 From: pypy.commits at gmail.com (plan_rich) Date: Thu, 16 Mar 2017 10:35:45 -0700 (PDT) Subject: [pypy-commit] pypy vmprof-native: remove more unused methods, revert changes to gcc/trackgcroot which is not relevant anymore Message-ID: <58caccf1.6911190a.fbb3e.63d6@mx.google.com> Author: Richard Plangger Branch: vmprof-native Changeset: r90731:0c5935077701 Date: 2017-03-16 18:35 +0100 http://bitbucket.org/pypy/pypy/changeset/0c5935077701/ Log: remove more unused methods, revert changes to gcc/trackgcroot which is not relevant anymore diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -247,26 +247,6 @@ return decorate -def dyn_register_jit_page(token, addr, end_addr): - try: - c = _get_vmprof().cintf - cname = lltype.nullptr(rffi.CCHARP.TO) - ref = c.vmp_dyn_register_jit_page(addr, end_addr, cname) - token.rvmprof_register(ref) - return True - except cintf.VMProfPlatformUnsupported: - return False - -def dyn_cancel(token): - try: - c = _get_vmprof().cintf - for ref in token._rvmprof_references: - c.vmp_dyn_cancel(ref) - token._rvmprof_references = [] - return True - except cintf.VMProfPlatformUnsupported: - return False - @specialize.memo() def _was_registered(CodeClass): return hasattr(CodeClass, '_vmprof_unique_id') diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py --- a/rpython/translator/c/gcc/trackgcroot.py +++ b/rpython/translator/c/gcc/trackgcroot.py @@ -368,8 +368,8 @@ def schedule(insn, state): for previnsn in insn.previous_insns: key = previnsn, state - if previnsn not in seen: - seen.add(previnsn) + if key not in seen: + seen.add(key) pending.append(key) schedule(initial_insn, initial_state) while pending: From pypy.commits at gmail.com Thu Mar 16 18:01:43 2017 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 16 Mar 2017 15:01:43 -0700 (PDT) Subject: [pypy-commit] pypy default: start listing the unimplemented things in pypy3 Message-ID: <58cb0b47.4ad9190a.ea4fc.6e47@mx.google.com> Author: Carl Friedrich Bolz Branch: Changeset: r90732:62778ea2a449 Date: 2017-03-16 23:01 +0100 http://bitbucket.org/pypy/pypy/changeset/62778ea2a449/ Log: start listing the unimplemented things in pypy3 diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -100,7 +100,6 @@ * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is now possible to obtain the address of any readonly object without pinning it * refactor the initialization code in translating cpyext - * make ``__pypy__.attach_gdb`` work on Windows (with Visual Studio Debugger) * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make available as ``__pypy__.move_to_end`` and, on Py3.5, ``OrderedDict.move_to_end()`` @@ -166,14 +165,15 @@ Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) ========================================================================== -Development moved from the py3k branch to the py3.5 branch in the PyPy bitbucket repo +Development moved from the py3k branch to the py3.5 branch in the PyPy bitbucket repo. * New features - * this first PyPy3.5 release implements much, but not all, of Python 3.5.3 + * this first PyPy3.5 release implements most of Python 3.5.3, exceptions are listed below * PEP 456 allowing secure and interchangable hash algorithms * use cryptography_'s cffi backend for SSL + * Bug Fixes * implement fixes for some CPython issues that arose since the last release @@ -186,6 +186,12 @@ * * +* The following features of Python 3.5 are not implemented yet in PyPy: + + * PEP 442: Safe object finalization + * PEP 489: Multi-phase extension module initialization + * XXX what else? + .. _resolved: whatsnew-pypy2-5.7.0.html .. _19542: https://bugs.python.org/issue19542 .. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys From pypy.commits at gmail.com Thu Mar 16 18:16:29 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 16 Mar 2017 15:16:29 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Skip test_importlib.frozen package from lib-python test suite. Message-ID: <58cb0ebd.110f2e0a.c307.6ced@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90733:f6376456fff5 Date: 2017-03-16 23:13 +0100 http://bitbucket.org/pypy/pypy/changeset/f6376456fff5/ Log: Skip test_importlib.frozen package from lib-python test suite. PyPy doesn't have frozen modules (in the sense of CPython's). diff --git a/lib-python/3/test/test_importlib/frozen/__init__.py b/lib-python/3/test/test_importlib/frozen/__init__.py --- a/lib-python/3/test/test_importlib/frozen/__init__.py +++ b/lib-python/3/test/test_importlib/frozen/__init__.py @@ -1,5 +1,10 @@ import os -from test.support import load_package_tests +import unittest +from test.support import load_package_tests, check_impl_detail def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) + + +if check_impl_detail(pypy=True): + raise unittest.SkipTest("PyPy doesn't have frozen modules") From pypy.commits at gmail.com Thu Mar 16 18:50:20 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 16 Mar 2017 15:50:20 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Call init_extra_module_attrs() for MixedModules. Message-ID: <58cb16ac.6718190a.bccf0.6dda@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90734:e311a2caebc8 Date: 2017-03-16 23:49 +0100 http://bitbucket.org/pypy/pypy/changeset/e311a2caebc8/ Log: Call init_extra_module_attrs() for MixedModules. This increases compatibilty with CPython, as this sets some module attributes to None, like on CPython. diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -1,4 +1,4 @@ -from pypy.interpreter.module import Module +from pypy.interpreter.module import Module, init_extra_module_attrs from pypy.interpreter.function import Function, BuiltinFunction from pypy.interpreter import gateway from pypy.interpreter.error import OperationError @@ -18,6 +18,7 @@ def __init__(self, space, w_name): """ NOT_RPYTHON """ Module.__init__(self, space, w_name) + init_extra_module_attrs(space, self) self.lazy = True self.lazy_initial_values_w = {} self.__class__.buildloaders() From pypy.commits at gmail.com Thu Mar 16 20:50:18 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 16 Mar 2017 17:50:18 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix nested namespace packages in zipimport. Message-ID: <58cb32ca.51a0190a.99993.70b4@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90735:9068bed7f373 Date: 2017-03-17 01:49 +0100 http://bitbucket.org/pypy/pypy/changeset/9068bed7f373/ Log: Fix nested namespace packages in zipimport. diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py --- a/pypy/module/zipimport/interp_zipimport.py +++ b/pypy/module/zipimport/interp_zipimport.py @@ -373,7 +373,7 @@ if self.have_modulefile(space, filename + ext): return True, None # See if this is a directory (part of a namespace pkg) - dirpath = self.prefix + fullname + dirpath = self.prefix + fullname.split(".")[-1] if self.have_modulefile(space, dirpath + ZIPSEP): return True, self.filename + os.path.sep + self.corr_zname(dirpath) return False, None diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py --- a/pypy/module/zipimport/test/test_zipimport.py +++ b/pypy/module/zipimport/test/test_zipimport.py @@ -446,6 +446,13 @@ foo = __import__('foo.one', None, None, []) assert foo.one.attr == 'portion1 foo one' + def test_namespace_pkg_nested(self): + self.writefile('foo/', '') + self.writefile('foo/bar/', '') + self.writefile('foo/bar/one.py', "attr = 'portion1 foo one'\n") + foo = __import__('foo.bar.one', None, None, []) + assert foo.bar.one.attr == 'portion1 foo one' + if os.sep != '/': class AppTestNativePathSep(AppTestZipimport): From pypy.commits at gmail.com Thu Mar 16 21:25:18 2017 From: pypy.commits at gmail.com (rlamy) Date: Thu, 16 Mar 2017 18:25:18 -0700 (PDT) Subject: [pypy-commit] pypy default: cffi is not required for translation, just having pycparser works too Message-ID: <58cb3afe.07452e0a.77100.734e@mx.google.com> Author: Ronan Lamy Branch: Changeset: r90736:e668451adc8d Date: 2017-03-17 01:20 +0000 http://bitbucket.org/pypy/pypy/changeset/e668451adc8d/ Log: cffi is not required for translation, just having pycparser works too diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -18,7 +18,7 @@ version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and as `these benchmarks show`_ it already gives a nice speed bump. -We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise +We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise "PyPy3.5" supports the Python 3.5 language). CFFI_ has been updated to 1.10, improving an already great package for @@ -100,8 +100,10 @@ * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is now possible to obtain the address of any readonly object without pinning it * refactor the initialization code in translating cpyext + * use a cffi-style C parser to create rffi objects in cpyext, now the + translating Python must have either ``cffi`` or ``pycparser`` available * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make - available as ``__pypy__.move_to_end`` and, on Py3.5, + available as ``__pypy__.move_to_end`` and, on py3.5, ``OrderedDict.move_to_end()`` * remove completely RPython ``space.wrap`` in a major cleanup, differentiate between ``space.newtext`` and ``space.newbytes`` on py3.5 @@ -156,8 +158,6 @@ * improve the consistency of RPython annotation unions * add translation option --keepgoing to continue after the first AnnotationError - * use a cffi-style C parser to create rffi objects in cpyext, now the - translating Python must have cffi available * improve shadowstack to where it is now the default in place of asmgcc * add a rpython implementation of siphash24, allow choosing hash algorithm randomizing the seed From pypy.commits at gmail.com Thu Mar 16 23:04:41 2017 From: pypy.commits at gmail.com (mjacob) Date: Thu, 16 Mar 2017 20:04:41 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix filename decoding for zipimport module. Message-ID: <58cb5249.02a2190a.b793.7363@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90737:978c95116422 Date: 2017-03-17 02:57 +0100 http://bitbucket.org/pypy/pypy/changeset/978c95116422/ Log: Fix filename decoding for zipimport module. diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -119,7 +119,7 @@ def _prepare_module(space, w_mod, filename, pkgdir): space.sys.setmodule(w_mod) - space.setattr(w_mod, space.newtext('__file__'), space.newtext(filename)) + space.setattr(w_mod, space.newtext('__file__'), space.newfilename(filename)) space.setattr(w_mod, space.newtext('__doc__'), space.w_None) if pkgdir is not None: space.setattr(w_mod, space.newtext('__path__'), diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py --- a/pypy/module/zipimport/test/test_zipimport.py +++ b/pypy/module/zipimport/test/test_zipimport.py @@ -431,7 +431,7 @@ z.writestr(zinfo, '') z.close() try: - zipimport.zipimporter(filename) + zipimport.zipimporter(filename).load_module('uu') finally: os.remove(filename) From pypy.commits at gmail.com Fri Mar 17 01:45:27 2017 From: pypy.commits at gmail.com (mattip) Date: Thu, 16 Mar 2017 22:45:27 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-5.x: set version for beta release (squeaky) Message-ID: <58cb77f7.cdd8190a.f875f.7c3a@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-5.x Changeset: r90738:88dd4b6290ca Date: 2017-03-17 07:41 +0200 http://bitbucket.org/pypy/pypy/changeset/88dd4b6290ca/ Log: set version for beta release (squeaky) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "3.5.2" /* PyPy version as a string */ -#define PYPY_VERSION "5.8.0-alpha0" -#define PYPY_VERSION_NUM 0x05080000 +#define PYPY_VERSION "5.7.0-beta0" +#define PYPY_VERSION_NUM 0x05070000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 8, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (5, 7, 0, "beta", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Fri Mar 17 03:29:59 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 17 Mar 2017 00:29:59 -0700 (PDT) Subject: [pypy-commit] pypy default: mention pypy3-c in this README too Message-ID: <58cb9077.554c190a.4cd76.7cd8@mx.google.com> Author: Armin Rigo Branch: Changeset: r90739:35e71a2df836 Date: 2017-03-17 08:27 +0100 http://bitbucket.org/pypy/pypy/changeset/35e71a2df836/ Log: mention pypy3-c in this README too diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -27,14 +27,19 @@ Building ======== -build with: +First switch to or download the correct branch. The basic choices are +``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X +branch (e.g. ``py3.5``). + +Build with: .. code-block:: console $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py -This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter; you can -find more details about various installation schemes here: +This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy +directory. We suggest to use virtualenv with the resulting +pypy-c/pypy3-c as the interpreter; you can find more details about +various installation schemes here: http://doc.pypy.org/en/latest/install.html From pypy.commits at gmail.com Fri Mar 17 03:30:01 2017 From: pypy.commits at gmail.com (arigo) Date: Fri, 17 Mar 2017 00:30:01 -0700 (PDT) Subject: [pypy-commit] pypy default: Tweak in the first paragraph Message-ID: <58cb9079.1f002e0a.eed98.7dd1@mx.google.com> Author: Armin Rigo Branch: Changeset: r90740:897b23340aaa Date: 2017-03-17 08:29 +0100 http://bitbucket.org/pypy/pypy/changeset/897b23340aaa/ Log: Tweak in the first paragraph diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -2,9 +2,10 @@ PyPy2.7 and PyPy3.5 v5.7 - two in one release ============================================= -We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7 for 64 bit linux. +We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for +Linux 64bit only at first). This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and -PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version +PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version 3.5.3. We continue to make incremental improvements to our C-API From pypy.commits at gmail.com Fri Mar 17 05:24:47 2017 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 17 Mar 2017 02:24:47 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: check if RAND_egd is present before builtinify is called, (also transform RAND_status) Message-ID: <58cbab5f.a215190a.c98de.7eef@mx.google.com> Author: Richard Plangger Branch: py3.5 Changeset: r90741:a320b2c592ec Date: 2017-03-17 10:24 +0100 http://bitbucket.org/pypy/pypy/changeset/a320b2c592ec/ Log: check if RAND_egd is present before builtinify is called, (also transform RAND_status) diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -1363,7 +1363,8 @@ return lib.BIO_ctrl_pending(self.bio) -RAND_status = lib.RAND_status +def RAND_status(): + return lib.RAND_status() def _RAND_bytes(count, pseudo): if count < 0: diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py --- a/lib_pypy/_ssl/__init__.py +++ b/lib_pypy/_ssl/__init__.py @@ -1,5 +1,6 @@ from _cffi_ssl._stdssl import (_PROTOCOL_NAMES, _OPENSSL_API_VERSION, _test_decode_cert, _SSLContext) +from _cffi_ssl import _stdssl from _cffi_ssl._stdssl import * @@ -8,5 +9,8 @@ RAND_add = builtinify(RAND_add) RAND_bytes = builtinify(RAND_bytes) -RAND_egd = builtinify(RAND_egd) RAND_pseudo_bytes = builtinify(RAND_pseudo_bytes) +RAND_status = builtinify(RAND_status) +# RAND_egd is optional and might not be available on e.g. libreoffice +if hasattr(_stdssl, 'RAND_egd'): + RAND_egd = builtinify(RAND_egd) From pypy.commits at gmail.com Fri Mar 17 05:31:57 2017 From: pypy.commits at gmail.com (plan_rich) Date: Fri, 17 Mar 2017 02:31:57 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: wow, what a typo Message-ID: <58cbad0d.4a572e0a.941df.83ea@mx.google.com> Author: Richard Plangger Branch: py3.5 Changeset: r90742:3e766f54a47c Date: 2017-03-17 10:31 +0100 http://bitbucket.org/pypy/pypy/changeset/3e766f54a47c/ Log: wow, what a typo diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py --- a/lib_pypy/_ssl/__init__.py +++ b/lib_pypy/_ssl/__init__.py @@ -11,6 +11,6 @@ RAND_bytes = builtinify(RAND_bytes) RAND_pseudo_bytes = builtinify(RAND_pseudo_bytes) RAND_status = builtinify(RAND_status) -# RAND_egd is optional and might not be available on e.g. libreoffice +# RAND_egd is optional and might not be available on e.g. libressl if hasattr(_stdssl, 'RAND_egd'): RAND_egd = builtinify(RAND_egd) From pypy.commits at gmail.com Fri Mar 17 05:53:18 2017 From: pypy.commits at gmail.com (mattip) Date: Fri, 17 Mar 2017 02:53:18 -0700 (PDT) Subject: [pypy-commit] pypy default: mention dropping support for python 2.6 Message-ID: <58cbb20e.1f002e0a.eed98.8607@mx.google.com> Author: Matti Picus Branch: Changeset: r90743:ef43d5aa82f7 Date: 2017-03-17 11:49 +0200 http://bitbucket.org/pypy/pypy/changeset/ef43d5aa82f7/ Log: mention dropping support for python 2.6 diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -83,6 +83,9 @@ Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) ===================================================================================== +Translation uses python 2.7 syntax, we could renew support for python 2.6 if +a downstream RPython-based project depends on it. + See also issues that were resolved_ * New features and cleanups @@ -129,8 +132,6 @@ ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` * improve support for creating C-extension objects from app-level classes, filling more slots, especially ``tp_new`` and ``tp_dealloc`` - * add rstack.stack_almost_full() and use it to avoid stack overflow due to - the JIT where possible * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_ * fix in corner cases with the GIL and C-API functions @@ -162,6 +163,8 @@ * improve shadowstack to where it is now the default in place of asmgcc * add a rpython implementation of siphash24, allow choosing hash algorithm randomizing the seed + * add rstack.stack_almost_full() and use it to avoid stack overflow due to + the JIT where possible Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) ========================================================================== From pypy.commits at gmail.com Fri Mar 17 08:34:00 2017 From: pypy.commits at gmail.com (mattip) Date: Fri, 17 Mar 2017 05:34:00 -0700 (PDT) Subject: [pypy-commit] pypy default: apparently the python 2.7 requirement is not new to this version Message-ID: <58cbd7b8.89012e0a.9670a.287a@mx.google.com> Author: Matti Picus Branch: Changeset: r90744:24ba403e24e6 Date: 2017-03-17 14:27 +0200 http://bitbucket.org/pypy/pypy/changeset/24ba403e24e6/ Log: apparently the python 2.7 requirement is not new to this version diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -83,9 +83,6 @@ Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) ===================================================================================== -Translation uses python 2.7 syntax, we could renew support for python 2.6 if -a downstream RPython-based project depends on it. - See also issues that were resolved_ * New features and cleanups From pypy.commits at gmail.com Sat Mar 18 02:45:46 2017 From: pypy.commits at gmail.com (mattip) Date: Fri, 17 Mar 2017 23:45:46 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into release Message-ID: <58ccd79a.0a482e0a.c35bf.afda@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-5.x Changeset: r90745:b702475ce4b0 Date: 2017-03-18 08:44 +0200 http://bitbucket.org/pypy/pypy/changeset/b702475ce4b0/ Log: merge py3.5 into release diff --git a/lib-python/3/test/test_importlib/frozen/__init__.py b/lib-python/3/test/test_importlib/frozen/__init__.py --- a/lib-python/3/test/test_importlib/frozen/__init__.py +++ b/lib-python/3/test/test_importlib/frozen/__init__.py @@ -1,5 +1,10 @@ import os -from test.support import load_package_tests +import unittest +from test.support import load_package_tests, check_impl_detail def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) + + +if check_impl_detail(pypy=True): + raise unittest.SkipTest("PyPy doesn't have frozen modules") diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -1363,7 +1363,8 @@ return lib.BIO_ctrl_pending(self.bio) -RAND_status = lib.RAND_status +def RAND_status(): + return lib.RAND_status() def _RAND_bytes(count, pseudo): if count < 0: diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py --- a/lib_pypy/_ssl/__init__.py +++ b/lib_pypy/_ssl/__init__.py @@ -1,5 +1,6 @@ from _cffi_ssl._stdssl import (_PROTOCOL_NAMES, _OPENSSL_API_VERSION, _test_decode_cert, _SSLContext) +from _cffi_ssl import _stdssl from _cffi_ssl._stdssl import * @@ -8,5 +9,8 @@ RAND_add = builtinify(RAND_add) RAND_bytes = builtinify(RAND_bytes) -RAND_egd = builtinify(RAND_egd) RAND_pseudo_bytes = builtinify(RAND_pseudo_bytes) +RAND_status = builtinify(RAND_status) +# RAND_egd is optional and might not be available on e.g. libressl +if hasattr(_stdssl, 'RAND_egd'): + RAND_egd = builtinify(RAND_egd) diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -1,4 +1,4 @@ -from pypy.interpreter.module import Module +from pypy.interpreter.module import Module, init_extra_module_attrs from pypy.interpreter.function import Function, BuiltinFunction from pypy.interpreter import gateway from pypy.interpreter.error import OperationError @@ -18,6 +18,7 @@ def __init__(self, space, w_name): """ NOT_RPYTHON """ Module.__init__(self, space, w_name) + init_extra_module_attrs(space, self) self.lazy = True self.lazy_initial_values_w = {} self.__class__.buildloaders() diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -119,7 +119,7 @@ def _prepare_module(space, w_mod, filename, pkgdir): space.sys.setmodule(w_mod) - space.setattr(w_mod, space.newtext('__file__'), space.newtext(filename)) + space.setattr(w_mod, space.newtext('__file__'), space.newfilename(filename)) space.setattr(w_mod, space.newtext('__doc__'), space.w_None) if pkgdir is not None: space.setattr(w_mod, space.newtext('__path__'), diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py --- a/pypy/module/zipimport/interp_zipimport.py +++ b/pypy/module/zipimport/interp_zipimport.py @@ -373,7 +373,7 @@ if self.have_modulefile(space, filename + ext): return True, None # See if this is a directory (part of a namespace pkg) - dirpath = self.prefix + fullname + dirpath = self.prefix + fullname.split(".")[-1] if self.have_modulefile(space, dirpath + ZIPSEP): return True, self.filename + os.path.sep + self.corr_zname(dirpath) return False, None diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py --- a/pypy/module/zipimport/test/test_zipimport.py +++ b/pypy/module/zipimport/test/test_zipimport.py @@ -431,7 +431,7 @@ z.writestr(zinfo, '') z.close() try: - zipimport.zipimporter(filename) + zipimport.zipimporter(filename).load_module('uu') finally: os.remove(filename) @@ -446,6 +446,13 @@ foo = __import__('foo.one', None, None, []) assert foo.one.attr == 'portion1 foo one' + def test_namespace_pkg_nested(self): + self.writefile('foo/', '') + self.writefile('foo/bar/', '') + self.writefile('foo/bar/one.py', "attr = 'portion1 foo one'\n") + foo = __import__('foo.bar.one', None, None, []) + assert foo.bar.one.attr == 'portion1 foo one' + if os.sep != '/': class AppTestNativePathSep(AppTestZipimport): From pypy.commits at gmail.com Sat Mar 18 11:21:09 2017 From: pypy.commits at gmail.com (arigo) Date: Sat, 18 Mar 2017 08:21:09 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2504: PyClass_New() can be called with NULL as its first argument Message-ID: <58cd5065.d8202e0a.a251f.c145@mx.google.com> Author: Armin Rigo Branch: Changeset: r90746:7d81a8c7bdae Date: 2017-03-18 16:20 +0100 http://bitbucket.org/pypy/pypy/changeset/7d81a8c7bdae/ Log: Issue #2504: PyClass_New() can be called with NULL as its first argument diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -39,6 +39,8 @@ @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyClass_New(space, w_bases, w_dict, w_name): + if w_bases is None: + w_bases = space.newtuple([]) w_classobj = space.gettypefor(W_ClassObject) return space.call_function(w_classobj, w_name, w_bases, w_dict) diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -77,3 +77,15 @@ class C: pass assert module.get_classtype() is type(C) + + def test_pyclass_new_no_bases(self): + module = self.import_extension('foo', [ + ("new_foo", "METH_O", + """ + return PyClass_New(NULL, PyDict_New(), args); + """)]) + FooClass = module.new_foo("FooClass") + class Cls1: + pass + assert type(FooClass) is type(Cls1) + assert FooClass.__bases__ == Cls1.__bases__ From pypy.commits at gmail.com Sat Mar 18 15:50:28 2017 From: pypy.commits at gmail.com (mattip) Date: Sat, 18 Mar 2017 12:50:28 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default into release Message-ID: <58cd8f84.41072e0a.4527a.c562@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-5.x Changeset: r90747:0aaffb19afb7 Date: 2017-03-18 21:17 +0200 http://bitbucket.org/pypy/pypy/changeset/0aaffb19afb7/ Log: merge default into release diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -27,14 +27,19 @@ Building ======== -build with: +First switch to or download the correct branch. The basic choices are +``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X +branch (e.g. ``py3.5``). + +Build with: .. code-block:: console $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py -This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter; you can -find more details about various installation schemes here: +This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy +directory. We suggest to use virtualenv with the resulting +pypy-c/pypy3-c as the interpreter; you can find more details about +various installation schemes here: http://doc.pypy.org/en/latest/install.html diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -2,23 +2,24 @@ PyPy2.7 and PyPy3.5 v5.7 - two in one release ============================================= -We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7. +We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for +Linux 64bit only at first). This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and -PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version +PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version 3.5.3. We continue to make incremental improvements to our C-API -compatibility layer (cpyext). PyPy2 can now import and run many c-extension -packages, among the most notable are numpy, cython, and pandas. Performance may +compatibility layer (cpyext). PyPy2 can now import and run many C-extension +packages, among the most notable are Numpy, Cython, and Pandas. Performance may be slower than CPython, especially for frequently-called short C functions. Please let us know if your use case is slow, we have ideas how to make things faster but need real-world examples (not micro-benchmarks) of problematic code. Work proceeds at a good pace on the PyPy3.5 version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta -release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and +release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and as `these benchmarks show`_ it already gives a nice speed bump. -We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise +We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise "PyPy3.5" supports the Python 3.5 language). CFFI_ has been updated to 1.10, improving an already great package for @@ -65,7 +66,7 @@ We also welcome developers of other `dynamic languages`_ to see what RPython can do for them. -This release supports: +The PyPy 2.7 release supports: * **x86** machines on most common operating systems (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) @@ -80,64 +81,58 @@ .. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) -============================================================================================= +===================================================================================== See also issues that were resolved_ * New features and cleanups * update the format of the PYPYLOG file and improvements to vmprof - * improve the consistency of RPython annotation unions * emit more sysconfig values for downstream cextension packages - * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*, - _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork, - * add translation option --keepgoing to continue after the first AnnotationError + * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``, + ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``, * detect and raise on recreation of a PyPy object from a PyObject during tp_dealloc * refactor and clean up poor handling of unicode exposed in work on py3.5 - * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) - * add translation time --disable_entrypoints option for embedding PyPy together - with another RPython VM - * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14 + * builtin module cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) + * adapt ``weakref`` according to CPython issue #19542_, will be in CPython 2.7.14 * support translations with cpyext and the Boehm GC (for special cases like - revdb + RevDB_ * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is now possible to obtain the address of any readonly object without pinning it * refactor the initialization code in translating cpyext - * fix ``"".replace("", "x", num)`` to give the same result as CPython * use a cffi-style C parser to create rffi objects in cpyext, now the - translating python must have cffi available - * add a rpython implementation of siphash24, allow choosing hash algorithm - randomizing the seed - * make ``attach_gdb`` work on Windows (with Visual Studio Debugger) + translating Python must have either ``cffi`` or ``pycparser`` available * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make available as ``__pypy__.move_to_end`` and, on py3.5, ``OrderedDict.move_to_end()`` * remove completely RPython ``space.wrap`` in a major cleanup, differentiate between ``space.newtext`` and ``space.newbytes`` on py3.5 - * improve shadowstack to where it is now the default in place of asmgcc + * any uncaught RPython exception in the interpreter is turned into a + SystemError (rather than a segfault) + * add translation time --disable_entrypoints option for embedding PyPy together + with another RPython VM + * Bug Fixes - * any uncaught RPython exception in the interpreter is turned into a - SystemError (rather than a segfault) + * fix ``"".replace("", "x", num)`` to give the same result as CPython * create log files without the executable bit - * disable clock_gettime() on OS/X, since we support 10.11 and it was only + * disable ``clock_gettime()`` on OS/X, since we support 10.11 and it was only added in 10.12 - * support HAVE_FSTATVFS which was unintentionally always false - * fix user-created C-API heaptype, issue #2434 - * fix PyDict_Update is not actually the same as dict.update - * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute - issue #2446 + * support ``HAVE_FSTATVFS`` which was unintentionally always false + * fix user-created C-API heaptype, issue #2434_ + * fix ``PyDict_Update`` is not actually the same as ``dict.update`` + * assign ``tp_doc`` on ``PyTypeObject`` and tie it to the app-level ``__doc__`` attribute + issue #2446_ * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``, ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` - * improve support for creating c-extension objects from app-level classes, - filling more slots especially ``tp_new`` and ``tp_dealloc`` - * add rstack.stack_almost_full() and use it to avoid stack overflow due to - the JIT where possible - * fix for ctypes.c_bool returning bool restype issue #2475 + * improve support for creating C-extension objects from app-level classes, + filling more slots, especially ``tp_new`` and ``tp_dealloc`` + * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_ * fix in corner cases with the GIL and C-API functions + * Performance improvements: * clean-ups in the jit optimizeopt @@ -158,17 +153,28 @@ information across failing guards * add optimized "zero-copy" path for ``io.FileIO.readinto`` +* RPython improvements + + * improve the consistency of RPython annotation unions + * add translation option --keepgoing to continue after the first AnnotationError + * improve shadowstack to where it is now the default in place of asmgcc + * add a rpython implementation of siphash24, allow choosing hash algorithm + randomizing the seed + * add rstack.stack_almost_full() and use it to avoid stack overflow due to + the JIT where possible + Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) -========================================================= +========================================================================== -Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo +Development moved from the py3k branch to the py3.5 branch in the PyPy bitbucket repo. * New features - * this first PyPy3.5 release implements much, but not all, of Python 3.5.3 + * this first PyPy3.5 release implements most of Python 3.5.3, exceptions are listed below * PEP 456 allowing secure and interchangable hash algorithms * use cryptography_'s cffi backend for SSL + * Bug Fixes * implement fixes for some CPython issues that arose since the last release @@ -176,12 +182,23 @@ * Performance improvements: - * do not create a list whenever descr_new of a bytesobject is called + * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called * * * +* The following features of Python 3.5 are not implemented yet in PyPy: + + * PEP 442: Safe object finalization + * PEP 489: Multi-phase extension module initialization + * XXX what else? + .. _resolved: whatsnew-pypy2-5.7.0.html +.. _19542: https://bugs.python.org/issue19542 +.. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys +.. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on +.. _2475: https://bitbucket.org/pypy/pypy/issues/2475 +.. _RevDB: https://bitbucket.org/pypy/revdb .. _cryptography: https://cryptography.io .. _cppyy: cppyy.html diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -39,6 +39,8 @@ @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyClass_New(space, w_bases, w_dict, w_name): + if w_bases is None: + w_bases = space.newtuple([]) w_classobj = space.gettypefor(W_ClassObject) return space.call_function(w_classobj, w_name, w_bases, w_dict) diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -77,3 +77,15 @@ class C: pass assert module.get_classtype() is type(C) + + def test_pyclass_new_no_bases(self): + module = self.import_extension('foo', [ + ("new_foo", "METH_O", + """ + return PyClass_New(NULL, PyDict_New(), args); + """)]) + FooClass = module.new_foo("FooClass") + class Cls1: + pass + assert type(FooClass) is type(Cls1) + assert FooClass.__bases__ == Cls1.__bases__ From pypy.commits at gmail.com Sat Mar 18 15:50:30 2017 From: pypy.commits at gmail.com (mattip) Date: Sat, 18 Mar 2017 12:50:30 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <58cd8f86.02502e0a.16894.b891@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r90748:b6b60142458f Date: 2017-03-18 21:47 +0200 http://bitbucket.org/pypy/pypy/changeset/b6b60142458f/ Log: merge default into py3.5 diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -27,14 +27,19 @@ Building ======== -build with: +First switch to or download the correct branch. The basic choices are +``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X +branch (e.g. ``py3.5``). + +Build with: .. code-block:: console $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py -This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter; you can -find more details about various installation schemes here: +This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy +directory. We suggest to use virtualenv with the resulting +pypy-c/pypy3-c as the interpreter; you can find more details about +various installation schemes here: http://doc.pypy.org/en/latest/install.html diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -2,23 +2,24 @@ PyPy2.7 and PyPy3.5 v5.7 - two in one release ============================================= -We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7. +We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for +Linux 64bit only at first). This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and -PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version +PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version 3.5.3. We continue to make incremental improvements to our C-API -compatibility layer (cpyext). PyPy2 can now import and run many c-extension -packages, among the most notable are numpy, cython, and pandas. Performance may +compatibility layer (cpyext). PyPy2 can now import and run many C-extension +packages, among the most notable are Numpy, Cython, and Pandas. Performance may be slower than CPython, especially for frequently-called short C functions. Please let us know if your use case is slow, we have ideas how to make things faster but need real-world examples (not micro-benchmarks) of problematic code. Work proceeds at a good pace on the PyPy3.5 version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta -release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and +release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and as `these benchmarks show`_ it already gives a nice speed bump. -We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise +We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise "PyPy3.5" supports the Python 3.5 language). CFFI_ has been updated to 1.10, improving an already great package for @@ -65,7 +66,7 @@ We also welcome developers of other `dynamic languages`_ to see what RPython can do for them. -This release supports: +The PyPy 2.7 release supports: * **x86** machines on most common operating systems (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) @@ -80,64 +81,58 @@ .. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) -============================================================================================= +===================================================================================== See also issues that were resolved_ * New features and cleanups * update the format of the PYPYLOG file and improvements to vmprof - * improve the consistency of RPython annotation unions * emit more sysconfig values for downstream cextension packages - * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*, - _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork, - * add translation option --keepgoing to continue after the first AnnotationError + * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``, + ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``, * detect and raise on recreation of a PyPy object from a PyObject during tp_dealloc * refactor and clean up poor handling of unicode exposed in work on py3.5 - * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) - * add translation time --disable_entrypoints option for embedding PyPy together - with another RPython VM - * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14 + * builtin module cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) + * adapt ``weakref`` according to CPython issue #19542_, will be in CPython 2.7.14 * support translations with cpyext and the Boehm GC (for special cases like - revdb + RevDB_ * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is now possible to obtain the address of any readonly object without pinning it * refactor the initialization code in translating cpyext - * fix ``"".replace("", "x", num)`` to give the same result as CPython * use a cffi-style C parser to create rffi objects in cpyext, now the - translating python must have cffi available - * add a rpython implementation of siphash24, allow choosing hash algorithm - randomizing the seed - * make ``attach_gdb`` work on Windows (with Visual Studio Debugger) + translating Python must have either ``cffi`` or ``pycparser`` available * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make available as ``__pypy__.move_to_end`` and, on py3.5, ``OrderedDict.move_to_end()`` * remove completely RPython ``space.wrap`` in a major cleanup, differentiate between ``space.newtext`` and ``space.newbytes`` on py3.5 - * improve shadowstack to where it is now the default in place of asmgcc + * any uncaught RPython exception in the interpreter is turned into a + SystemError (rather than a segfault) + * add translation time --disable_entrypoints option for embedding PyPy together + with another RPython VM + * Bug Fixes - * any uncaught RPython exception in the interpreter is turned into a - SystemError (rather than a segfault) + * fix ``"".replace("", "x", num)`` to give the same result as CPython * create log files without the executable bit - * disable clock_gettime() on OS/X, since we support 10.11 and it was only + * disable ``clock_gettime()`` on OS/X, since we support 10.11 and it was only added in 10.12 - * support HAVE_FSTATVFS which was unintentionally always false - * fix user-created C-API heaptype, issue #2434 - * fix PyDict_Update is not actually the same as dict.update - * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute - issue #2446 + * support ``HAVE_FSTATVFS`` which was unintentionally always false + * fix user-created C-API heaptype, issue #2434_ + * fix ``PyDict_Update`` is not actually the same as ``dict.update`` + * assign ``tp_doc`` on ``PyTypeObject`` and tie it to the app-level ``__doc__`` attribute + issue #2446_ * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``, ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` - * improve support for creating c-extension objects from app-level classes, - filling more slots especially ``tp_new`` and ``tp_dealloc`` - * add rstack.stack_almost_full() and use it to avoid stack overflow due to - the JIT where possible - * fix for ctypes.c_bool returning bool restype issue #2475 + * improve support for creating C-extension objects from app-level classes, + filling more slots, especially ``tp_new`` and ``tp_dealloc`` + * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_ * fix in corner cases with the GIL and C-API functions + * Performance improvements: * clean-ups in the jit optimizeopt @@ -158,17 +153,28 @@ information across failing guards * add optimized "zero-copy" path for ``io.FileIO.readinto`` +* RPython improvements + + * improve the consistency of RPython annotation unions + * add translation option --keepgoing to continue after the first AnnotationError + * improve shadowstack to where it is now the default in place of asmgcc + * add a rpython implementation of siphash24, allow choosing hash algorithm + randomizing the seed + * add rstack.stack_almost_full() and use it to avoid stack overflow due to + the JIT where possible + Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) -========================================================= +========================================================================== -Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo +Development moved from the py3k branch to the py3.5 branch in the PyPy bitbucket repo. * New features - * this first PyPy3.5 release implements much, but not all, of Python 3.5.3 + * this first PyPy3.5 release implements most of Python 3.5.3, exceptions are listed below * PEP 456 allowing secure and interchangable hash algorithms * use cryptography_'s cffi backend for SSL + * Bug Fixes * implement fixes for some CPython issues that arose since the last release @@ -176,12 +182,23 @@ * Performance improvements: - * do not create a list whenever descr_new of a bytesobject is called + * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called * * * +* The following features of Python 3.5 are not implemented yet in PyPy: + + * PEP 442: Safe object finalization + * PEP 489: Multi-phase extension module initialization + * XXX what else? + .. _resolved: whatsnew-pypy2-5.7.0.html +.. _19542: https://bugs.python.org/issue19542 +.. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys +.. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on +.. _2475: https://bitbucket.org/pypy/pypy/issues/2475 +.. _RevDB: https://bitbucket.org/pypy/revdb .. _cryptography: https://cryptography.io .. _cppyy: cppyy.html diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -70,4 +70,6 @@ checking.""" return space.interp_w(InstanceMethod, w_im).w_function + if w_bases is None: + w_bases = space.newtuple([]) diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -27,3 +27,15 @@ InstanceMethod.testmethod.attribute = "test" assert testfunction.attribute == "test" raises(AttributeError, setattr, inst.testmethod, "attribute", "test") + + def test_pyclass_new_no_bases(self): + module = self.import_extension('foo', [ + ("new_foo", "METH_O", + """ + return PyClass_New(NULL, PyDict_New(), args); + """)]) + FooClass = module.new_foo("FooClass") + class Cls1: + pass + assert type(FooClass) is type(Cls1) + assert FooClass.__bases__ == Cls1.__bases__ From pypy.commits at gmail.com Sat Mar 18 15:50:33 2017 From: pypy.commits at gmail.com (mattip) Date: Sat, 18 Mar 2017 12:50:33 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into pypy3.5 release Message-ID: <58cd8f89.5c582e0a.921d3.c703@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-5.x Changeset: r90749:479ecd023f5d Date: 2017-03-18 21:48 +0200 http://bitbucket.org/pypy/pypy/changeset/479ecd023f5d/ Log: merge py3.5 into pypy3.5 release diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -27,14 +27,19 @@ Building ======== -build with: +First switch to or download the correct branch. The basic choices are +``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X +branch (e.g. ``py3.5``). + +Build with: .. code-block:: console $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py -This ends up with ``pypy-c`` binary in the main pypy directory. We suggest -to use virtualenv with the resulting pypy-c as the interpreter; you can -find more details about various installation schemes here: +This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy +directory. We suggest to use virtualenv with the resulting +pypy-c/pypy3-c as the interpreter; you can find more details about +various installation schemes here: http://doc.pypy.org/en/latest/install.html diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -2,23 +2,24 @@ PyPy2.7 and PyPy3.5 v5.7 - two in one release ============================================= -We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7. +We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for +Linux 64bit only at first). This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and -PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version +PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version 3.5.3. We continue to make incremental improvements to our C-API -compatibility layer (cpyext). PyPy2 can now import and run many c-extension -packages, among the most notable are numpy, cython, and pandas. Performance may +compatibility layer (cpyext). PyPy2 can now import and run many C-extension +packages, among the most notable are Numpy, Cython, and Pandas. Performance may be slower than CPython, especially for frequently-called short C functions. Please let us know if your use case is slow, we have ideas how to make things faster but need real-world examples (not micro-benchmarks) of problematic code. Work proceeds at a good pace on the PyPy3.5 version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta -release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and +release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and as `these benchmarks show`_ it already gives a nice speed bump. -We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise +We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise "PyPy3.5" supports the Python 3.5 language). CFFI_ has been updated to 1.10, improving an already great package for @@ -65,7 +66,7 @@ We also welcome developers of other `dynamic languages`_ to see what RPython can do for them. -This release supports: +The PyPy 2.7 release supports: * **x86** machines on most common operating systems (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) @@ -80,64 +81,58 @@ .. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016) -============================================================================================= +===================================================================================== See also issues that were resolved_ * New features and cleanups * update the format of the PYPYLOG file and improvements to vmprof - * improve the consistency of RPython annotation unions * emit more sysconfig values for downstream cextension packages - * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*, - _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork, - * add translation option --keepgoing to continue after the first AnnotationError + * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``, + ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``, * detect and raise on recreation of a PyPy object from a PyObject during tp_dealloc * refactor and clean up poor handling of unicode exposed in work on py3.5 - * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) - * add translation time --disable_entrypoints option for embedding PyPy together - with another RPython VM - * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14 + * builtin module cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed) + * adapt ``weakref`` according to CPython issue #19542_, will be in CPython 2.7.14 * support translations with cpyext and the Boehm GC (for special cases like - revdb + RevDB_ * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is now possible to obtain the address of any readonly object without pinning it * refactor the initialization code in translating cpyext - * fix ``"".replace("", "x", num)`` to give the same result as CPython * use a cffi-style C parser to create rffi objects in cpyext, now the - translating python must have cffi available - * add a rpython implementation of siphash24, allow choosing hash algorithm - randomizing the seed - * make ``attach_gdb`` work on Windows (with Visual Studio Debugger) + translating Python must have either ``cffi`` or ``pycparser`` available * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make available as ``__pypy__.move_to_end`` and, on py3.5, ``OrderedDict.move_to_end()`` * remove completely RPython ``space.wrap`` in a major cleanup, differentiate between ``space.newtext`` and ``space.newbytes`` on py3.5 - * improve shadowstack to where it is now the default in place of asmgcc + * any uncaught RPython exception in the interpreter is turned into a + SystemError (rather than a segfault) + * add translation time --disable_entrypoints option for embedding PyPy together + with another RPython VM + * Bug Fixes - * any uncaught RPython exception in the interpreter is turned into a - SystemError (rather than a segfault) + * fix ``"".replace("", "x", num)`` to give the same result as CPython * create log files without the executable bit - * disable clock_gettime() on OS/X, since we support 10.11 and it was only + * disable ``clock_gettime()`` on OS/X, since we support 10.11 and it was only added in 10.12 - * support HAVE_FSTATVFS which was unintentionally always false - * fix user-created C-API heaptype, issue #2434 - * fix PyDict_Update is not actually the same as dict.update - * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute - issue #2446 + * support ``HAVE_FSTATVFS`` which was unintentionally always false + * fix user-created C-API heaptype, issue #2434_ + * fix ``PyDict_Update`` is not actually the same as ``dict.update`` + * assign ``tp_doc`` on ``PyTypeObject`` and tie it to the app-level ``__doc__`` attribute + issue #2446_ * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``, ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release`` - * improve support for creating c-extension objects from app-level classes, - filling more slots especially ``tp_new`` and ``tp_dealloc`` - * add rstack.stack_almost_full() and use it to avoid stack overflow due to - the JIT where possible - * fix for ctypes.c_bool returning bool restype issue #2475 + * improve support for creating C-extension objects from app-level classes, + filling more slots, especially ``tp_new`` and ``tp_dealloc`` + * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_ * fix in corner cases with the GIL and C-API functions + * Performance improvements: * clean-ups in the jit optimizeopt @@ -158,17 +153,28 @@ information across failing guards * add optimized "zero-copy" path for ``io.FileIO.readinto`` +* RPython improvements + + * improve the consistency of RPython annotation unions + * add translation option --keepgoing to continue after the first AnnotationError + * improve shadowstack to where it is now the default in place of asmgcc + * add a rpython implementation of siphash24, allow choosing hash algorithm + randomizing the seed + * add rstack.stack_almost_full() and use it to avoid stack overflow due to + the JIT where possible + Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016) -========================================================= +========================================================================== -Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo +Development moved from the py3k branch to the py3.5 branch in the PyPy bitbucket repo. * New features - * this first PyPy3.5 release implements much, but not all, of Python 3.5.3 + * this first PyPy3.5 release implements most of Python 3.5.3, exceptions are listed below * PEP 456 allowing secure and interchangable hash algorithms * use cryptography_'s cffi backend for SSL + * Bug Fixes * implement fixes for some CPython issues that arose since the last release @@ -176,12 +182,23 @@ * Performance improvements: - * do not create a list whenever descr_new of a bytesobject is called + * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called * * * +* The following features of Python 3.5 are not implemented yet in PyPy: + + * PEP 442: Safe object finalization + * PEP 489: Multi-phase extension module initialization + * XXX what else? + .. _resolved: whatsnew-pypy2-5.7.0.html +.. _19542: https://bugs.python.org/issue19542 +.. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys +.. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on +.. _2475: https://bitbucket.org/pypy/pypy/issues/2475 +.. _RevDB: https://bitbucket.org/pypy/revdb .. _cryptography: https://cryptography.io .. _cppyy: cppyy.html diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -70,4 +70,6 @@ checking.""" return space.interp_w(InstanceMethod, w_im).w_function + if w_bases is None: + w_bases = space.newtuple([]) diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -27,3 +27,15 @@ InstanceMethod.testmethod.attribute = "test" assert testfunction.attribute == "test" raises(AttributeError, setattr, inst.testmethod, "attribute", "test") + + def test_pyclass_new_no_bases(self): + module = self.import_extension('foo', [ + ("new_foo", "METH_O", + """ + return PyClass_New(NULL, PyDict_New(), args); + """)]) + FooClass = module.new_foo("FooClass") + class Cls1: + pass + assert type(FooClass) is type(Cls1) + assert FooClass.__bases__ == Cls1.__bases__ From pypy.commits at gmail.com Sat Mar 18 21:00:57 2017 From: pypy.commits at gmail.com (rlamy) Date: Sat, 18 Mar 2017 18:00:57 -0700 (PDT) Subject: [pypy-commit] pypy default: Set 'purelib' and 'platlib' to point to site-packages (issue #2506) Message-ID: <58cdd849.02502e0a.16894.c04a@mx.google.com> Author: Ronan Lamy Branch: Changeset: r90750:0e77a1cc8097 Date: 2017-03-19 01:00 +0000 http://bitbucket.org/pypy/pypy/changeset/0e77a1cc8097/ Log: Set 'purelib' and 'platlib' to point to site-packages (issue #2506) diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -29,8 +29,8 @@ 'pypy': { 'stdlib': '{base}/lib-{implementation_lower}/{py_version_short}', 'platstdlib': '{base}/lib-{implementation_lower}/{py_version_short}', - 'purelib': '{base}/lib-{implementation_lower}/{py_version_short}', - 'platlib': '{base}/lib-{implementation_lower}/{py_version_short}', + 'purelib': '{base}/site-packages', + 'platlib': '{base}/site-packages', 'include': '{base}/include', 'platinclude': '{base}/include', 'scripts': '{base}/bin', From pypy.commits at gmail.com Sat Mar 18 21:26:17 2017 From: pypy.commits at gmail.com (rlamy) Date: Sat, 18 Mar 2017 18:26:17 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Set 'purelib' and 'platlib' to point to site-packages (issue #2506) Message-ID: <58cdde39.d3d4190a.e0aa3.18a7@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r90751:137377d9971f Date: 2017-03-19 01:25 +0000 http://bitbucket.org/pypy/pypy/changeset/137377d9971f/ Log: Set 'purelib' and 'platlib' to point to site-packages (issue #2506) diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py --- a/lib-python/3/sysconfig.py +++ b/lib-python/3/sysconfig.py @@ -44,8 +44,8 @@ 'pypy': { 'stdlib': '{installed_base}/lib-python', 'platstdlib': '{base}/lib-python', - 'purelib': '{base}/lib-python', - 'platlib': '{base}/lib-python', + 'purelib': '{base}/site-packages', + 'platlib': '{base}/site-packages', 'include': '{installed_base}/include', 'platinclude': '{installed_base}/include', 'scripts': '{base}/bin', From pypy.commits at gmail.com Sun Mar 19 10:03:28 2017 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 19 Mar 2017 07:03:28 -0700 (PDT) Subject: [pypy-commit] pypy default: ssue #2501: Don't implement local.__init__, allow the other base classes have their own __init__. Message-ID: <58ce8fb0.039d190a.1d9a7.e718@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r90752:e626065a891e Date: 2017-03-19 15:02 +0100 http://bitbucket.org/pypy/pypy/changeset/e626065a891e/ Log: ssue #2501: Don't implement local.__init__, allow the other base classes have their own __init__. diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -1,6 +1,7 @@ import weakref from rpython.rlib import jit from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import oefmt from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty, descr_get_dict) @@ -74,18 +75,18 @@ return w_dict def descr_local__new__(space, w_subtype, __args__): + if __args__.arguments_w or __args__.keywords: + w_parent_init, _ = space.lookup_in_type_where(w_subtype, '__init__') + if w_parent_init is space.w_object: + raise oefmt(space.w_TypeError, + "Initialization arguments are not supported") local = space.allocate_instance(Local, w_subtype) Local.__init__(local, space, __args__) return local - def descr_local__init__(self, space): - # No arguments allowed - pass - Local.typedef = TypeDef("thread._local", __doc__ = "Thread-local data", __new__ = interp2app(Local.descr_local__new__.im_func), - __init__ = interp2app(Local.descr_local__init__), __dict__ = GetSetProperty(descr_get_dict, cls=Local), ) diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -72,6 +72,19 @@ assert seen1 == [1, 2, 3, 4, 5] assert tags == ['???'] + def test_local_init2(self): + import thread + + class A(object): + def __init__(self, n): + assert n == 42 + self.n = n + class X(thread._local, A): + pass + + x = X(42) + assert x.n == 42 + def test_local_setdict(self): import thread x = thread._local() From pypy.commits at gmail.com Sun Mar 19 13:54:32 2017 From: pypy.commits at gmail.com (mjacob) Date: Sun, 19 Mar 2017 10:54:32 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Implement PyModule_GetDef(). Message-ID: <58cec5d8.4aa7190a.d122b.ef45@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r90753:023f7c3d05ee Date: 2017-03-19 18:52 +0100 http://bitbucket.org/pypy/pypy/changeset/023f7c3d05ee/ Log: Implement PyModule_GetDef(). diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -565,6 +565,7 @@ 'PyUnicode_FromFormat', 'PyUnicode_FromFormatV', 'PyUnicode_AsWideCharString', 'PyUnicode_GetSize', 'PyUnicode_GetLength', 'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant', + 'PyModule_GetDef', 'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack', '_PyArg_Parse_SizeT', '_PyArg_ParseTuple_SizeT', '_PyArg_ParseTupleAndKeywords_SizeT', '_PyArg_VaParse_SizeT', diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h --- a/pypy/module/cpyext/include/modsupport.h +++ b/pypy/module/cpyext/include/modsupport.h @@ -70,6 +70,8 @@ #define PyModule_AddIntMacro(m, c) PyModule_AddIntConstant(m, #c, c) #define PyModule_AddStringMacro(m, c) PyModule_AddStringConstant(m, #c, c) +PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*); + PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...); PyAPI_FUNC(PyObject *) Py_VaBuildValue(const char *, va_list); diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -1,8 +1,8 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cts, - parse_dir) -from pypy.module.cpyext.pyobject import PyObject, as_pyobj + parse_dir, bootstrap_function) +from pypy.module.cpyext.pyobject import PyObject, as_pyobj, make_typedescr from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, @@ -13,6 +13,11 @@ cts.parse_header(parse_dir / 'cpyext_moduleobject.h') PyModuleDef = cts.gettype('PyModuleDef *') +PyModuleObject = cts.gettype('PyModuleObject *') + + at bootstrap_function +def init_moduleobject(space): + make_typedescr(Module.typedef, basestruct=PyModuleObject.TO) @cpython_api([PyModuleDef, rffi.INT_real], PyObject) def PyModule_Create2(space, module, api_version): @@ -35,6 +40,7 @@ if f_name is not None: modname = f_name w_mod = Module(space, space.newtext(modname)) + rffi.cast(PyModuleObject, as_pyobj(space, w_mod)).c_md_def = module state.package_context = None, None if f_path is not None: diff --git a/pypy/module/cpyext/parse/cpyext_moduleobject.h b/pypy/module/cpyext/parse/cpyext_moduleobject.h --- a/pypy/module/cpyext/parse/cpyext_moduleobject.h +++ b/pypy/module/cpyext/parse/cpyext_moduleobject.h @@ -36,3 +36,9 @@ inquiry m_clear; freefunc m_free; } PyModuleDef; + +typedef struct { + PyObject_HEAD + struct PyModuleDef *md_def; + //void *md_state; +} PyModuleObject; diff --git a/pypy/module/cpyext/src/modsupport.c b/pypy/module/cpyext/src/modsupport.c --- a/pypy/module/cpyext/src/modsupport.c +++ b/pypy/module/cpyext/src/modsupport.c @@ -592,3 +592,13 @@ Py_DECREF(o); return result < 0 ? -1 : 0; } + +PyModuleDef* +PyModule_GetDef(PyObject* m) +{ + if (!PyModule_Check(m)) { + PyErr_BadArgument(); + return NULL; + } + return ((PyModuleObject *)m)->md_def; +} diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py --- a/pypy/module/cpyext/test/test_module.py +++ b/pypy/module/cpyext/test/test_module.py @@ -1,4 +1,5 @@ from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from rpython.rtyper.lltypesystem import rffi @@ -10,3 +11,19 @@ p2 = api.PyModule_GetName(w_sys) assert p2 == p self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True) + + +class AppTestModuleObject(AppTestCpythonExtensionBase): + def test_getdef(self): + module = self.import_extension('foo', [ + ("check_getdef_same", "METH_NOARGS", + """ + return PyBool_FromLong(PyModule_GetDef(mod_global) == &moduledef); + """ + )], prologue=""" + static struct PyModuleDef moduledef; + static PyObject *mod_global; + """, more_init=""" + mod_global = mod; + """) + assert module.check_getdef_same() From pypy.commits at gmail.com Sun Mar 19 15:37:52 2017 From: pypy.commits at gmail.com (mattip) Date: Sun, 19 Mar 2017 12:37:52 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default into release Message-ID: <58cede10.86d0190a.35f2e.7dc5@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-5.x Changeset: r90754:b4750d2d0bee Date: 2017-03-19 21:35 +0200 http://bitbucket.org/pypy/pypy/changeset/b4750d2d0bee/ Log: merge default into release diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -29,8 +29,8 @@ 'pypy': { 'stdlib': '{base}/lib-{implementation_lower}/{py_version_short}', 'platstdlib': '{base}/lib-{implementation_lower}/{py_version_short}', - 'purelib': '{base}/lib-{implementation_lower}/{py_version_short}', - 'platlib': '{base}/lib-{implementation_lower}/{py_version_short}', + 'purelib': '{base}/site-packages', + 'platlib': '{base}/site-packages', 'include': '{base}/include', 'platinclude': '{base}/include', 'scripts': '{base}/bin', diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -1,6 +1,7 @@ import weakref from rpython.rlib import jit from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import oefmt from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty, descr_get_dict) @@ -74,18 +75,18 @@ return w_dict def descr_local__new__(space, w_subtype, __args__): + if __args__.arguments_w or __args__.keywords: + w_parent_init, _ = space.lookup_in_type_where(w_subtype, '__init__') + if w_parent_init is space.w_object: + raise oefmt(space.w_TypeError, + "Initialization arguments are not supported") local = space.allocate_instance(Local, w_subtype) Local.__init__(local, space, __args__) return local - def descr_local__init__(self, space): - # No arguments allowed - pass - Local.typedef = TypeDef("thread._local", __doc__ = "Thread-local data", __new__ = interp2app(Local.descr_local__new__.im_func), - __init__ = interp2app(Local.descr_local__init__), __dict__ = GetSetProperty(descr_get_dict, cls=Local), ) diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -72,6 +72,19 @@ assert seen1 == [1, 2, 3, 4, 5] assert tags == ['???'] + def test_local_init2(self): + import thread + + class A(object): + def __init__(self, n): + assert n == 42 + self.n = n + class X(thread._local, A): + pass + + x = X(42) + assert x.n == 42 + def test_local_setdict(self): import thread x = thread._local() From pypy.commits at gmail.com Sun Mar 19 15:37:54 2017 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 19 Mar 2017 12:37:54 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: ssue #2501: Don't implement local.__init__, allow the other base classes have their own __init__. Message-ID: <58cede12.46162e0a.800d.ee38@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.5 Changeset: r90755:30d5769bd07d Date: 2017-03-19 21:35 +0200 http://bitbucket.org/pypy/pypy/changeset/30d5769bd07d/ Log: ssue #2501: Don't implement local.__init__, allow the other base classes have their own __init__. diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -1,6 +1,7 @@ import weakref from rpython.rlib import jit from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import oefmt from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty, descr_get_dict) @@ -75,18 +76,18 @@ return w_dict def descr_local__new__(space, w_subtype, __args__): + if __args__.arguments_w or __args__.keywords: + w_parent_init, _ = space.lookup_in_type_where(w_subtype, '__init__') + if w_parent_init is space.w_object: + raise oefmt(space.w_TypeError, + "Initialization arguments are not supported") local = space.allocate_instance(Local, w_subtype) Local.__init__(local, space, __args__) return local - def descr_local__init__(self, space): - # No arguments allowed - pass - Local.typedef = TypeDef("thread._local", __doc__ = "Thread-local data", __new__ = interp2app(Local.descr_local__new__.im_func), - __init__ = interp2app(Local.descr_local__init__), __dict__ = GetSetProperty(descr_get_dict, cls=Local), ) diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -72,6 +72,19 @@ assert seen1 == [1, 2, 3, 4, 5] assert tags == ['???'] + def test_local_init2(self): + import thread + + class A(object): + def __init__(self, n): + assert n == 42 + self.n = n + class X(thread._local, A): + pass + + x = X(42) + assert x.n == 42 + def test_local_setdict(self): import _thread x = _thread._local() From pypy.commits at gmail.com Sun Mar 19 15:53:05 2017 From: pypy.commits at gmail.com (pjenvey) Date: Sun, 19 Mar 2017 12:53:05 -0700 (PDT) Subject: [pypy-commit] pypy default: fix translation Message-ID: <58cee1a1.47c6190a.d281e.4535@mx.google.com> Author: Philip Jenvey Branch: Changeset: r90756:a2617768d238 Date: 2017-03-19 12:52 -0700 http://bitbucket.org/pypy/pypy/changeset/a2617768d238/ Log: fix translation diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -75,6 +75,8 @@ return w_dict def descr_local__new__(space, w_subtype, __args__): + from pypy.objspace.std.typeobject import _precheck_for_new + w_subtype = _precheck_for_new(space, w_subtype) if __args__.arguments_w or __args__.keywords: w_parent_init, _ = space.lookup_in_type_where(w_subtype, '__init__') if w_parent_init is space.w_object: From pypy.commits at gmail.com Sun Mar 19 15:56:33 2017 From: pypy.commits at gmail.com (pjenvey) Date: Sun, 19 Mar 2017 12:56:33 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default Message-ID: <58cee271.46012e0a.d47cd.f7f3@mx.google.com> Author: Philip Jenvey Branch: release-pypy2.7-5.x Changeset: r90757:fa3249d55d15 Date: 2017-03-19 12:55 -0700 http://bitbucket.org/pypy/pypy/changeset/fa3249d55d15/ Log: merge default diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -75,6 +75,8 @@ return w_dict def descr_local__new__(space, w_subtype, __args__): + from pypy.objspace.std.typeobject import _precheck_for_new + w_subtype = _precheck_for_new(space, w_subtype) if __args__.arguments_w or __args__.keywords: w_parent_init, _ = space.lookup_in_type_where(w_subtype, '__init__') if w_parent_init is space.w_object: From pypy.commits at gmail.com Sun Mar 19 16:26:05 2017 From: pypy.commits at gmail.com (mattip) Date: Sun, 19 Mar 2017 13:26:05 -0700 (PDT) Subject: [pypy-commit] pypy default: document last changes to release Message-ID: <58cee95d.53be190a.40d1d.f0b4@mx.google.com> Author: Matti Picus Branch: Changeset: r90758:e8899d2b7da9 Date: 2017-03-19 22:25 +0200 http://bitbucket.org/pypy/pypy/changeset/e8899d2b7da9/ Log: document last changes to release diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -88,7 +88,8 @@ * New features and cleanups * update the format of the PYPYLOG file and improvements to vmprof - * emit more sysconfig values for downstream cextension packages + * emit more sysconfig values for downstream cextension packages including + properly setting purelib and platlib to site-packages * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``, ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``, * detect and raise on recreation of a PyPy object from a PyObject during @@ -131,6 +132,8 @@ filling more slots, especially ``tp_new`` and ``tp_dealloc`` * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_ * fix in corner cases with the GIL and C-API functions + * allow overriding thread.local.__init__ in a subclass, issue #2501_ + * allow ``PyClass_New`` to be called with NULL as the first arguemnt, issue #2504_ * Performance improvements: @@ -183,21 +186,19 @@ * Performance improvements: * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called - * - * - * * The following features of Python 3.5 are not implemented yet in PyPy: * PEP 442: Safe object finalization * PEP 489: Multi-phase extension module initialization - * XXX what else? .. _resolved: whatsnew-pypy2-5.7.0.html .. _19542: https://bugs.python.org/issue19542 .. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys .. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on .. _2475: https://bitbucket.org/pypy/pypy/issues/2475 +.. _2501: https://bitbucket.org/pypy/pypy/issues/2501 +.. _2504: https://bitbucket.org/pypy/pypy/issues/2504 .. _RevDB: https://bitbucket.org/pypy/revdb .. _cryptography: https://cryptography.io .. _cppyy: cppyy.html From pypy.commits at gmail.com Mon Mar 20 09:22:21 2017 From: pypy.commits at gmail.com (Raemi) Date: Mon, 20 Mar 2017 06:22:21 -0700 (PDT) Subject: [pypy-commit] pypy nogil-unsafe-2: make test pass by fixing expected output Message-ID: <58cfd78d.a2acdf0a.443e3.d535@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90759:2f39a22e1310 Date: 2017-03-20 14:20 +0100 http://bitbucket.org/pypy/pypy/changeset/2f39a22e1310/ Log: make test pass by fixing expected output diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1490,11 +1490,12 @@ t, cbuilder = self.compile(entry_point, no__thread=no__thread) data = cbuilder.cmdexec('5') assert data.splitlines() == ['hello world', - '1 ok', - '2 ok', - '3 ok', - '4 ok', - '5 ok'] + 'counter=5', + 'counter=4', + 'counter=3', + 'counter=2', + 'counter=1', + 'all threads done'] if SUPPORT__THREAD: runme(no__thread=False) From pypy.commits at gmail.com Mon Mar 20 09:37:06 2017 From: pypy.commits at gmail.com (Raemi) Date: Mon, 20 Mar 2017 06:37:06 -0700 (PDT) Subject: [pypy-commit] pypy nogil-unsafe-2: move nogil test to own file Message-ID: <58cfdb02.58092e0a.7252.19c4@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90760:2edf4f9d6779 Date: 2017-03-20 14:36 +0100 http://bitbucket.org/pypy/pypy/changeset/2edf4f9d6779/ Log: move nogil test to own file diff --git a/rpython/translator/c/test/test_nogil.py b/rpython/translator/c/test/test_nogil.py new file mode 100644 --- /dev/null +++ b/rpython/translator/c/test/test_nogil.py @@ -0,0 +1,98 @@ +import py +import os + +from rpython.translator.translator import TranslationContext +from rpython.translator.c.genc import CStandaloneBuilder + +from rpython.annotator.listdef import s_list_of_strings + + + + + + + +class TestThread(object): + gcrootfinder = 'shadowstack' + config = None + + def compile(self, entry_point, no__thread=False): + t = TranslationContext(self.config) + t.config.translation.gc = "incminimark" + t.config.translation.gcrootfinder = self.gcrootfinder + t.config.translation.thread = True + t.config.translation.no__thread = no__thread + t.buildannotator().build_types(entry_point, [s_list_of_strings]) + t.buildrtyper().specialize() + # + cbuilder = CStandaloneBuilder(t, entry_point, t.config) + cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) + cbuilder.compile() + # + return t, cbuilder + + + + def test_concurrent_allocate(self): + import time + from rpython.rlib import rthread, rposix + + class X: + def __init__(self, prev, i): + self.prev = prev + self.i = i + + class State: + pass + state = State() + + def thread(): + rthread.gc_thread_start() + x = None + for i in range(100000000): + prev_x = x + + x = X(x, i) + + if prev_x is not None: + assert prev_x.i == i - 1 + + if i % 5001 == 0: + x = None + + state.lock.acquire(True) + os.write(1, "counter=%d\n" % state.counter) + state.counter -= 1 + state.lock.release() + rthread.gc_thread_die() + + def entry_point(argv): + os.write(1, "hello world\n") + # start 5 new threads + TS = int(argv[1]) + state.lock = rthread.allocate_lock() + state.counter = TS + + for _ in range(TS): + rthread.start_new_thread(thread, ()) + + i = 0 + while True: + x = X(None, i) + time.sleep(0.1) + assert x.i == i + if state.counter == 0: + break + i += 1 + os.write(1, "all threads done\n") + return 0 + + t, cbuilder = self.compile(entry_point) + data = cbuilder.cmdexec('5') + assert data.splitlines() == ['hello world', + 'counter=5', + 'counter=4', + 'counter=3', + 'counter=2', + 'counter=1', + 'all threads done'] diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1427,79 +1427,6 @@ and result.count('a') == 1 and result.count('d') == 6) - def test_thread_and_gc_nogil(self): - import time, gc - from rpython.rlib import rthread, rposix - - class X: - def __init__(self, prev, i): - self.prev = prev - self.i = i - - class State: - pass - state = State() - - def bootstrap(): - rthread.gc_thread_start() - x = None - for i in range(100000000): - prev_x = x - - x = X(x, i) - - if prev_x is not None: - assert prev_x.i == i - 1 - - if i % 5001 == 0: - x = None - - state.lock.acquire(True) - os.write(1, "counter=%d\n" % state.counter) - state.counter -= 1 - state.lock.release() - rthread.gc_thread_die() - - def new_thread(): - ident = rthread.start_new_thread(bootstrap, ()) - return ident - - def entry_point(argv): - os.write(1, "hello world\n") - # start 5 new threads - TS = int(argv[1]) - state.lock = rthread.allocate_lock() - state.counter = TS - - for _ in range(TS): - new_thread() - - i = 0 - while True: - x = X(None, i) - time.sleep(0.1) - assert x.i == i - #gc.collect() - if state.counter == 0: - break - i += 1 - os.write(1, "all threads done\n") - return 0 - - def runme(no__thread): - t, cbuilder = self.compile(entry_point, no__thread=no__thread) - data = cbuilder.cmdexec('5') - assert data.splitlines() == ['hello world', - 'counter=5', - 'counter=4', - 'counter=3', - 'counter=2', - 'counter=1', - 'all threads done'] - - if SUPPORT__THREAD: - runme(no__thread=False) - #runme(no__thread=True) class TestShared(StandaloneTests): From pypy.commits at gmail.com Mon Mar 20 10:17:44 2017 From: pypy.commits at gmail.com (Raemi) Date: Mon, 20 Mar 2017 07:17:44 -0700 (PDT) Subject: [pypy-commit] pypy nogil-unsafe-2: add test for locks in write-barrier slowpath Message-ID: <58cfe488.d5c1190a.bdeed.1577@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90761:e55ed339c048 Date: 2017-03-20 15:17 +0100 http://bitbucket.org/pypy/pypy/changeset/e55ed339c048/ Log: add test for locks in write-barrier slowpath diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -860,7 +860,6 @@ major collection, and finally reserve totalsize bytes. """ - # rthread.acquire_NOAUTO(self.wb_slowpath_lock, 1) rgil.enter_master_section() minor_collection_count = 0 @@ -948,7 +947,6 @@ self.set_nursery_free(self.get_nursery_top() - self.debug_tiny_nursery) # - # rthread.release_NOAUTO(self.wb_slowpath_lock) return result collect_and_reserve._dont_inline_ = True diff --git a/rpython/translator/c/test/test_nogil.py b/rpython/translator/c/test/test_nogil.py --- a/rpython/translator/c/test/test_nogil.py +++ b/rpython/translator/c/test/test_nogil.py @@ -96,3 +96,67 @@ 'counter=2', 'counter=1', 'all threads done'] + + + def test_array_wb_slowpath(self): + import time, gc + from rpython.rlib import rthread, rposix + + class X: + def __init__(self, i): + self.i = i + + class State: pass + state = State() + + def worker(): + rthread.gc_thread_start() + + arrays = [] + for _ in range(state.arrays): + arrays.append([None]) + + x = None + xi = 0 + for i in range(1000): + xi = i + for idx in range(state.arrays): + x = X(xi) + arrays[idx][0] = x + x = None + + gc.collect(0) + + for idx in range(state.arrays): + assert arrays[idx][0].i == xi + + state.lock.acquire(True) + state.counter -= 1 + state.lock.release() + rthread.gc_thread_die() + + def entry_point(argv): + os.write(1, "hello\n") + + TS = int(argv[1]) + ARRS = int(argv[2]) + state.lock = rthread.allocate_lock() + state.counter = TS + state.arrays = ARRS + + for tid in range(TS): + rthread.start_new_thread(worker, ()) + + while True: + time.sleep(0.1) + if state.counter == 0: + break + os.write(1, "all threads done\n") + return 0 + + # + THREADS, ARRAYS = 4, 1000 + t, cbuilder = self.compile(entry_point) + data = cbuilder.cmdexec("%s %s" % (THREADS, ARRAYS)) + assert data.splitlines() == ['hello', + 'all threads done'] From pypy.commits at gmail.com Mon Mar 20 12:05:55 2017 From: pypy.commits at gmail.com (mattip) Date: Mon, 20 Mar 2017 09:05:55 -0700 (PDT) Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v5.7.0 for changeset fa3249d55d15 Message-ID: <58cffde3.45e3190a.fec1d.25b5@mx.google.com> Author: Matti Picus Branch: Changeset: r90762:07d36dd3280f Date: 2017-03-20 17:58 +0200 http://bitbucket.org/pypy/pypy/changeset/07d36dd3280f/ Log: Added tag release-pypy2.7-v5.7.0 for changeset fa3249d55d15 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -34,3 +34,4 @@ 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0 +fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0 From pypy.commits at gmail.com Mon Mar 20 12:05:58 2017 From: pypy.commits at gmail.com (pjenvey) Date: Mon, 20 Mar 2017 09:05:58 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix translation Message-ID: <58cffde6.8ed1190a.f6fd6.e46d@mx.google.com> Author: Philip Jenvey Branch: py3.5 Changeset: r90763:b1691349d5f1 Date: 2017-03-20 18:03 +0200 http://bitbucket.org/pypy/pypy/changeset/b1691349d5f1/ Log: fix translation diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -76,6 +76,8 @@ return w_dict def descr_local__new__(space, w_subtype, __args__): + from pypy.objspace.std.typeobject import _precheck_for_new + w_subtype = _precheck_for_new(space, w_subtype) if __args__.arguments_w or __args__.keywords: w_parent_init, _ = space.lookup_in_type_where(w_subtype, '__init__') if w_parent_init is space.w_object: From pypy.commits at gmail.com Mon Mar 20 12:06:01 2017 From: pypy.commits at gmail.com (mattip) Date: Mon, 20 Mar 2017 09:06:01 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into release-pypy3 Message-ID: <58cffde9.1099190a.d572f.19bf@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-5.x Changeset: r90764:b16a4363e930 Date: 2017-03-20 18:04 +0200 http://bitbucket.org/pypy/pypy/changeset/b16a4363e930/ Log: merge py3.5 into release-pypy3 diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py --- a/lib-python/3/sysconfig.py +++ b/lib-python/3/sysconfig.py @@ -44,8 +44,8 @@ 'pypy': { 'stdlib': '{installed_base}/lib-python', 'platstdlib': '{base}/lib-python', - 'purelib': '{base}/lib-python', - 'platlib': '{base}/lib-python', + 'purelib': '{base}/site-packages', + 'platlib': '{base}/site-packages', 'include': '{installed_base}/include', 'platinclude': '{installed_base}/include', 'scripts': '{base}/bin', diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -565,6 +565,7 @@ 'PyUnicode_FromFormat', 'PyUnicode_FromFormatV', 'PyUnicode_AsWideCharString', 'PyUnicode_GetSize', 'PyUnicode_GetLength', 'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant', + 'PyModule_GetDef', 'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack', '_PyArg_Parse_SizeT', '_PyArg_ParseTuple_SizeT', '_PyArg_ParseTupleAndKeywords_SizeT', '_PyArg_VaParse_SizeT', diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h --- a/pypy/module/cpyext/include/modsupport.h +++ b/pypy/module/cpyext/include/modsupport.h @@ -70,6 +70,8 @@ #define PyModule_AddIntMacro(m, c) PyModule_AddIntConstant(m, #c, c) #define PyModule_AddStringMacro(m, c) PyModule_AddStringConstant(m, #c, c) +PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*); + PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...); PyAPI_FUNC(PyObject *) Py_VaBuildValue(const char *, va_list); diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -1,8 +1,8 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cts, - parse_dir) -from pypy.module.cpyext.pyobject import PyObject, as_pyobj + parse_dir, bootstrap_function) +from pypy.module.cpyext.pyobject import PyObject, as_pyobj, make_typedescr from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, @@ -13,6 +13,11 @@ cts.parse_header(parse_dir / 'cpyext_moduleobject.h') PyModuleDef = cts.gettype('PyModuleDef *') +PyModuleObject = cts.gettype('PyModuleObject *') + + at bootstrap_function +def init_moduleobject(space): + make_typedescr(Module.typedef, basestruct=PyModuleObject.TO) @cpython_api([PyModuleDef, rffi.INT_real], PyObject) def PyModule_Create2(space, module, api_version): @@ -35,6 +40,7 @@ if f_name is not None: modname = f_name w_mod = Module(space, space.newtext(modname)) + rffi.cast(PyModuleObject, as_pyobj(space, w_mod)).c_md_def = module state.package_context = None, None if f_path is not None: diff --git a/pypy/module/cpyext/parse/cpyext_moduleobject.h b/pypy/module/cpyext/parse/cpyext_moduleobject.h --- a/pypy/module/cpyext/parse/cpyext_moduleobject.h +++ b/pypy/module/cpyext/parse/cpyext_moduleobject.h @@ -36,3 +36,9 @@ inquiry m_clear; freefunc m_free; } PyModuleDef; + +typedef struct { + PyObject_HEAD + struct PyModuleDef *md_def; + //void *md_state; +} PyModuleObject; diff --git a/pypy/module/cpyext/src/modsupport.c b/pypy/module/cpyext/src/modsupport.c --- a/pypy/module/cpyext/src/modsupport.c +++ b/pypy/module/cpyext/src/modsupport.c @@ -592,3 +592,13 @@ Py_DECREF(o); return result < 0 ? -1 : 0; } + +PyModuleDef* +PyModule_GetDef(PyObject* m) +{ + if (!PyModule_Check(m)) { + PyErr_BadArgument(); + return NULL; + } + return ((PyModuleObject *)m)->md_def; +} diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py --- a/pypy/module/cpyext/test/test_module.py +++ b/pypy/module/cpyext/test/test_module.py @@ -1,4 +1,5 @@ from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from rpython.rtyper.lltypesystem import rffi @@ -10,3 +11,19 @@ p2 = api.PyModule_GetName(w_sys) assert p2 == p self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True) + + +class AppTestModuleObject(AppTestCpythonExtensionBase): + def test_getdef(self): + module = self.import_extension('foo', [ + ("check_getdef_same", "METH_NOARGS", + """ + return PyBool_FromLong(PyModule_GetDef(mod_global) == &moduledef); + """ + )], prologue=""" + static struct PyModuleDef moduledef; + static PyObject *mod_global; + """, more_init=""" + mod_global = mod; + """) + assert module.check_getdef_same() diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -1,6 +1,7 @@ import weakref from rpython.rlib import jit from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import oefmt from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty, descr_get_dict) @@ -75,18 +76,20 @@ return w_dict def descr_local__new__(space, w_subtype, __args__): + from pypy.objspace.std.typeobject import _precheck_for_new + w_subtype = _precheck_for_new(space, w_subtype) + if __args__.arguments_w or __args__.keywords: + w_parent_init, _ = space.lookup_in_type_where(w_subtype, '__init__') + if w_parent_init is space.w_object: + raise oefmt(space.w_TypeError, + "Initialization arguments are not supported") local = space.allocate_instance(Local, w_subtype) Local.__init__(local, space, __args__) return local - def descr_local__init__(self, space): - # No arguments allowed - pass - Local.typedef = TypeDef("thread._local", __doc__ = "Thread-local data", __new__ = interp2app(Local.descr_local__new__.im_func), - __init__ = interp2app(Local.descr_local__init__), __dict__ = GetSetProperty(descr_get_dict, cls=Local), ) diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -72,6 +72,19 @@ assert seen1 == [1, 2, 3, 4, 5] assert tags == ['???'] + def test_local_init2(self): + import thread + + class A(object): + def __init__(self, n): + assert n == 42 + self.n = n + class X(thread._local, A): + pass + + x = X(42) + assert x.n == 42 + def test_local_setdict(self): import _thread x = _thread._local() From pypy.commits at gmail.com Mon Mar 20 12:43:09 2017 From: pypy.commits at gmail.com (Raemi) Date: Mon, 20 Mar 2017 09:43:09 -0700 (PDT) Subject: [pypy-commit] pypy nogil-unsafe-2: small cleanup Message-ID: <58d0069d.09132e0a.5c69b.1fcf@mx.google.com> Author: Remi Meier Branch: nogil-unsafe-2 Changeset: r90765:d29b5e7da8b0 Date: 2017-03-20 17:42 +0100 http://bitbucket.org/pypy/pypy/changeset/d29b5e7da8b0/ Log: small cleanup diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -439,10 +439,7 @@ self.updated_old_objects_pointing_to_pinned = False # # Allocate lock(s) - wb_slowpath_lock = lltype.malloc(rthread.TLOCKP.TO, flavor='raw', - track_allocation=False) - rthread.c_thread_lock_init(wb_slowpath_lock) - self.wb_slowpath_lock = wb_slowpath_lock + self.wb_slowpath_lock = rthread.allocate_ll_lock_NOAUTO() # # Allocate a nursery. In case of auto_nursery_size, start by # allocating a very small nursery, enough to do things like look diff --git a/rpython/memory/gc/test/test_thread.py b/rpython/memory/gc/test/test_thread.py --- a/rpython/memory/gc/test/test_thread.py +++ b/rpython/memory/gc/test/test_thread.py @@ -11,7 +11,6 @@ class ThreadSwitcher: all_threadlocals = [incminimark.NURSERY_FREE, incminimark.NURSERY_TOP, - incminimark.NEXT_NUBLOCK, ] def __init__(self, gc): diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -59,6 +59,9 @@ TLOCKP_SIZE = rffi_platform.sizeof('struct RPyOpaque_ThreadLock', eci) c_thread_lock_init = llexternal('RPyThreadLockInit', [TLOCKP], rffi.INT, releasegil=False) # may add in a global list +c_thread_lock_init_NOAUTO = llexternal('RPyThreadLockInit', [TLOCKP], rffi.INT, + _nowrapper=True) # may add in a global list! + c_thread_lock_dealloc_NOAUTO = llexternal('RPyOpaqueDealloc_ThreadLock', [TLOCKP], lltype.Void, _nowrapper=True) @@ -248,6 +251,13 @@ rgc.add_memory_pressure(TLOCKP_SIZE) return ll_lock + +def allocate_ll_lock_NOAUTO(): + ll_lock = lltype.malloc(TLOCKP.TO, flavor='raw', track_allocation=False) + c_thread_lock_init_NOAUTO(ll_lock) + return ll_lock + + def free_ll_lock(ll_lock): acquire_NOAUTO(ll_lock, False) release_NOAUTO(ll_lock) diff --git a/rpython/translator/c/test/test_nogil.py b/rpython/translator/c/test/test_nogil.py --- a/rpython/translator/c/test/test_nogil.py +++ b/rpython/translator/c/test/test_nogil.py @@ -112,6 +112,7 @@ def worker(): rthread.gc_thread_start() + # these arrays should be old soon: arrays = [] for _ in range(state.arrays): arrays.append([None]) @@ -120,6 +121,7 @@ xi = 0 for i in range(1000): xi = i + # put a young X() in each old array for idx in range(state.arrays): x = X(xi) arrays[idx][0] = x @@ -127,6 +129,7 @@ gc.collect(0) + # check that Xs are not lost for idx in range(state.arrays): assert arrays[idx][0].i == xi From pypy.commits at gmail.com Mon Mar 20 13:12:41 2017 From: pypy.commits at gmail.com (mattip) Date: Mon, 20 Mar 2017 10:12:41 -0700 (PDT) Subject: [pypy-commit] pypy default: Added tag release-pypy3.5-v5.7.0 for changeset b16a4363e930 Message-ID: <58d00d89.89012e0a.9670a.bedd@mx.google.com> Author: Matti Picus Branch: Changeset: r90766:8fc1737ccf20 Date: 2017-03-20 19:09 +0200 http://bitbucket.org/pypy/pypy/changeset/8fc1737ccf20/ Log: Added tag release-pypy3.5-v5.7.0 for changeset b16a4363e930 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -35,3 +35,4 @@ 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0 fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0 +b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0 From pypy.commits at gmail.com Mon Mar 20 13:42:44 2017 From: pypy.commits at gmail.com (mattip) Date: Mon, 20 Mar 2017 10:42:44 -0700 (PDT) Subject: [pypy-commit] pypy default: tweak script and release notice Message-ID: <58d01494.0e472e0a.1db99.78c8@mx.google.com> Author: Matti Picus Branch: Changeset: r90767:23145de5988b Date: 2017-03-20 19:39 +0200 http://bitbucket.org/pypy/pypy/changeset/23145de5988b/ Log: tweak script and release notice diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst --- a/pypy/doc/release-v5.7.0.rst +++ b/pypy/doc/release-v5.7.0.rst @@ -2,8 +2,11 @@ PyPy2.7 and PyPy3.5 v5.7 - two in one release ============================================= -We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for -Linux 64bit only at first). +The PyPy team is proud to release both PyPy2.7 v5.7 (an interpreter supporting +Python v2.7 syntax), and a beta-quality PyPy3.5 v5.7 (an interpreter for Python +v3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. Note that PyPy3.5 supports Linux 64bit only for now. + This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version 3.5.3. diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,6 +1,6 @@ # Edit these appropriately before running this script maj=5 -min=6 +min=7 rev=0 branchname=release-pypy2.7-5.x # ==OR== release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy2.7-v$maj.$min.$rev # ==OR== release-$maj.$min From pypy.commits at gmail.com Mon Mar 20 14:00:04 2017 From: pypy.commits at gmail.com (mattip) Date: Mon, 20 Mar 2017 11:00:04 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update download page for release Message-ID: <58d018a4.c4052e0a.30073.2617@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r868:fa787b19005c Date: 2017-03-20 19:59 +0200 http://bitbucket.org/pypy/pypy.org/changeset/fa787b19005c/ Log: update download page for release diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -77,8 +77,8 @@

    We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as Linux, Mac OS/X and Windows:

      -
    • the Python2.7 compatible release — PyPy2.7 v5.6.0 — (what's new in PyPy2.7?)
      • -
    • the Python3.3 compatible release — PyPy3.3 v5.5 — (what's new in PyPy3.3?).
      • +
    • the Python2.7 compatible release — PyPy2.7 v5.7.0 — (what's new in PyPy2.7?)
      • +
    • the Python3.5 compatible beta quality release — PyPy3.5 v5.7.0 — (what's new in PyPy3.5?).
    • the Python2.7 Software Transactional Memory special release — PyPy-STM 2.5.1 (Linux x86-64 only)
      @@ -117,45 +117,36 @@
    • or translate your own PyPy.
    -
    -

    Python2.7 compatible PyPy 5.6.0

    +
    +

    Python2.7 compatible PyPy 5.7.0

    -
    -

    Python 3.3.5 compatible PyPy3.3 v5.5

    -

    Warning: PyPy3.3 is considered alpha/beta software. All binaries -are thus called “alpha”. It is known to be sometimes much slower than +

    +

    Python 3.5.3 compatible PyPy3.5 v5.7

    +

    Warning: PyPy3.5 is considered beta software. All binaries +are thus called “beta”. It is known to be rarely much slower than PyPy 2. You are welcome to use it anyway; if you're lucky it will -be fast in your case. We are currently working on PyPy3.5 supporting -Python 3.5.x, but it is not ready for release yet.

    +be fast in your case.

    @@ -209,7 +200,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in /opt, and if you want, put a symlink from somewhere like -/usr/local/bin/pypy to /path/to/pypy2-5.6.0/bin/pypy. Do +/usr/local/bin/pypy to /path/to/pypy2-5.7.0/bin/pypy. Do not move or copy the executable pypy outside the tree – put a symlink to it, otherwise it will not find its libraries.

    @@ -289,7 +280,7 @@

    Alternatively, the following smaller package contains the source at the same revision as the above binaries:

  • Make sure you installed the dependencies. See the list here.

    @@ -301,7 +292,7 @@

  • Run the rpython script. Here are the common combinations of options (works also with python instead of pypy; -requires Python 2.x or PyPy 2):

    +requires CPython 2.7 or PyPy 2):

     pypy ../../rpython/bin/rpython -Ojit targetpypystandalone           # get the JIT version
 pypy ../../rpython/bin/rpython -O2 targetpypystandalone             # get the no-jit version
@@ -352,12 +343,12 @@
 .../pypy/tool/build_cffi_imports.py if you want to be able to import
 the cffi-based modules.
    • -

  • On Linux, because of asmgcroot, compiling the generated C files -is delicate. It requires using gcc with no particularly +

  • On Linux, translating with asmgcroot, is delicate. +It requires using gcc with no particularly fancy options. It does not work e.g. with clang, or if you pass uncommon options with the CFLAGS environment variable. If you insist on passing these options or using clang, then you can compile PyPy with -the shadow stack option instead (for some performance price in +the default shadow stack option instead (for a small performance price in non-JITted code).

  • Like other JITs, PyPy doesn't work out of the box on some Linux @@ -395,7 +386,7 @@ see the exact list in package.py). Users seeing a broken installation of PyPy can fix it after-the-fact, by running pypy /path/to/lib_pypy/_gdbm_build.py. This command produces a file -called _gdbm_cffi.pypy-26.so locally, which is a C extension +called _gdbm_cffi.pypy-41.so locally, which is a C extension module for PyPy. You can move it at any place where modules are normally found: e.g. in your project's main directory, or in a directory that you add to the env var PYTHONPATH.

    • @@ -404,21 +395,24 @@

    Checksums

    Here are the checksums for each of the downloads

    -

    pypy2.7-v5.6.0 md5:

    +

    pypy2.7-v5.7.0 md5:

    -f804f1ee631125aa0a3c8eb8a9f09f20  pypy2-v5.6.0-linux32.tar.bz2
-4d7514f43c5e92e2d066480db86d1932  pypy2-v5.6.0-linux64.tar.bz2
-112b340c407f406dbf79ce715f111467  pypy2-v5.6.0-linux-armel.tar.bz2
-736bd4fb085959394c97df5e40bec6e6  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
-65beda937953e697f212c9acfde91e8c  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
-2d59a3fb9abdc224193b4206814fb5ad  pypy2-v5.6.0-osx64.tar.bz2
-6aaf25bf60e7a37a2a920eaa7b0e35c5  pypy2-v5.6.0-ppc64.tar.bz2
-57557db44b623047062e2fbd0628dfed  pypy2-v5.6.0-ppc64le.tar.bz2
-f027e7818a1c4a8ad14259f8bc6cbeec  pypy2-v5.6.0-s390x.tar.bz2
-c3fc7187061fec762269496f1f5daa86  pypy2-v5.6.0-src.tar.bz2
-d5c3a0cce5f07c739b9f6e63b55247e8  pypy2-v5.6.0-src.zip
-3579eeea2caa74accf5d35a8cb406a83  pypy2-v5.6.0-win32.zip
+bb55f4a28d64d40367a58139b25b02fe  pypy2-v5.7.0-linux32.tar.bz2
+84b542e3aaca2aec9b0ca3afb6a3373b  pypy2-v5.7.0-linux64.tar.bz2
+9f98c917ec1dbbda821bd23734fdc4d4  pypy2-v5.7.0-linux-armel.tar.bz2
+509539d3f5876584fa5cad6fd125e446  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
+34739358ba2e438d27b777838154f255  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
+6b6180f4b0932d5a3a7499ddef1c5129  pypy2-v5.7.0-osx64.tar.bz2
+180d5cc3f58a9f9088481c7d4c6b5d68  pypy2-v5.7.0-s390x.tar.bz2
+aa26e72c1f30b77734198af6fb6f9ad7  pypy2-v5.7.0-src.tar.bz2
+f1633ad18e0af309aa11e3425f628cf9  pypy2-v5.7.0-src.zip
+23f15d2bc0c29fe6948a99352962ebf0  pypy2-v5.7.0-win32.zip
    +

    pypy 3.5-v5.7.0 md5:

    +
    +410fb506ac436c344f785bb77820e13e pypy3-v5.7.0-linux64.tar.bz2 +79b1d5749eb97be36933dbdf156fec02 pypy3-v5.7.0-src.tar.bz2 +594f3d17be1c80181d13c0f025ae25f6 pypy3-v5.7.0-src.zip

    pypy3.3-v5.5.0-alpha md5:

     e9bdfb330a30765eaf76ddd4e826a45e  pypy3.3-v5.5.0-alpha-linux32.tar.bz2
@@ -437,19 +431,22 @@

    pypy2.7-5.6.0 sha1:

    -78124bd560ab5bd4902a4f55e776c917d851736c  pypy2-v5.6.0-linux32.tar.bz2
-6a5a254ceeb92b0bb9c2e3bfdcaf5a0d9f34df83  pypy2-v5.6.0-linux64.tar.bz2
-8b452ae867f0a5131427386263abe458bf8db845  pypy2-v5.6.0-linux-armel.tar.bz2
-190295288f9b6f938b112109f6594235a14f1df3  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
-0004ccae2872b106b46f479a94c43c8dd7811eba  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
-5021b044ddf68155ca1bb2b41acc5b41d44dd2db  pypy2-v5.6.0-osx64.tar.bz2
-94f4cd90a98425e30dfccab8f9704328da25a6a2  pypy2-v5.6.0-ppc64.tar.bz2
-d3e39c88d8711b49c6584508c6966a80d5cb772c  pypy2-v5.6.0-ppc64le.tar.bz2
-35056ff8fba233ecd896da226bf0949b83dfb2ca  pypy2-v5.6.0-s390x.tar.bz2
-ecfd7efdee340ee170db56a79c18bde3e37cd326  pypy2-v5.6.0-src.tar.bz2
-bf805de05a11bda6182efd7f14c3b6b1945a6f9a  pypy2-v5.6.0-src.zip
-a7d5d5287c84df3cbbebadee3c455cd4f88528d6  pypy2-v5.6.0-win32.zip
+c8cc2cbc2b75962d0e5920f4a756f7fe3f13596c  pypy2-v5.7.0-linux32.tar.bz2
+0a831dae298fce2110626637342d0e3545a8ab47  pypy2-v5.7.0-linux64.tar.bz2
+556face1374a75720177176a4c7517bb4f9a62a5  pypy2-v5.7.0-linux-armel.tar.bz2
+bec04d53cd2a7cc89cf67cfdab2207ad6c08781b  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
+c510f53447fc800b4611545d2679d596d8f0c461  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
+097e418d5aa61c7697a50536c5b173d8211603a9  pypy2-v5.7.0-osx64.tar.bz2
+20f33a4d0c08a5d4935086caec73be3d92b60b2b  pypy2-v5.7.0-s390x.tar.bz2
+5976e1f19a42f64f778d385322723200b5c6d7c9  pypy2-v5.7.0-src.tar.bz2
+c84ad90184edb315031074696e805ecbc08494e3  pypy2-v5.7.0-src.zip
+15df479b08870dc6ec9cab0e4587935fc0694cb8  pypy2-v5.7.0-win32.zip
    +

    pypy 3.5-v5.7.0 sha1:

    +
    +e10511bde5a88c5de9fdf9234188b8df23382a6a pypy3-v5.7.0-linux64.tar.bz2 +0b12230025d77be26bc10cbc81671c4b3719528c pypy3-v5.7.0-src.tar.bz2 +ae256d83173faaadce9602594fd752fb664b813b pypy3-v5.7.0-src.zip

    pypy3.3-v5.5.0-alpha sha1:

     1de17b61b8d21e408a46114961659b4c0f15cb54  pypy3.3-v5.5.0-alpha-linux32.tar.bz2
@@ -463,19 +460,22 @@

    pypy2.7-5.6.0 sha256:

    -5d4ad43aed5c5e147f7e7c84766c729f34f63b714b6d224e912a2bb42dc95d62  pypy2-v5.6.0-linux32.tar.bz2
-aad55328cb0673a60b2633dcc3c36cf452917ac906b577eb3aed5876a7666fca  pypy2-v5.6.0-linux64.tar.bz2
-2d1c7820f6368c92bb43a153d2c995f70aa183ff8f1df6916b0d2e57838d8a30  pypy2-v5.6.0-linux-armel.tar.bz2
-2c430240cecb562102c677598f106aa57899f00cd37f719989e18ed9ca44bd01  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
-0f69c40a38d72254bf12094620bda9d2face758f763cd0d989588642d81eccae  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
-6121f791f440564d3a0a41315e9f2d2d61bc484654acffd85d9e1c6e92b85c36  pypy2-v5.6.0-osx64.tar.bz2
-51bbe172b7b631fa48fab6a2bd2b5411f99a87cb7ebb47ed0a29affd147667c6  pypy2-v5.6.0-ppc64.tar.bz2
-81a82bcb87cb95101e6019a404eca956e4f18a685f588ae88720026641c40d0f  pypy2-v5.6.0-ppc64le.tar.bz2
-534284c5a35da647f623d1937f99ae720927de8b4b1ce20143d860050f644376  pypy2-v5.6.0-s390x.tar.bz2
-7411448045f77eb9e087afdce66fe7eafda1876c9e17aad88cf891f762b608b0  pypy2-v5.6.0-src.tar.bz2
-f9f86ac8267d9997c314d1345d09a20a4528352285ed5622e35cf6e79480fe28  pypy2-v5.6.0-src.zip
-bab4fa37ef2d32660e291393d955a4e951d5e883abea8bee83be1ec044ddcaac  pypy2-v5.6.0-win32.zip
+aaa55085d11a49cb982b059b4159495d7a4fb9afcfda7bfd680cc0175e9fa33b  pypy2-v5.7.0-linux32.tar.bz2
+64bed80e299b09c13296f577a0f52c5d4be9f7c699a390ca6026f967aeff3846  pypy2-v5.7.0-linux64.tar.bz2
+3bc0d2616cacef9f544e739d4c25a89e16d7aa8c1ccb18c32dceeb021ed73711  pypy2-v5.7.0-linux-armel.tar.bz2
+413df8097c87a8e79cc46df65837b1d533de83b7cb06ce88b168697807fff696  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
+6328e4767cf20d20a290739db2d1ec769f3c9c360fa9824f61581b3356add79b  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
+9204aa1b55771a374a3118bb43e498c88caca8c33710eecf3249855c4dd1352a  pypy2-v5.7.0-osx64.tar.bz2
+8820b36f38b4aa44c1ef2be213c7d13818c9fdb61795f24f4fb18a7a78d2dde7  pypy2-v5.7.0-s390x.tar.bz2
+e522ea7ca51b16ee5505f22b86803664b762a263a6d69ba84c359fcf8365ad3e  pypy2-v5.7.0-src.tar.bz2
+784be3d2b8bb12a0d88d088fd991b03324d9f84a48af5dbaf2a536eaee91d8b5  pypy2-v5.7.0-src.zip
+09c8da3a7e09cea821f9b300c6f4f52f4a571d949d91839ecbce93b84a08d570  pypy2-v5.7.0-win32.zip
    +

    pypy 3.5-v5.7.0 sha256:

    +
    +921894884a647220a712ecdaad516d9c22fbadf3b4bb3a5db8f3635c60eabc7b pypy3-v5.7.0-linux64.tar.bz2 +f0f563b74f8b82ec33b022393219b93cc0d81e9f9500614fe8417b67a52e9569 pypy3-v5.7.0-src.tar.bz2 +fe8b9804a8599ef5b16114fb64ba8392342f0066dfe81103b89198a3c5d10989 pypy3-v5.7.0-src.zip

    pypy3.3-v5.5.0-alpha sha256:

     966ee7951ad497ac907e01554fe48da77cc64a5e35a1307477c2f78652eba622  pypy3.3-v5.5.0-alpha-linux32.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -15,14 +15,14 @@
 We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
 Linux, Mac OS/X and Windows:
 
-* the Python2.7 compatible release — **PyPy2.7 v5.6.0** — (`what's new in PyPy2.7?`_)
+* the Python2.7 compatible release — **PyPy2.7 v5.7.0** — (`what's new in PyPy2.7?`_)
 
-* the Python3.3 compatible release — **PyPy3.3 v5.5** — (`what's new in PyPy3.3?`_).
+* the Python3.5 compatible beta quality release — **PyPy3.5 v5.7.0** — (`what's new in PyPy3.5?`_).
 
 * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only)
 
-.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html
-.. _what's new in PyPy3.3?: http://doc.pypy.org/en/latest/release-pypy3.3-v5.5.0.html
+.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-v5.7.0.html
+.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v5.7.0.html
 
 
 .. class:: download_menu
@@ -79,7 +79,7 @@
 
 .. _release:
 
-Python2.7 compatible PyPy 5.6.0
+Python2.7 compatible PyPy 5.7.0
 -------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -98,60 +98,42 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux32.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-ppc64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-ppc64le.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
 .. _FreshPorts: http://www.freshports.org/lang/pypy
 
-Python 3.3.5 compatible PyPy3.3 v5.5
--------------------------------------
+Python 3.5.3 compatible PyPy3.5 v5.7
+------------------------------------
 
 .. class:: download_menu
 
-    Warning: PyPy3.3 is considered **alpha/beta software.**  All binaries
-    are thus called "alpha".  It is known to be sometimes much slower than
+    Warning: PyPy3.5 is considered **beta software.**  All binaries
+    are thus called "beta".  It is known to be rarely much slower than
     PyPy 2.  You are welcome to use it anyway; if you're lucky it will
-    be fast in your case.  We are currently working on `PyPy3.5 supporting
-    Python 3.5.x`__, but it is not ready for release yet.
+    be fast in your case.  
 
-.. __: http://buildbot.pypy.org/nightly/py3.5/
-
-* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
 * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
-* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below)
-* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below)
-* `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)`__ (see ``[1]`` below)
-* `Mac OS/X binary (64bit)`__
-* Windows binary (32bit) (hopefully availabe soon) (you might need the `VS 2008 runtime library
-  installer vcredist_x86.exe`_.)
-* `s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)`__ (see ``[1]`` below)
 * `Source (tar.bz2)`__
 * `Source (zip)`__
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-osx64.tar.bz2
-.. _`VS 2008 runtime library installer vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.7.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.7.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.7.0-src.zip
 .. __: https://bitbucket.org/pypy/pypy/downloads
 
 If your CPU is really, really old, it may be a x86-32 without SSE2.
@@ -221,7 +203,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.6.0/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.7.0/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -324,9 +306,9 @@
    Alternatively, the following smaller package contains the source at
    the same revision as the above binaries:
 
-   * `pypy2-v5.6.0-src.tar.bz2`__ (sources)
+   * `pypy2-v5.7.0-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-src.tar.bz2
 
 
 2. Make sure you **installed the dependencies.**  See the list here__.
@@ -339,7 +321,7 @@
 
 4. Run the ``rpython`` script.  Here are the common combinations
    of options (works also with ``python`` instead of ``pypy``;
-   requires Python 2.x or PyPy 2)::
+   requires CPython 2.7 or PyPy 2)::
 
      pypy ../../rpython/bin/rpython -Ojit targetpypystandalone           # get the JIT version
      pypy ../../rpython/bin/rpython -O2 targetpypystandalone             # get the no-jit version
@@ -389,12 +371,12 @@
   .../pypy/tool/build_cffi_imports.py`` if you want to be able to import
   the cffi-based modules.
 
-* On Linux, because of ``asmgcroot``, compiling the generated C files
-  is delicate.  It requires using gcc with no particularly
+* On Linux, translating with ``asmgcroot``, is delicate.  
+  It requires using gcc with no particularly
   fancy options.  It does not work e.g. with clang, or if you pass uncommon
   options with the ``CFLAGS`` environment variable.  If you insist on
   passing these options or using clang, then you can compile PyPy with
-  the `shadow stack`_ option instead (for some performance price in
+  the default `shadow stack`_ option instead (for a small performance price in
   non-JITted code).
 
 * Like other JITs, PyPy doesn't work out of the box on some Linux
@@ -444,7 +426,7 @@
   see the exact list in `package.py`_).  Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
-  called ``_gdbm_cffi.pypy-26.so`` locally, which is a C extension
+  called ``_gdbm_cffi.pypy-41.so`` locally, which is a C extension
   module for PyPy.  You can move it at any place where modules are
   normally found: e.g. in your project's main directory, or in a
   directory that you add to the env var ``PYTHONPATH``.
@@ -456,20 +438,24 @@
 
 Here are the checksums for each of the downloads
 
-pypy2.7-v5.6.0 md5::
+pypy2.7-v5.7.0 md5::
 
-    f804f1ee631125aa0a3c8eb8a9f09f20  pypy2-v5.6.0-linux32.tar.bz2
-    4d7514f43c5e92e2d066480db86d1932  pypy2-v5.6.0-linux64.tar.bz2
-    112b340c407f406dbf79ce715f111467  pypy2-v5.6.0-linux-armel.tar.bz2
-    736bd4fb085959394c97df5e40bec6e6  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
-    65beda937953e697f212c9acfde91e8c  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
-    2d59a3fb9abdc224193b4206814fb5ad  pypy2-v5.6.0-osx64.tar.bz2
-    6aaf25bf60e7a37a2a920eaa7b0e35c5  pypy2-v5.6.0-ppc64.tar.bz2
-    57557db44b623047062e2fbd0628dfed  pypy2-v5.6.0-ppc64le.tar.bz2
-    f027e7818a1c4a8ad14259f8bc6cbeec  pypy2-v5.6.0-s390x.tar.bz2
-    c3fc7187061fec762269496f1f5daa86  pypy2-v5.6.0-src.tar.bz2
-    d5c3a0cce5f07c739b9f6e63b55247e8  pypy2-v5.6.0-src.zip
-    3579eeea2caa74accf5d35a8cb406a83  pypy2-v5.6.0-win32.zip
+    bb55f4a28d64d40367a58139b25b02fe  pypy2-v5.7.0-linux32.tar.bz2
+    84b542e3aaca2aec9b0ca3afb6a3373b  pypy2-v5.7.0-linux64.tar.bz2
+    9f98c917ec1dbbda821bd23734fdc4d4  pypy2-v5.7.0-linux-armel.tar.bz2
+    509539d3f5876584fa5cad6fd125e446  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
+    34739358ba2e438d27b777838154f255  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
+    6b6180f4b0932d5a3a7499ddef1c5129  pypy2-v5.7.0-osx64.tar.bz2
+    180d5cc3f58a9f9088481c7d4c6b5d68  pypy2-v5.7.0-s390x.tar.bz2
+    aa26e72c1f30b77734198af6fb6f9ad7  pypy2-v5.7.0-src.tar.bz2
+    f1633ad18e0af309aa11e3425f628cf9  pypy2-v5.7.0-src.zip
+    23f15d2bc0c29fe6948a99352962ebf0  pypy2-v5.7.0-win32.zip
+
+pypy 3.5-v5.7.0 md5:
+
+    410fb506ac436c344f785bb77820e13e  pypy3-v5.7.0-linux64.tar.bz2
+    79b1d5749eb97be36933dbdf156fec02  pypy3-v5.7.0-src.tar.bz2
+    594f3d17be1c80181d13c0f025ae25f6  pypy3-v5.7.0-src.zip
 
 pypy3.3-v5.5.0-alpha md5::
 
@@ -489,18 +475,22 @@
 
 pypy2.7-5.6.0 sha1::
 
-    78124bd560ab5bd4902a4f55e776c917d851736c  pypy2-v5.6.0-linux32.tar.bz2
-    6a5a254ceeb92b0bb9c2e3bfdcaf5a0d9f34df83  pypy2-v5.6.0-linux64.tar.bz2
-    8b452ae867f0a5131427386263abe458bf8db845  pypy2-v5.6.0-linux-armel.tar.bz2
-    190295288f9b6f938b112109f6594235a14f1df3  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
-    0004ccae2872b106b46f479a94c43c8dd7811eba  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
-    5021b044ddf68155ca1bb2b41acc5b41d44dd2db  pypy2-v5.6.0-osx64.tar.bz2
-    94f4cd90a98425e30dfccab8f9704328da25a6a2  pypy2-v5.6.0-ppc64.tar.bz2
-    d3e39c88d8711b49c6584508c6966a80d5cb772c  pypy2-v5.6.0-ppc64le.tar.bz2
-    35056ff8fba233ecd896da226bf0949b83dfb2ca  pypy2-v5.6.0-s390x.tar.bz2
-    ecfd7efdee340ee170db56a79c18bde3e37cd326  pypy2-v5.6.0-src.tar.bz2
-    bf805de05a11bda6182efd7f14c3b6b1945a6f9a  pypy2-v5.6.0-src.zip
-    a7d5d5287c84df3cbbebadee3c455cd4f88528d6  pypy2-v5.6.0-win32.zip
+    c8cc2cbc2b75962d0e5920f4a756f7fe3f13596c  pypy2-v5.7.0-linux32.tar.bz2
+    0a831dae298fce2110626637342d0e3545a8ab47  pypy2-v5.7.0-linux64.tar.bz2
+    556face1374a75720177176a4c7517bb4f9a62a5  pypy2-v5.7.0-linux-armel.tar.bz2
+    bec04d53cd2a7cc89cf67cfdab2207ad6c08781b  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
+    c510f53447fc800b4611545d2679d596d8f0c461  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
+    097e418d5aa61c7697a50536c5b173d8211603a9  pypy2-v5.7.0-osx64.tar.bz2
+    20f33a4d0c08a5d4935086caec73be3d92b60b2b  pypy2-v5.7.0-s390x.tar.bz2
+    5976e1f19a42f64f778d385322723200b5c6d7c9  pypy2-v5.7.0-src.tar.bz2
+    c84ad90184edb315031074696e805ecbc08494e3  pypy2-v5.7.0-src.zip
+    15df479b08870dc6ec9cab0e4587935fc0694cb8  pypy2-v5.7.0-win32.zip
+
+pypy 3.5-v5.7.0 sha1:
+
+    e10511bde5a88c5de9fdf9234188b8df23382a6a  pypy3-v5.7.0-linux64.tar.bz2
+    0b12230025d77be26bc10cbc81671c4b3719528c  pypy3-v5.7.0-src.tar.bz2
+    ae256d83173faaadce9602594fd752fb664b813b  pypy3-v5.7.0-src.zip
 
 pypy3.3-v5.5.0-alpha sha1::
 
@@ -515,18 +505,22 @@
 
 pypy2.7-5.6.0 sha256::
 
-    5d4ad43aed5c5e147f7e7c84766c729f34f63b714b6d224e912a2bb42dc95d62  pypy2-v5.6.0-linux32.tar.bz2
-    aad55328cb0673a60b2633dcc3c36cf452917ac906b577eb3aed5876a7666fca  pypy2-v5.6.0-linux64.tar.bz2
-    2d1c7820f6368c92bb43a153d2c995f70aa183ff8f1df6916b0d2e57838d8a30  pypy2-v5.6.0-linux-armel.tar.bz2
-    2c430240cecb562102c677598f106aa57899f00cd37f719989e18ed9ca44bd01  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
-    0f69c40a38d72254bf12094620bda9d2face758f763cd0d989588642d81eccae  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
-    6121f791f440564d3a0a41315e9f2d2d61bc484654acffd85d9e1c6e92b85c36  pypy2-v5.6.0-osx64.tar.bz2
-    51bbe172b7b631fa48fab6a2bd2b5411f99a87cb7ebb47ed0a29affd147667c6  pypy2-v5.6.0-ppc64.tar.bz2
-    81a82bcb87cb95101e6019a404eca956e4f18a685f588ae88720026641c40d0f  pypy2-v5.6.0-ppc64le.tar.bz2
-    534284c5a35da647f623d1937f99ae720927de8b4b1ce20143d860050f644376  pypy2-v5.6.0-s390x.tar.bz2
-    7411448045f77eb9e087afdce66fe7eafda1876c9e17aad88cf891f762b608b0  pypy2-v5.6.0-src.tar.bz2
-    f9f86ac8267d9997c314d1345d09a20a4528352285ed5622e35cf6e79480fe28  pypy2-v5.6.0-src.zip
-    bab4fa37ef2d32660e291393d955a4e951d5e883abea8bee83be1ec044ddcaac  pypy2-v5.6.0-win32.zip
+    aaa55085d11a49cb982b059b4159495d7a4fb9afcfda7bfd680cc0175e9fa33b  pypy2-v5.7.0-linux32.tar.bz2
+    64bed80e299b09c13296f577a0f52c5d4be9f7c699a390ca6026f967aeff3846  pypy2-v5.7.0-linux64.tar.bz2
+    3bc0d2616cacef9f544e739d4c25a89e16d7aa8c1ccb18c32dceeb021ed73711  pypy2-v5.7.0-linux-armel.tar.bz2
+    413df8097c87a8e79cc46df65837b1d533de83b7cb06ce88b168697807fff696  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
+    6328e4767cf20d20a290739db2d1ec769f3c9c360fa9824f61581b3356add79b  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
+    9204aa1b55771a374a3118bb43e498c88caca8c33710eecf3249855c4dd1352a  pypy2-v5.7.0-osx64.tar.bz2
+    8820b36f38b4aa44c1ef2be213c7d13818c9fdb61795f24f4fb18a7a78d2dde7  pypy2-v5.7.0-s390x.tar.bz2
+    e522ea7ca51b16ee5505f22b86803664b762a263a6d69ba84c359fcf8365ad3e  pypy2-v5.7.0-src.tar.bz2
+    784be3d2b8bb12a0d88d088fd991b03324d9f84a48af5dbaf2a536eaee91d8b5  pypy2-v5.7.0-src.zip
+    09c8da3a7e09cea821f9b300c6f4f52f4a571d949d91839ecbce93b84a08d570  pypy2-v5.7.0-win32.zip
+
+pypy 3.5-v5.7.0 sha256:
+
+    921894884a647220a712ecdaad516d9c22fbadf3b4bb3a5db8f3635c60eabc7b  pypy3-v5.7.0-linux64.tar.bz2
+    f0f563b74f8b82ec33b022393219b93cc0d81e9f9500614fe8417b67a52e9569  pypy3-v5.7.0-src.tar.bz2
+    fe8b9804a8599ef5b16114fb64ba8392342f0066dfe81103b89198a3c5d10989  pypy3-v5.7.0-src.zip
 
 pypy3.3-v5.5.0-alpha sha256::
 

From pypy.commits at gmail.com  Mon Mar 20 15:08:13 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 Mar 2017 12:08:13 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: formatting typo,
 make box a wee bit bigger for sha256
Message-ID: <58d0289d.815c2e0a.b82a1.2890@mx.google.com>

Author: Matti Picus 
Branch: extradoc
Changeset: r869:a33e1e498f71
Date: 2017-03-20 21:07 +0200
http://bitbucket.org/pypy/pypy.org/changeset/a33e1e498f71/

Log:	formatting typo, make box a wee bit bigger for sha256

diff --git a/css/site.css b/css/site.css
--- a/css/site.css
+++ b/css/site.css
@@ -223,7 +223,7 @@
 #main {
 	float: left;
 	padding: 10px 30px 0 10px;
-	width: 800px;
+	width: 820px;
 	line-height: 2em;
 	font-size: 0.9em;
 }
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -409,10 +409,11 @@
 23f15d2bc0c29fe6948a99352962ebf0  pypy2-v5.7.0-win32.zip

    pypy 3.5-v5.7.0 md5:

    -
    +
     410fb506ac436c344f785bb77820e13e  pypy3-v5.7.0-linux64.tar.bz2
 79b1d5749eb97be36933dbdf156fec02  pypy3-v5.7.0-src.tar.bz2
-594f3d17be1c80181d13c0f025ae25f6  pypy3-v5.7.0-src.zip
    +594f3d17be1c80181d13c0f025ae25f6 pypy3-v5.7.0-src.zip +

    pypy3.3-v5.5.0-alpha md5:

     e9bdfb330a30765eaf76ddd4e826a45e  pypy3.3-v5.5.0-alpha-linux32.tar.bz2
@@ -443,10 +444,11 @@
 15df479b08870dc6ec9cab0e4587935fc0694cb8  pypy2-v5.7.0-win32.zip

    pypy 3.5-v5.7.0 sha1:

    -
    +
     e10511bde5a88c5de9fdf9234188b8df23382a6a  pypy3-v5.7.0-linux64.tar.bz2
 0b12230025d77be26bc10cbc81671c4b3719528c  pypy3-v5.7.0-src.tar.bz2
-ae256d83173faaadce9602594fd752fb664b813b  pypy3-v5.7.0-src.zip
    +ae256d83173faaadce9602594fd752fb664b813b pypy3-v5.7.0-src.zip +

    pypy3.3-v5.5.0-alpha sha1:

     1de17b61b8d21e408a46114961659b4c0f15cb54  pypy3.3-v5.5.0-alpha-linux32.tar.bz2
@@ -472,10 +474,11 @@
 09c8da3a7e09cea821f9b300c6f4f52f4a571d949d91839ecbce93b84a08d570  pypy2-v5.7.0-win32.zip

    pypy 3.5-v5.7.0 sha256:

    -
    +
     921894884a647220a712ecdaad516d9c22fbadf3b4bb3a5db8f3635c60eabc7b  pypy3-v5.7.0-linux64.tar.bz2
 f0f563b74f8b82ec33b022393219b93cc0d81e9f9500614fe8417b67a52e9569  pypy3-v5.7.0-src.tar.bz2
-fe8b9804a8599ef5b16114fb64ba8392342f0066dfe81103b89198a3c5d10989  pypy3-v5.7.0-src.zip
    +fe8b9804a8599ef5b16114fb64ba8392342f0066dfe81103b89198a3c5d10989 pypy3-v5.7.0-src.zip +

    pypy3.3-v5.5.0-alpha sha256:

     966ee7951ad497ac907e01554fe48da77cc64a5e35a1307477c2f78652eba622  pypy3.3-v5.5.0-alpha-linux32.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -451,7 +451,7 @@
     f1633ad18e0af309aa11e3425f628cf9  pypy2-v5.7.0-src.zip
     23f15d2bc0c29fe6948a99352962ebf0  pypy2-v5.7.0-win32.zip
 
-pypy 3.5-v5.7.0 md5:
+pypy 3.5-v5.7.0 md5::
 
     410fb506ac436c344f785bb77820e13e  pypy3-v5.7.0-linux64.tar.bz2
     79b1d5749eb97be36933dbdf156fec02  pypy3-v5.7.0-src.tar.bz2
@@ -486,7 +486,7 @@
     c84ad90184edb315031074696e805ecbc08494e3  pypy2-v5.7.0-src.zip
     15df479b08870dc6ec9cab0e4587935fc0694cb8  pypy2-v5.7.0-win32.zip
 
-pypy 3.5-v5.7.0 sha1:
+pypy 3.5-v5.7.0 sha1::
 
     e10511bde5a88c5de9fdf9234188b8df23382a6a  pypy3-v5.7.0-linux64.tar.bz2
     0b12230025d77be26bc10cbc81671c4b3719528c  pypy3-v5.7.0-src.tar.bz2
@@ -516,7 +516,7 @@
     784be3d2b8bb12a0d88d088fd991b03324d9f84a48af5dbaf2a536eaee91d8b5  pypy2-v5.7.0-src.zip
     09c8da3a7e09cea821f9b300c6f4f52f4a571d949d91839ecbce93b84a08d570  pypy2-v5.7.0-win32.zip
 
-pypy 3.5-v5.7.0 sha256:
+pypy 3.5-v5.7.0 sha256::
 
     921894884a647220a712ecdaad516d9c22fbadf3b4bb3a5db8f3635c60eabc7b  pypy3-v5.7.0-linux64.tar.bz2
     f0f563b74f8b82ec33b022393219b93cc0d81e9f9500614fe8417b67a52e9569  pypy3-v5.7.0-src.tar.bz2

From pypy.commits at gmail.com  Mon Mar 20 15:26:25 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 20 Mar 2017 12:26:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add .cache to .hgignore.
Message-ID: <58d02ce1.44db190a.d26e.20e0@mx.google.com>

Author: Manuel Jacob 
Branch: 
Changeset: r90768:733a31146466
Date: 2017-03-20 05:12 +0100
http://bitbucket.org/pypy/pypy/changeset/733a31146466/

Log:	Add .cache to .hgignore.

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -80,5 +80,6 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
+^\.cache$
 
 pypy/module/cppyy/.+/*\.pcm

From pypy.commits at gmail.com  Mon Mar 20 15:26:27 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 20 Mar 2017 12:26:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a bit of logging to the zipimport
 module, like CPython.
Message-ID: <58d02ce3.06d2190a.846bc.2025@mx.google.com>

Author: Manuel Jacob 
Branch: 
Changeset: r90769:95e281b50ed4
Date: 2017-03-20 20:24 +0100
http://bitbucket.org/pypy/pypy/changeset/95e281b50ed4/

Log:	Add a bit of logging to the zipimport module, like CPython.

diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -263,11 +263,18 @@
                     if compiled:
                         w_result = self.import_pyc_file(space, fullname, fname,
                                                         buf, pkgpath)
-                        if w_result is not None:
-                            return w_result
+                        if w_result is None:
+                            continue
                     else:
-                        return self.import_py_file(space, fullname, fname,
+                        w_result = self.import_py_file(space, fullname, fname,
                                                    buf, pkgpath)
+                    if space.sys.get_flag('verbose') >= 1:
+                        w_stderr = space.sys.get('stderr')
+                        message = "import %s # loaded from Zip %s%s%s\n" % (
+                                fullname, self.filename, os.path.sep, fname)
+                        space.call_method(w_stderr, "write",
+                                          space.newtext(message))
+                    return w_result
                 except:
                     w_mods = space.sys.get('modules')
                     space.call_method(w_mods, 'pop', space.newtext(fullname), space.w_None)

From pypy.commits at gmail.com  Mon Mar 20 15:26:29 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 20 Mar 2017 12:26:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge
Message-ID: <58d02ce5.44db190a.d26e.20e2@mx.google.com>

Author: Manuel Jacob 
Branch: 
Changeset: r90770:57f992e266e6
Date: 2017-03-20 20:25 +0100
http://bitbucket.org/pypy/pypy/changeset/57f992e266e6/

Log:	hg merge

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -34,3 +34,5 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
+b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst
--- a/pypy/doc/release-v5.7.0.rst
+++ b/pypy/doc/release-v5.7.0.rst
@@ -2,8 +2,11 @@
 PyPy2.7 and PyPy3.5 v5.7 - two in one release
 =============================================
 
-We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for
-Linux 64bit only at first).
+The PyPy team is proud to release both PyPy2.7 v5.7 (an interpreter supporting
+Python v2.7 syntax), and a beta-quality PyPy3.5 v5.7 (an interpreter for Python
+v3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
 3.5.3.
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,6 +1,6 @@
 # Edit these appropriately before running this script
 maj=5
-min=6
+min=7
 rev=0
 branchname=release-pypy2.7-5.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min

From pypy.commits at gmail.com  Mon Mar 20 15:45:19 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 20 Mar 2017 12:45:19 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: more typos
Message-ID: <58d0314f.1a142e0a.2297c.d2fd@mx.google.com>

Author: Matti Picus 
Branch: extradoc
Changeset: r870:536afa5d31cf
Date: 2017-03-20 21:45 +0200
http://bitbucket.org/pypy/pypy.org/changeset/536afa5d31cf/

Log:	more typos

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -430,7 +430,7 @@
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
    -

    pypy2.7-5.6.0 sha1:

    +

    pypy2.7-v5.7.0 sha1:

     c8cc2cbc2b75962d0e5920f4a756f7fe3f13596c  pypy2-v5.7.0-linux32.tar.bz2
 0a831dae298fce2110626637342d0e3545a8ab47  pypy2-v5.7.0-linux64.tar.bz2
@@ -460,7 +460,7 @@
 9eb6aad2d41f7db16c19ae6e438c8f195b0d44fc  pypy3.3-v5.5.0-alpha-src.tar.bz2
 b670cdc685c323b11bfdca2a15f12c5953e6d8b4  pypy3.3-v5.5.0-alpha-src.zip
    -

    pypy2.7-5.6.0 sha256:

    +

    pypy2.7-v5.7.0 sha256:

     aaa55085d11a49cb982b059b4159495d7a4fb9afcfda7bfd680cc0175e9fa33b  pypy2-v5.7.0-linux32.tar.bz2
 64bed80e299b09c13296f577a0f52c5d4be9f7c699a390ca6026f967aeff3846  pypy2-v5.7.0-linux64.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -473,7 +473,7 @@
     2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
     009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 
-pypy2.7-5.6.0 sha1::
+pypy2.7-v5.7.0 sha1::
 
     c8cc2cbc2b75962d0e5920f4a756f7fe3f13596c  pypy2-v5.7.0-linux32.tar.bz2
     0a831dae298fce2110626637342d0e3545a8ab47  pypy2-v5.7.0-linux64.tar.bz2
@@ -503,7 +503,7 @@
     9eb6aad2d41f7db16c19ae6e438c8f195b0d44fc  pypy3.3-v5.5.0-alpha-src.tar.bz2
     b670cdc685c323b11bfdca2a15f12c5953e6d8b4  pypy3.3-v5.5.0-alpha-src.zip
 
-pypy2.7-5.6.0 sha256::
+pypy2.7-v5.7.0 sha256::
 
     aaa55085d11a49cb982b059b4159495d7a4fb9afcfda7bfd680cc0175e9fa33b  pypy2-v5.7.0-linux32.tar.bz2
     64bed80e299b09c13296f577a0f52c5d4be9f7c699a390ca6026f967aeff3846  pypy2-v5.7.0-linux64.tar.bz2

From pypy.commits at gmail.com  Mon Mar 20 15:46:46 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 20 Mar 2017 12:46:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58d031a6.020f2e0a.e2bee.56f0@mx.google.com>

Author: Manuel Jacob 
Branch: py3.5
Changeset: r90771:1bfadfa3ba91
Date: 2017-03-20 20:35 +0100
http://bitbucket.org/pypy/pypy/changeset/1bfadfa3ba91/

Log:	hg merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -83,5 +83,6 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
+^\.cache$
 
 pypy/module/cppyy/.+/*\.pcm
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -34,4 +34,5 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
-ea9979b550eeae87924dc4bef06070e8f8d0e22f release-pypy3.3-5.5.0
+fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
+b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -29,8 +29,8 @@
     'pypy': {
         'stdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
         'platstdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
-        'purelib': '{base}/lib-{implementation_lower}/{py_version_short}',
-        'platlib': '{base}/lib-{implementation_lower}/{py_version_short}',
+        'purelib': '{base}/site-packages',
+        'platlib': '{base}/site-packages',
         'include': '{base}/include',
         'platinclude': '{base}/include',
         'scripts': '{base}/bin',
diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst
--- a/pypy/doc/release-v5.7.0.rst
+++ b/pypy/doc/release-v5.7.0.rst
@@ -2,8 +2,11 @@
 PyPy2.7 and PyPy3.5 v5.7 - two in one release
 =============================================
 
-We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for
-Linux 64bit only at first).
+The PyPy team is proud to release both PyPy2.7 v5.7 (an interpreter supporting
+Python v2.7 syntax), and a beta-quality PyPy3.5 v5.7 (an interpreter for Python
+v3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
 3.5.3.
@@ -88,7 +91,8 @@
 * New features and cleanups
 
   * update the format of the PYPYLOG file and improvements to vmprof
-  * emit more sysconfig values for downstream cextension packages
+  * emit more sysconfig values for downstream cextension packages including
+    properly setting purelib and platlib to site-packages
   * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``,
     ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``,
   * detect and raise on recreation of a PyPy object from a PyObject during
@@ -131,6 +135,8 @@
     filling more slots, especially ``tp_new`` and ``tp_dealloc``
   * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_
   * fix in corner cases with the GIL and C-API functions
+  * allow overriding thread.local.__init__ in a subclass, issue #2501_
+  * allow ``PyClass_New`` to be called with NULL as the first arguemnt, issue #2504_
 
 
 * Performance improvements:
@@ -183,21 +189,19 @@
 * Performance improvements:
 
   * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called
-  * 
-  * 
-  * 
 
 * The following features of Python 3.5 are not implemented yet in PyPy:
 
   * PEP 442: Safe object finalization
   * PEP 489: Multi-phase extension module initialization
-  * XXX what else?
 
 .. _resolved: whatsnew-pypy2-5.7.0.html
 .. _19542: https://bugs.python.org/issue19542
 .. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys
 .. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on
 .. _2475: https://bitbucket.org/pypy/pypy/issues/2475
+.. _2501: https://bitbucket.org/pypy/pypy/issues/2501
+.. _2504: https://bitbucket.org/pypy/pypy/issues/2504
 .. _RevDB: https://bitbucket.org/pypy/revdb
 .. _cryptography: https://cryptography.io
 .. _cppyy: cppyy.html
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -269,11 +269,18 @@
                     if compiled:
                         w_result = self.import_pyc_file(space, fullname, fname,
                                                         buf, pkgpath)
-                        if w_result is not None:
-                            return w_result
+                        if w_result is None:
+                            continue
                     else:
-                        return self.import_py_file(space, fullname, fname,
+                        w_result = self.import_py_file(space, fullname, fname,
                                                    buf, pkgpath)
+                    if space.sys.get_flag('verbose') >= 1:
+                        w_stderr = space.sys.get('stderr')
+                        message = "import %s # loaded from Zip %s%s%s\n" % (
+                                fullname, self.filename, os.path.sep, fname)
+                        space.call_method(w_stderr, "write",
+                                          space.newtext(message))
+                    return w_result
                 except:
                     w_mods = space.sys.get('modules')
                     space.call_method(w_mods, 'pop', w_fullname, space.w_None)
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,6 +1,6 @@
 # Edit these appropriately before running this script
 maj=5
-min=6
+min=7
 rev=0
 branchname=release-pypy2.7-5.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min

From pypy.commits at gmail.com  Mon Mar 20 15:46:49 2017
From: pypy.commits at gmail.com (mjacob)
Date: Mon, 20 Mar 2017 12:46:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove verbose logging from _imp. This is
 now redundant because importlib and zipimport already do it.
Message-ID: <58d031a9.1a062e0a.fb580.da79@mx.google.com>

Author: Manuel Jacob 
Branch: py3.5
Changeset: r90772:ccee05685caa
Date: 2017-03-20 20:42 +0100
http://bitbucket.org/pypy/pypy/changeset/ccee05685caa/

Log:	Remove verbose logging from _imp. This is now redundant because
	importlib and zipimport already do it.

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -56,14 +56,6 @@
 
     return '.' + soabi + SO
 
-def log_pyverbose(space, level, message):
-    if space.sys.w_initialdict is None:
-        return # sys module not initialised, avoid recursion
-    verbose = space.sys.get_flag('verbose')
-    if verbose >= level:
-        w_stderr = space.sys.get('stderr')
-        space.call_method(w_stderr, "write", space.newtext(message))
-
 def has_so_extension(space):
     return (space.config.objspace.usemodules.cpyext or
             space.config.objspace.usemodules._cffi_backend)
@@ -136,8 +128,6 @@
 
 def load_c_extension(space, filename, modulename):
     from pypy.module.cpyext.api import load_extension_module
-    log_pyverbose(space, 1, "import %s # from %s\n" %
-                  (modulename, filename))
     load_extension_module(space, filename, modulename)
     # NB. cpyext.api.load_extension_module() can also delegate to _cffi_backend
 
@@ -391,9 +381,6 @@
     Load a module from a compiled file, execute it, and return its
     module object.
     """
-    log_pyverbose(space, 1, "import %s # compiled from %s\n" %
-                  (space.text_w(w_modulename), cpathname))
-
     if magic != get_pyc_magic(space):
         raise oefmt(space.w_ImportError, "Bad magic number in %s", cpathname)
     #print "loading pyc file:", cpathname

From pypy.commits at gmail.com  Mon Mar 20 16:29:48 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Mon, 20 Mar 2017 13:29:48 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: fix pypy3 download links
Message-ID: <58d03bbc.87212e0a.d5b9.8330@mx.google.com>

Author: Philip Jenvey 
Branch: extradoc
Changeset: r871:65741cf1b8eb
Date: 2017-03-20 13:29 -0700
http://bitbucket.org/pypy/pypy.org/changeset/65741cf1b8eb/

Log:	fix pypy3 download links

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -144,9 +144,9 @@
 PyPy 2.  You are welcome to use it anyway; if you're lucky it will
 be fast in your case.
    
 
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -131,9 +131,9 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.7.0-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.7.0-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.7.0-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.0-src.zip
 .. __: https://bitbucket.org/pypy/pypy/downloads
 
 If your CPU is really, really old, it may be a x86-32 without SSE2.

From pypy.commits at gmail.com  Mon Mar 20 18:31:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Mar 2017 15:31:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: Finally found the cause of "global name
 'self' is not defined" when
Message-ID: <58d05838.1a482e0a.158d9.067c@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90773:fff8e5f21bfd
Date: 2017-03-20 23:30 +0100
http://bitbucket.org/pypy/pypy/changeset/fff8e5f21bfd/

Log:	Finally found the cause of "global name 'self' is not defined" when
	running some combination of this lib-python and an older pypy

diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -36,9 +36,9 @@
 except ImportError:
     def _delitem_if_value_is(d, key, value):
         try:
-            if self.data[key] is value:  # fall-back: there is a potential
+            if d[key] is value:  # fall-back: there is a potential
                 #             race condition in multithreaded programs HERE
-                del self.data[key]
+                del d[key]
         except KeyError:
             pass
 

From pypy.commits at gmail.com  Mon Mar 20 18:33:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 20 Mar 2017 15:33:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: Support translation with older Linux
 kernel headers
Message-ID: <58d058a7.020f2e0a.e2bee.5cdb@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90774:e78b4d3e1d8d
Date: 2017-03-20 23:32 +0100
http://bitbucket.org/pypy/pypy/changeset/e78b4d3e1d8d/

Log:	Support translation with older Linux kernel headers

diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -99,8 +99,11 @@
         eci = eci.merge(ExternalCompilationInfo(includes=['linux/random.h']))
         class CConfig:
             _compilation_info_ = eci
-            GRND_NONBLOCK = rffi_platform.ConstantInteger('GRND_NONBLOCK')
+            GRND_NONBLOCK = rffi_platform.DefinedConstantInteger(
+                'GRND_NONBLOCK')
         globals().update(rffi_platform.configure(CConfig))
+        if GRND_NONBLOCK is None:
+            GRND_NONBLOCK = 0x0001      # from linux/random.h
 
         # On Linux, use the syscall() function because the GNU libc doesn't
         # expose the Linux getrandom() syscall yet.

From pypy.commits at gmail.com  Tue Mar 21 04:00:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 01:00:36 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Update the version numbers of
 Python
Message-ID: <58d0dda4.44db190a.d26e.3556@mx.google.com>

Author: Armin Rigo 
Branch: extradoc
Changeset: r872:bb778b31861b
Date: 2017-03-21 09:00 +0100
http://bitbucket.org/pypy/pypy.org/changeset/bb778b31861b/

Log:	Update the version numbers of Python

diff --git a/source/compat.txt b/source/compat.txt
--- a/source/compat.txt
+++ b/source/compat.txt
@@ -3,11 +3,15 @@
 title: Python compatibility
 ---
 
-PyPy implements the Python language version 2.7.12. It supports all of the core
+PyPy implements the Python language version 2.7.13. It supports all of the core
 language, passing Python test suite (with minor modifications that were
 already accepted in the main python in newer versions). It supports most
 of the commonly used Python `standard library modules`_; details below.
 
+(PyPy3 implements the Python language version 3.5.3.  It is beta right now,
+and it is quite possible that a few things are missing.  The rest of this
+document only describes the situation of the 2.7.x implementation.)
+
 .. class:: download_menu
 
    `List of installable top 1000 PyPI packages`_
diff --git a/source/features.txt b/source/features.txt
--- a/source/features.txt
+++ b/source/features.txt
@@ -10,7 +10,7 @@
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).
 
-**PyPy** implements **Python 2.7.12**.
+**PyPy** implements **Python 2.7.13**.
 It supports all of the core language, passing the Python test suite
 (with minor modifications that were already accepted in the main python
 in newer versions). It supports most of the commonly used Python
diff --git a/source/index.txt b/source/index.txt
--- a/source/index.txt
+++ b/source/index.txt
@@ -4,7 +4,7 @@
 ---
 
 PyPy is a `fast`_, `compliant`_ alternative implementation of the `Python`_
-language (2.7.12 and 3.3.5). It has several advantages and distinct features:
+language (2.7.13 and 3.5.3). It has several advantages and distinct features:
 
   * **Speed:** thanks to its Just-in-Time compiler, Python programs
     often run `faster`_ on PyPy.  `(What is a JIT compiler?)`_

From pypy.commits at gmail.com  Tue Mar 21 04:35:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 01:35:00 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: regen
Message-ID: <58d0e5b4.06d2190a.846bc.3628@mx.google.com>

Author: Armin Rigo 
Branch: extradoc
Changeset: r873:9c84049ed3ea
Date: 2017-03-21 09:34 +0100
http://bitbucket.org/pypy/pypy.org/changeset/9c84049ed3ea/

Log:	regen

diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -71,10 +71,13 @@
 
    
 
    
 
    Python compatibility
    
-
    PyPy implements the Python language version 2.7.12. It supports all of the core
+
    PyPy implements the Python language version 2.7.13. It supports all of the core
 language, passing Python test suite (with minor modifications that were
 already accepted in the main python in newer versions). It supports most
 of the commonly used Python standard library modules; details below.
    
+
    (PyPy3 implements the Python language version 3.5.3.  It is beta right now,
+and it is quite possible that a few things are missing.  The rest of this
+document only describes the situation of the 2.7.x implementation.)
    
 
    List of installable top 1000 PyPI packages
    
 
    PyPy has alpha/beta-level support for the CPython C API, however,
 this feature is not yet complete. We strongly advise use of CFFI
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -74,7 +74,7 @@
 
    PyPy is a replacement for CPython.  It is built using the RPython
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).
    
-
    PyPy implements Python 2.7.12.
+
    PyPy implements Python 2.7.13.
 It supports all of the core language, passing the Python test suite
 (with minor modifications that were already accepted in the main python
 in newer versions). It supports most of the commonly used Python
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -72,7 +72,7 @@
 
    
 
    Welcome to PyPy
    
 
    PyPy is a fast, compliant alternative implementation of the Python
-language (2.7.12 and 3.3.5). It has several advantages and distinct features:
    
+language (2.7.13 and 3.5.3). It has several advantages and distinct features:
    
 
    
 
      
 
    • Speed: thanks to its Just-in-Time compiler, Python programs

From pypy.commits at gmail.com  Tue Mar 21 05:17:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 02:17:07 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: sha1/sha256 of the ppc builds
Message-ID: <58d0ef93.0d152e0a.50667.6d47@mx.google.com>

Author: Armin Rigo 
Branch: extradoc
Changeset: r874:12c2667b1e20
Date: 2017-03-21 10:16 +0100
http://bitbucket.org/pypy/pypy.org/changeset/12c2667b1e20/

Log:	sha1/sha256 of the ppc builds

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -438,6 +438,8 @@
 bec04d53cd2a7cc89cf67cfdab2207ad6c08781b  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
 c510f53447fc800b4611545d2679d596d8f0c461  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
 097e418d5aa61c7697a50536c5b173d8211603a9  pypy2-v5.7.0-osx64.tar.bz2
+a11cf7d95a909c8e60bafc79ff491711ecd0ee29  pypy2-v5.7.0-ppc64.tar.bz2
+0ccb73316935c5610e5170330556443abd87d3cd  pypy2-v5.7.0-ppc64le.tar.bz2
 20f33a4d0c08a5d4935086caec73be3d92b60b2b  pypy2-v5.7.0-s390x.tar.bz2
 5976e1f19a42f64f778d385322723200b5c6d7c9  pypy2-v5.7.0-src.tar.bz2
 c84ad90184edb315031074696e805ecbc08494e3  pypy2-v5.7.0-src.zip
@@ -468,6 +470,8 @@
 413df8097c87a8e79cc46df65837b1d533de83b7cb06ce88b168697807fff696  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
 6328e4767cf20d20a290739db2d1ec769f3c9c360fa9824f61581b3356add79b  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
 9204aa1b55771a374a3118bb43e498c88caca8c33710eecf3249855c4dd1352a  pypy2-v5.7.0-osx64.tar.bz2
+ca936b034a5cd2decfd3b500fad3c28e5a6327a77bb56b2d2ba6ff0159c00f2c  pypy2-v5.7.0-ppc64.tar.bz2
+3ecb3165c85df60f8e59bfdf28304cd40894d304ee04a4d58dc8b9947cbb2b01  pypy2-v5.7.0-ppc64le.tar.bz2
 8820b36f38b4aa44c1ef2be213c7d13818c9fdb61795f24f4fb18a7a78d2dde7  pypy2-v5.7.0-s390x.tar.bz2
 e522ea7ca51b16ee5505f22b86803664b762a263a6d69ba84c359fcf8365ad3e  pypy2-v5.7.0-src.tar.bz2
 784be3d2b8bb12a0d88d088fd991b03324d9f84a48af5dbaf2a536eaee91d8b5  pypy2-v5.7.0-src.zip
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -481,6 +481,8 @@
     bec04d53cd2a7cc89cf67cfdab2207ad6c08781b  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
     c510f53447fc800b4611545d2679d596d8f0c461  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
     097e418d5aa61c7697a50536c5b173d8211603a9  pypy2-v5.7.0-osx64.tar.bz2
+    a11cf7d95a909c8e60bafc79ff491711ecd0ee29  pypy2-v5.7.0-ppc64.tar.bz2
+    0ccb73316935c5610e5170330556443abd87d3cd  pypy2-v5.7.0-ppc64le.tar.bz2
     20f33a4d0c08a5d4935086caec73be3d92b60b2b  pypy2-v5.7.0-s390x.tar.bz2
     5976e1f19a42f64f778d385322723200b5c6d7c9  pypy2-v5.7.0-src.tar.bz2
     c84ad90184edb315031074696e805ecbc08494e3  pypy2-v5.7.0-src.zip
@@ -511,6 +513,8 @@
     413df8097c87a8e79cc46df65837b1d533de83b7cb06ce88b168697807fff696  pypy2-v5.7.0-linux-armhf-raring.tar.bz2
     6328e4767cf20d20a290739db2d1ec769f3c9c360fa9824f61581b3356add79b  pypy2-v5.7.0-linux-armhf-raspbian.tar.bz2
     9204aa1b55771a374a3118bb43e498c88caca8c33710eecf3249855c4dd1352a  pypy2-v5.7.0-osx64.tar.bz2
+    ca936b034a5cd2decfd3b500fad3c28e5a6327a77bb56b2d2ba6ff0159c00f2c  pypy2-v5.7.0-ppc64.tar.bz2
+    3ecb3165c85df60f8e59bfdf28304cd40894d304ee04a4d58dc8b9947cbb2b01  pypy2-v5.7.0-ppc64le.tar.bz2
     8820b36f38b4aa44c1ef2be213c7d13818c9fdb61795f24f4fb18a7a78d2dde7  pypy2-v5.7.0-s390x.tar.bz2
     e522ea7ca51b16ee5505f22b86803664b762a263a6d69ba84c359fcf8365ad3e  pypy2-v5.7.0-src.tar.bz2
     784be3d2b8bb12a0d88d088fd991b03324d9f84a48af5dbaf2a536eaee91d8b5  pypy2-v5.7.0-src.zip

From pypy.commits at gmail.com  Tue Mar 21 05:25:15 2017
From: pypy.commits at gmail.com (tobweber)
Date: Tue, 21 Mar 2017 02:25:15 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Fix major GC
 duration was not logged in case of abort due to long jump out of the abort
 function
Message-ID: <58d0f17b.c819190a.5824d.453a@mx.google.com>

Author: Tobias Weber 
Branch: c8-overheads-instrumentation
Changeset: r2033:5da1d40bd38d
Date: 2017-03-20 17:47 +0100
http://bitbucket.org/pypy/stmgc/changeset/5da1d40bd38d/

Log:	Fix major GC duration was not logged in case of abort due to long
	jump out of the abort function

diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -783,12 +783,12 @@
         dprintf((" | used after collection:  %ld\n",
                 (long)pages_ctl.total_allocated));
         dprintf((" `----------------------------------------------\n"));
-        if (must_abort())
-            abort_with_mutex();
 
         stop_timer_and_publish_for_thread(
             thread_local_for_logging, STM_DURATION_MAJOR_GC_LOG_ONLY);
 
+        if (must_abort())
+            abort_with_mutex();
         return;
 #endif
     }
@@ -843,9 +843,10 @@
        must abort, do it now. The others are in safe-points that will
        abort if they need to. */
     dprintf(("must abort?:%d\n", (int)must_abort()));
-    if (must_abort())
-        abort_with_mutex();
 
     stop_timer_and_publish_for_thread(
         thread_local_for_logging, STM_DURATION_MAJOR_GC_FULL);
+
+    if (must_abort())
+        abort_with_mutex();
 }

From pypy.commits at gmail.com  Tue Mar 21 06:21:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 03:21:49 -0700 (PDT)
Subject: [pypy-commit] cffi default: Add the module name in front of the
 error for missing def_extern().
Message-ID: <58d0febd.4ee4190a.35e41.41a3@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r2915:85994689bab3
Date: 2017-03-21 11:21 +0100
http://bitbucket.org/cffi/cffi/changeset/85994689bab3/

Log:	Add the module name in front of the error for missing def_extern().
	Useful in cases we mistakenly define the same name in more than one
	module.

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1180,7 +1180,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1552,7 +1552,8 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function bar() called, but no code was attached "
+        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
+        b"but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")

From pypy.commits at gmail.com  Tue Mar 21 06:29:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 03:29:20 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.10: hg merge default
Message-ID: <58d10080.011c190a.b1c04.4399@mx.google.com>

Author: Armin Rigo 
Branch: release-1.10
Changeset: r2916:06af0f6155b5
Date: 2017-03-21 11:22 +0100
http://bitbucket.org/cffi/cffi/changeset/06af0f6155b5/

Log:	hg merge default

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1180,7 +1180,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1552,7 +1552,8 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function bar() called, but no code was attached "
+        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
+        b"but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")

From pypy.commits at gmail.com  Tue Mar 21 06:43:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 03:43:08 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.10: update the whatsnew
Message-ID: <58d103bc.d3d4190a.e0aa3.8ea2@mx.google.com>

Author: Armin Rigo 
Branch: release-1.10
Changeset: r2917:230849ed1d47
Date: 2017-03-21 11:41 +0100
http://bitbucket.org/cffi/cffi/changeset/230849ed1d47/

Log:	update the whatsnew

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -9,18 +9,16 @@
 * Issue #295: use calloc() directly instead of
   PyObject_Malloc()+memset() to handle ffi.new() with a default
   allocator.  Speeds up ``ffi.new(large-array)`` where most of the time
-  you never touch most of the array.  (But avoid doing that too often:
-  on 32-bit PyPy it will quickly exhaust the address space.  If possible,
-  use instead explicit calls to calloc() and free().)
+  you never touch most of the array.
 
 * Some OS/X build fixes ("only with Xcode but without CLT").
 
 * Improve a couple of error messages: when getting mismatched versions
   of cffi and its backend; and when calling functions which cannot be
   called with libffi because an argument is a struct that is "too
-  complicated" (not a struct *pointer*).
+  complicated" (and not a struct *pointer*, which always works).
 
-* Add support for some obscure compilers (non-msvc, non-gcc, non-icc,
+* Add support for some unusual compilers (non-msvc, non-gcc, non-icc,
   non-clang)
 
 * Implemented the remaining cases for ``ffi.from_buffer``.  Now all
@@ -33,15 +31,15 @@
 
 * The C type ``_Bool`` or ``bool`` now converts to a Python boolean
   when reading, instead of the content of the byte as an integer.  The
-  change here is mostly what occurs if the byte happens to contain a
+  potential incompatibility here is what occurs if the byte contains a
   value different from 0 and 1.  Previously, it would just return it;
   with this change, CFFI raises an exception in this case.  But this
   case means "undefined behavior" in C; if you really have to interface
-  with a library relying on this, don't use ``_Bool`` in the CFFI side.
+  with a library relying on this, don't use ``bool`` in the CFFI side.
   Also, it is still valid to use a byte string as initializer for a
-  ``_Bool[]``, but now it must only contain ``\x00`` or ``\x01``.  As an
-  aside, ``ffi.string()`` no longer works on ``_Bool[]`` (but it never
-  made much sense, as this function stops on the first zero).
+  ``bool[]``, but now it must only contain ``\x00`` or ``\x01``.  As an
+  aside, ``ffi.string()`` no longer works on ``bool[]`` (but it never
+  made much sense, as this function stops at the first zero).
 
 * ``ffi.buffer`` is now the name of cffi's buffer type, and
   ``ffi.buffer()`` works like before but is the constructor of that type.
@@ -61,6 +59,11 @@
   "memory pressure", causing the GC to run too infrequently if you call
   ``ffi.new()`` very often and/or with large arrays.  Fixed in PyPy 5.7.
 
+* Support in ``ffi.cdef()`` for numeric expressions with ``+`` or
+  ``-``.  Assumes that there is no overflow; it should be fixed first
+  before we add more general support for arbitrary arithmetic on
+  constants.
+
 
 v1.9
 ====

From pypy.commits at gmail.com  Tue Mar 21 06:43:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 03:43:10 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.10: md5/sha
Message-ID: <58d103be.cf4d2e0a.8f819.40cd@mx.google.com>

Author: Armin Rigo 
Branch: release-1.10
Changeset: r2918:1de684a0845b
Date: 2017-03-21 11:42 +0100
http://bitbucket.org/cffi/cffi/changeset/1de684a0845b/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.10.0.tar.gz
 
-   - MD5: ...
+   - MD5: 2b5fa41182ed0edaf929a789e602a070
 
-   - SHA: ...
+   - SHA: 8484aba03d1e64367d3110c0e36c1ed052b43f12
 
-   - SHA256: ...
+   - SHA256: b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Tue Mar 21 06:43:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 03:43:13 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge release-1.10
Message-ID: <58d103c1.5e152e0a.a16fd.6e56@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r2919:486d919c0b87
Date: 2017-03-21 11:42 +0100
http://bitbucket.org/cffi/cffi/changeset/486d919c0b87/

Log:	hg merge release-1.10

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.10.0.tar.gz
 
-   - MD5: ...
+   - MD5: 2b5fa41182ed0edaf929a789e602a070
 
-   - SHA: ...
+   - SHA: 8484aba03d1e64367d3110c0e36c1ed052b43f12
 
-   - SHA256: ...
+   - SHA256: b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -9,18 +9,16 @@
 * Issue #295: use calloc() directly instead of
   PyObject_Malloc()+memset() to handle ffi.new() with a default
   allocator.  Speeds up ``ffi.new(large-array)`` where most of the time
-  you never touch most of the array.  (But avoid doing that too often:
-  on 32-bit PyPy it will quickly exhaust the address space.  If possible,
-  use instead explicit calls to calloc() and free().)
+  you never touch most of the array.
 
 * Some OS/X build fixes ("only with Xcode but without CLT").
 
 * Improve a couple of error messages: when getting mismatched versions
   of cffi and its backend; and when calling functions which cannot be
   called with libffi because an argument is a struct that is "too
-  complicated" (not a struct *pointer*).
+  complicated" (and not a struct *pointer*, which always works).
 
-* Add support for some obscure compilers (non-msvc, non-gcc, non-icc,
+* Add support for some unusual compilers (non-msvc, non-gcc, non-icc,
   non-clang)
 
 * Implemented the remaining cases for ``ffi.from_buffer``.  Now all
@@ -33,15 +31,15 @@
 
 * The C type ``_Bool`` or ``bool`` now converts to a Python boolean
   when reading, instead of the content of the byte as an integer.  The
-  change here is mostly what occurs if the byte happens to contain a
+  potential incompatibility here is what occurs if the byte contains a
   value different from 0 and 1.  Previously, it would just return it;
   with this change, CFFI raises an exception in this case.  But this
   case means "undefined behavior" in C; if you really have to interface
-  with a library relying on this, don't use ``_Bool`` in the CFFI side.
+  with a library relying on this, don't use ``bool`` in the CFFI side.
   Also, it is still valid to use a byte string as initializer for a
-  ``_Bool[]``, but now it must only contain ``\x00`` or ``\x01``.  As an
-  aside, ``ffi.string()`` no longer works on ``_Bool[]`` (but it never
-  made much sense, as this function stops on the first zero).
+  ``bool[]``, but now it must only contain ``\x00`` or ``\x01``.  As an
+  aside, ``ffi.string()`` no longer works on ``bool[]`` (but it never
+  made much sense, as this function stops at the first zero).
 
 * ``ffi.buffer`` is now the name of cffi's buffer type, and
   ``ffi.buffer()`` works like before but is the constructor of that type.
@@ -61,6 +59,11 @@
   "memory pressure", causing the GC to run too infrequently if you call
   ``ffi.new()`` very often and/or with large arrays.  Fixed in PyPy 5.7.
 
+* Support in ``ffi.cdef()`` for numeric expressions with ``+`` or
+  ``-``.  Assumes that there is no overflow; it should be fixed first
+  before we add more general support for arbitrary arithmetic on
+  constants.
+
 
 v1.9
 ====

From pypy.commits at gmail.com  Tue Mar 21 06:59:45 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 21 Mar 2017 03:59:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: add python-cffi to the list of debian
 packages
Message-ID: <58d107a1.10142e0a.238cb.4501@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r90775:b7089fe8b581
Date: 2017-03-21 11:58 +0100
http://bitbucket.org/pypy/pypy/changeset/b7089fe8b581/

Log:	add python-cffi to the list of debian packages

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -107,7 +107,7 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev \
+    tk-dev libgc-dev python-cffi \
     liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::

From pypy.commits at gmail.com  Tue Mar 21 07:10:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 04:10:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: Temporary fix?
Message-ID: <58d10a11.548d190a.345a4.4754@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90776:f3f2566b1a08
Date: 2017-03-21 12:09 +0100
http://bitbucket.org/pypy/pypy/changeset/f3f2566b1a08/

Log:	Temporary fix?

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -429,7 +429,11 @@
                 gc._trace_callback(callback, arg, p + offset)
             llop.threadlocalref_release(lltype.Void)
         _lambda_trace_tlref = lambda: _trace_tlref
-        TRACETLREF = lltype.GcStruct('TRACETLREF')
+        # WAAAH obscurity: can't use a name that may be non-unique,
+        # otherwise the types compare equal, even though we call
+        # register_custom_trace_hook() to register different trace
+        # functions...
+        TRACETLREF = lltype.GcStruct('TRACETLREF%d' % unique_id)
         _tracetlref_obj = lltype.malloc(TRACETLREF, immortal=True)
 
     @staticmethod

From pypy.commits at gmail.com  Tue Mar 21 07:34:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Mar 2017 04:34:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a previously-failing test for the
 GC not handling multiple
Message-ID: <58d10fc1.18532e0a.af04.4a99@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90777:761bc7504874
Date: 2017-03-21 12:33 +0100
http://bitbucket.org/pypy/pypy/changeset/761bc7504874/

Log:	Add a previously-failing test for the GC not handling multiple
	ThreadLocalReferences, fixed by f3f2566b1a08

diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py
--- a/rpython/rlib/test/test_rthread.py
+++ b/rpython/rlib/test/test_rthread.py
@@ -262,15 +262,23 @@
             py.test.skip("no __thread support here")
 
         class FooBar(object):
-            pass
+            def __init__(self, a, b):
+                self.lst = [a, b]
         t = ThreadLocalReference(FooBar)
+        t2 = ThreadLocalReference(FooBar)
 
         def tset():
-            x1 = FooBar()
+            x1 = FooBar(40, 2)
             t.set(x1)
             return weakref.ref(x1)
         tset._dont_inline_ = True
 
+        def t2set():
+            x1 = FooBar(50, 3)
+            t2.set(x1)
+            return weakref.ref(x1)
+        t2set._dont_inline_ = True
+
         class WrFromThread:
             pass
         wr_from_thread = WrFromThread()
@@ -279,22 +287,30 @@
             config = objectmodel.fetch_translated_config()
             assert t.automatic_keepalive(config) is True
             wr = tset()
-            import gc; gc.collect()   # 'x1' should not be collected
-            x2 = t.get()
+            wr2 = t2set()
+            import gc; gc.collect()   # the two 'x1' should not be collected
+            x1 = t.get()
+            assert x1 is not None
+            assert wr() is not None
+            assert wr() is x1
+            assert x1.lst == [40, 2]
+            x2 = t2.get()
             assert x2 is not None
-            assert wr() is not None
-            assert wr() is x2
-            return wr
+            assert wr2() is not None
+            assert wr2() is x2
+            assert x2.lst == [50, 3]
+            return wr, wr2
 
         def thread_entry_point():
-            wr = f()
+            wr, wr2 = f()
             wr_from_thread.wr = wr
+            wr_from_thread.wr2 = wr2
             wr_from_thread.seen = True
 
         def main():
             wr_from_thread.seen = False
             start_new_thread(thread_entry_point, ())
-            wr1 = f()
+            wr1, wr2 = f()
             count = 0
             while True:
                 time.sleep(0.5)
@@ -302,10 +318,15 @@
                     break
                 count += 1
             assert wr_from_thread.seen is True
-            wr2 = wr_from_thread.wr
-            import gc; gc.collect()      # wr2() should be collected here
+            wr_other_1 = wr_from_thread.wr
+            wr_other_2 = wr_from_thread.wr2
+            import gc; gc.collect()      # wr_other_*() should be collected here
             assert wr1() is not None     # this thread, still running
-            assert wr2() is None         # other thread, not running any more
+            assert wr2() is not None     # this thread, still running
+            assert wr_other_1() is None  # other thread, not running any more
+            assert wr_other_2() is None  # other thread, not running any more
+            assert wr1().lst == [40, 2]
+            assert wr2().lst == [50, 3]
             return 42
 
         extra_options = {'no__thread': no__thread, 'shared': True}

From pypy.commits at gmail.com  Tue Mar 21 14:02:51 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Tue, 21 Mar 2017 11:02:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fixes #2508 -- correctly handle
 dict.pop where the popping key is not the same type as the dict's and pop
 is called with a default
Message-ID: <58d16acb.1a0e2e0a.49c1a.2443@mx.google.com>

Author: Alex Gaynor 
Branch: 
Changeset: r90778:a1b0ce5e4915
Date: 2017-03-21 14:02 -0400
http://bitbucket.org/pypy/pypy/changeset/a1b0ce5e4915/

Log:	Fixes #2508 -- correctly handle dict.pop where the popping key is
	not the same type as the dict's and pop is called with a default

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1049,6 +1049,8 @@
             else:
                 return d.pop(key, w_default)
         elif self._never_equal_to(space.type(w_key)):
+            if w_default is not None:
+                return w_default
             raise KeyError
         else:
             self.switch_to_object_strategy(w_dict)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -161,7 +161,7 @@
         w_d.initialize_content([(w(1), wb("a")), (w(2), wb("b"))])
         w_l = self.space.call_method(w_d, "keys")
         assert sorted(self.space.listview_int(w_l)) == [1,2]
-        
+
         # make sure that .keys() calls newlist_bytes for string dicts
         def not_allowed(*args):
             assert False, 'should not be called'
@@ -174,7 +174,7 @@
 
         # XXX: it would be nice if the test passed without monkeypatch.undo(),
         # but we need space.newlist_unicode for it
-        monkeypatch.undo() 
+        monkeypatch.undo()
         w_d = self.space.newdict()
         w_d.initialize_content([(w(u"a"), w(1)), (w(u"b"), w(6))])
         w_l = self.space.call_method(w_d, "keys")
@@ -223,6 +223,10 @@
         assert len(dd) == 1
         raises(KeyError, dd.pop, 33)
 
+        assert d.pop("abc", None) is None
+        raises(KeyError, d.pop, "abc")
+        assert len(d) == 2
+
     def test_has_key(self):
         d = {1: 2, 3: 4}
         assert d.has_key(1)
@@ -1466,4 +1470,3 @@
     fakespace = FakeSpace()
     d = fakespace.newdict(module=True)
     assert type(d.get_strategy()) is BytesDictStrategy
-

From pypy.commits at gmail.com  Tue Mar 21 14:57:55 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 21 Mar 2017 11:57:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: extend b7089fe8b581 to other platforms
Message-ID: <58d177b3.1c10190a.2eba4.59c3@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90779:361a7fb3e056
Date: 2017-03-21 20:57 +0200
http://bitbucket.org/pypy/pypy/changeset/361a7fb3e056/

Log:	extend b7089fe8b581 to other platforms

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -114,15 +114,15 @@
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel \
+    gdbm-devel python-cffi\
     xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses \
-    xz-devel  # For lzma on PyPy3.
+    libexpat-devel libffi-devel python-curses python-cffi \
+    xz-devel # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
 On Mac OS X, most of these build-time dependencies are installed alongside

From pypy.commits at gmail.com  Tue Mar 21 16:23:01 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 21 Mar 2017 13:23:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: blindly change this from -1 to an
 arbitrary large number on 32 bit systems
Message-ID: <58d18ba5.09482e0a.f8bd4.7aea@mx.google.com>

Author: Matti Picus 
Branch: py3.5
Changeset: r90780:eecdab8e95b5
Date: 2017-03-21 22:22 +0200
http://bitbucket.org/pypy/pypy/changeset/eecdab8e95b5/

Log:	blindly change this from -1 to an arbitrary large number on 32 bit
	systems

diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -28,7 +28,7 @@
     MAXGROUPS = int(2**31 - 1)
 else:
     MAXREPEAT = int(2**31 - 1)
-    MAXGROUPS = int((2**31 / sys.maxint / 2) - 1)
+    MAXGROUPS = int(2**30 - 1)
 
 # In _sre.c this is bytesize of the code word type of the C implementation.
 # There it's 2 for normal Python builds and more for wide unicode builds (large

From pypy.commits at gmail.com  Tue Mar 21 18:17:21 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 21 Mar 2017 15:17:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Clean up extension module loading
 slightly in anticipation of upcoming py3.5 changes.
Message-ID: <58d1a671.de13190a.17a47.6865@mx.google.com>

Author: Manuel Jacob 
Branch: 
Changeset: r90781:b9542a9fa14e
Date: 2017-03-21 22:58 +0100
http://bitbucket.org/pypy/pypy/changeset/b9542a9fa14e/

Log:	Clean up extension module loading slightly in anticipation of
	upcoming py3.5 changes.

diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -48,3 +48,4 @@
     w_modules_dict = space.sys.get('modules')
     space.setitem(w_modules_dict, w_name, module)
     space.setitem(w_modules_dict, space.newtext(name + '.lib'), lib)
+    return module
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1469,10 +1469,6 @@
     copy_header_files(cts, trunk_include, use_micronumpy)
 
 
-def _load_from_cffi(space, name, path, initptr):
-    from pypy.module._cffi_backend import cffi1_module
-    cffi1_module.load_cffi1_module(space, name, path, initptr)
-
 @unwrap_spec(path='text', name='text')
 def load_extension_module(space, path, name):
     # note: this is used both to load CPython-API-style C extension
@@ -1505,11 +1501,11 @@
             pass
         else:
             try:
-                _load_from_cffi(space, name, path, initptr)
+                from pypy.module._cffi_backend import cffi1_module
+                return cffi1_module.load_cffi1_module(space, name, path, initptr)
             except:
                 rdynload.dlclose(dll)
                 raise
-            return
     #
     if space.config.objspace.usemodules.cpyext:
         also_look_for = 'init%s' % (basename,)
@@ -1518,8 +1514,7 @@
         except KeyError:
             pass
         else:
-            load_cpyext_module(space, name, path, dll, initptr)
-            return
+            return load_cpyext_module(space, name, path, dll, initptr)
         if look_for is not None:
             look_for += ' or ' + also_look_for
         else:
@@ -1535,9 +1530,10 @@
 
     space.getbuiltinmodule("cpyext")    # mandatory to init cpyext
     state = space.fromcache(State)
-    if state.find_extension(name, path) is not None:
+    w_mod = state.find_extension(name, path)
+    if w_mod is not None:
         rdynload.dlclose(dll)
-        return
+        return w_mod
     old_context = state.package_context
     state.package_context = name, path
     try:
@@ -1546,7 +1542,8 @@
         state.check_and_raise_exception()
     finally:
         state.package_context = old_context
-    state.fixup_extension(name, path)
+    w_mod = state.fixup_extension(name, path)
+    return w_mod
 
 @specialize.ll()
 def generic_cpy_call(space, func, *args):
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -165,6 +165,7 @@
         w_dict = w_mod.getdict(space)
         w_copy = space.call_method(w_dict, 'copy')
         self.extensions[path] = w_copy
+        return w_mod
 
 
 def _rawrefcount_perform(space):
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -616,7 +616,7 @@
     from pypy.module.cpyext.api import load_extension_module
     log_pyverbose(space, 1, "import %s # from %s\n" %
                   (modulename, filename))
-    load_extension_module(space, filename, modulename)
+    return load_extension_module(space, filename, modulename)
     # NB. cpyext.api.load_extension_module() can also delegate to _cffi_backend
 
 @jit.dont_look_inside
@@ -680,8 +680,8 @@
                         pass
                 return w_mod
             elif find_info.modtype == C_EXTENSION and has_so_extension(space):
-                load_c_extension(space, find_info.filename, space.text_w(w_modulename))
-                return check_sys_modules(space, w_modulename)
+                return load_c_extension(space, find_info.filename,
+                                        space.text_w(w_modulename))
         except OperationError:
             w_mods = space.sys.get('modules')
             space.call_method(w_mods, 'pop', w_modulename, space.w_None)
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -136,8 +136,8 @@
 def load_dynamic(space, w_modulename, filename, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
-    importing.load_c_extension(space, filename, space.text_w(w_modulename))
-    return importing.check_sys_modules(space, w_modulename)
+    return importing.load_c_extension(space, filename,
+                                      space.text_w(w_modulename))
 
 def new_module(space, w_name):
     return Module(space, w_name, add_package=False)

From pypy.commits at gmail.com  Tue Mar 21 18:58:14 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 21 Mar 2017 15:58:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58d1b006.4b542e0a.95843.5b22@mx.google.com>

Author: Manuel Jacob 
Branch: py3.5
Changeset: r90782:9ca6b5dc03b0
Date: 2017-03-21 23:56 +0100
http://bitbucket.org/pypy/pypy/changeset/9ca6b5dc03b0/

Log:	hg merge default

diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -36,9 +36,9 @@
 except ImportError:
     def _delitem_if_value_is(d, key, value):
         try:
-            if self.data[key] is value:  # fall-back: there is a potential
+            if d[key] is value:  # fall-back: there is a potential
                 #             race condition in multithreaded programs HERE
-                del self.data[key]
+                del d[key]
         except KeyError:
             pass
 
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -107,22 +107,22 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev \
+    tk-dev libgc-dev python-cffi \
     liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel \
+    gdbm-devel python-cffi\
     xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses \
-    xz-devel  # For lzma on PyPy3.
+    libexpat-devel libffi-devel python-curses python-cffi \
+    xz-devel # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
 On Mac OS X, most of these build-time dependencies are installed alongside
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -48,3 +48,4 @@
     w_modules_dict = space.sys.get('modules')
     space.setitem(w_modules_dict, w_name, module)
     space.setitem(w_modules_dict, space.newtext(name + '.lib'), lib)
+    return module
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1489,10 +1489,6 @@
     copy_header_files(cts, trunk_include, use_micronumpy)
 
 
-def _load_from_cffi(space, name, path, initptr):
-    from pypy.module._cffi_backend import cffi1_module
-    cffi1_module.load_cffi1_module(space, name, path, initptr)
-
 @unwrap_spec(path='fsencode', name='text')
 def load_extension_module(space, path, name):
     # note: this is used both to load CPython-API-style C extension
@@ -1526,11 +1522,11 @@
             pass
         else:
             try:
-                _load_from_cffi(space, name, path, initptr)
+                from pypy.module._cffi_backend import cffi1_module
+                return cffi1_module.load_cffi1_module(space, name, path, initptr)
             except:
                 rdynload.dlclose(dll)
                 raise
-            return
     #
     if space.config.objspace.usemodules.cpyext:
         also_look_for = 'PyInit_%s' % (basename,)
@@ -1539,8 +1535,7 @@
         except KeyError:
             pass
         else:
-            load_cpyext_module(space, name, path, dll, initptr)
-            return
+            return load_cpyext_module(space, name, path, dll, initptr)
         if look_for is not None:
             look_for += ' or ' + also_look_for
         else:
@@ -1559,9 +1554,10 @@
 
     space.getbuiltinmodule("cpyext")    # mandatory to init cpyext
     state = space.fromcache(State)
-    if state.find_extension(name, path) is not None:
+    w_mod = state.find_extension(name, path)
+    if w_mod is not None:
         rdynload.dlclose(dll)
-        return
+        return w_mod
     old_context = state.package_context
     state.package_context = name, path
     try:
@@ -1571,6 +1567,7 @@
     finally:
         state.package_context = old_context
     state.fixup_extension(w_mod, name, path)
+    return w_mod
 
 @specialize.ll()
 def generic_cpy_call(space, func, *args):
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -163,6 +163,7 @@
         w_dict = w_mod.getdict(space)
         w_copy = space.call_method(w_dict, 'copy')
         self.extensions[path] = w_copy
+        return w_mod
 
 
 def _rawrefcount_perform(space):
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -128,7 +128,7 @@
 
 def load_c_extension(space, filename, modulename):
     from pypy.module.cpyext.api import load_extension_module
-    load_extension_module(space, filename, modulename)
+    return load_extension_module(space, filename, modulename)
     # NB. cpyext.api.load_extension_module() can also delegate to _cffi_backend
 
 # __________________________________________________________________
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -52,8 +52,8 @@
     w_modulename = space.getattr(w_spec, space.newtext("name"))
     w_path = space.getattr(w_spec, space.newtext("origin"))
     filename = space.fsencode_w(w_path)
-    importing.load_c_extension(space, filename, space.text_w(w_modulename))
-    return importing.check_sys_modules(space, w_modulename)
+    return importing.load_c_extension(space, filename,
+                                      space.text_w(w_modulename))
 
 def create_builtin(space, w_spec):
     w_name = space.getattr(w_spec, space.newtext("name"))
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1007,6 +1007,8 @@
             else:
                 return d.pop(key, w_default)
         elif self._never_equal_to(space.type(w_key)):
+            if w_default is not None:
+                return w_default
             raise KeyError
         else:
             self.switch_to_object_strategy(w_dict)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -227,6 +227,10 @@
         assert len(dd) == 1
         raises(KeyError, dd.pop, 33)
 
+        assert d.pop("abc", None) is None
+        raises(KeyError, d.pop, "abc")
+        assert len(d) == 2
+
     def test_items(self):
         d = {1: 2, 3: 4}
         its = list(d.items())
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -429,7 +429,11 @@
                 gc._trace_callback(callback, arg, p + offset)
             llop.threadlocalref_release(lltype.Void)
         _lambda_trace_tlref = lambda: _trace_tlref
-        TRACETLREF = lltype.GcStruct('TRACETLREF')
+        # WAAAH obscurity: can't use a name that may be non-unique,
+        # otherwise the types compare equal, even though we call
+        # register_custom_trace_hook() to register different trace
+        # functions...
+        TRACETLREF = lltype.GcStruct('TRACETLREF%d' % unique_id)
         _tracetlref_obj = lltype.malloc(TRACETLREF, immortal=True)
 
     @staticmethod
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -99,8 +99,11 @@
         eci = eci.merge(ExternalCompilationInfo(includes=['linux/random.h']))
         class CConfig:
             _compilation_info_ = eci
-            GRND_NONBLOCK = rffi_platform.ConstantInteger('GRND_NONBLOCK')
+            GRND_NONBLOCK = rffi_platform.DefinedConstantInteger(
+                'GRND_NONBLOCK')
         globals().update(rffi_platform.configure(CConfig))
+        if GRND_NONBLOCK is None:
+            GRND_NONBLOCK = 0x0001      # from linux/random.h
 
         # On Linux, use the syscall() function because the GNU libc doesn't
         # expose the Linux getrandom() syscall yet.
diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py
--- a/rpython/rlib/test/test_rthread.py
+++ b/rpython/rlib/test/test_rthread.py
@@ -262,15 +262,23 @@
             py.test.skip("no __thread support here")
 
         class FooBar(object):
-            pass
+            def __init__(self, a, b):
+                self.lst = [a, b]
         t = ThreadLocalReference(FooBar)
+        t2 = ThreadLocalReference(FooBar)
 
         def tset():
-            x1 = FooBar()
+            x1 = FooBar(40, 2)
             t.set(x1)
             return weakref.ref(x1)
         tset._dont_inline_ = True
 
+        def t2set():
+            x1 = FooBar(50, 3)
+            t2.set(x1)
+            return weakref.ref(x1)
+        t2set._dont_inline_ = True
+
         class WrFromThread:
             pass
         wr_from_thread = WrFromThread()
@@ -279,22 +287,30 @@
             config = objectmodel.fetch_translated_config()
             assert t.automatic_keepalive(config) is True
             wr = tset()
-            import gc; gc.collect()   # 'x1' should not be collected
-            x2 = t.get()
+            wr2 = t2set()
+            import gc; gc.collect()   # the two 'x1' should not be collected
+            x1 = t.get()
+            assert x1 is not None
+            assert wr() is not None
+            assert wr() is x1
+            assert x1.lst == [40, 2]
+            x2 = t2.get()
             assert x2 is not None
-            assert wr() is not None
-            assert wr() is x2
-            return wr
+            assert wr2() is not None
+            assert wr2() is x2
+            assert x2.lst == [50, 3]
+            return wr, wr2
 
         def thread_entry_point():
-            wr = f()
+            wr, wr2 = f()
             wr_from_thread.wr = wr
+            wr_from_thread.wr2 = wr2
             wr_from_thread.seen = True
 
         def main():
             wr_from_thread.seen = False
             start_new_thread(thread_entry_point, ())
-            wr1 = f()
+            wr1, wr2 = f()
             count = 0
             while True:
                 time.sleep(0.5)
@@ -302,10 +318,15 @@
                     break
                 count += 1
             assert wr_from_thread.seen is True
-            wr2 = wr_from_thread.wr
-            import gc; gc.collect()      # wr2() should be collected here
+            wr_other_1 = wr_from_thread.wr
+            wr_other_2 = wr_from_thread.wr2
+            import gc; gc.collect()      # wr_other_*() should be collected here
             assert wr1() is not None     # this thread, still running
-            assert wr2() is None         # other thread, not running any more
+            assert wr2() is not None     # this thread, still running
+            assert wr_other_1() is None  # other thread, not running any more
+            assert wr_other_2() is None  # other thread, not running any more
+            assert wr1().lst == [40, 2]
+            assert wr2().lst == [50, 3]
             return 42
 
         extra_options = {'no__thread': no__thread, 'shared': True}

From pypy.commits at gmail.com  Tue Mar 21 19:16:05 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 21 Mar 2017 16:16:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix py.test.skip call in app-level test.
Message-ID: <58d1b435.86c4190a.d9039.621a@mx.google.com>

Author: Manuel Jacob 
Branch: py3.5
Changeset: r90783:7db7635bfea8
Date: 2017-03-22 00:14 +0100
http://bitbucket.org/pypy/pypy/changeset/7db7635bfea8/

Log:	Fix py.test.skip call in app-level test.

diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -685,7 +685,7 @@
         assert isinstance(list({b'a': 1})[0], bytes)
 
     def test_interned_keywords(self):
-        py.test.skip("no longer works")
+        skip("no longer works")
         # At some point in the past, we had kwargsdict automatically
         # intern every single key we get out of it.  That's a big
         # pointless waste of time.  So the following test fails now.

From pypy.commits at gmail.com  Wed Mar 22 04:01:21 2017
From: pypy.commits at gmail.com (mjacob)
Date: Wed, 22 Mar 2017 01:01:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Implement PyModule_New().
Message-ID: <58d22f51.904d190a.a9636.12ac@mx.google.com>

Author: Manuel Jacob 
Branch: 
Changeset: r90784:af67b08ae873
Date: 2017-03-22 08:53 +0100
http://bitbucket.org/pypy/pypy/changeset/af67b08ae873/

Log:	Implement PyModule_New().

diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -10,6 +10,14 @@
 from pypy.module.cpyext.state import State
 from pypy.interpreter.error import oefmt
 
+ at cpython_api([rffi.CCHARP], PyObject)
+def PyModule_New(space, name):
+    """
+    Return a new module object with the __name__ attribute set to name.
+    Only the module's __doc__ and __name__ attributes are filled in;
+    the caller is responsible for providing a __file__ attribute."""
+    return Module(space, space.newtext(rffi.charp2str(name)))
+
 #@cpython_api([rffi.CCHARP], PyObject)
 def PyImport_AddModule(space, name):
     """Return the module object corresponding to a module name.  The name argument
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1365,13 +1365,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], PyObject)
-def PyModule_New(space, name):
-    """Return a new module object with the __name__ attribute set to name.  Only
-    the module's __doc__ and __name__ attributes are filled in; the caller is
-    responsible for providing a __file__ attribute."""
-    raise NotImplementedError
-
 @cpython_api([PyObject], rffi.CCHARP)
 def PyModule_GetFilename(space, module):
     """Return the name of the file from which module was loaded using module's
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -1,8 +1,15 @@
+from pypy.module.cpyext.modsupport import PyModule_New
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from rpython.rtyper.lltypesystem import rffi
 
 
 class TestModuleObject(BaseApiTest):
+    def test_module_new(self, space):
+        with rffi.scoped_str2charp('testname') as buf:
+            w_mod = PyModule_New(space, buf)
+        assert space.eq_w(space.getattr(w_mod, space.newtext('__name__')),
+                          space.newtext('testname'))
+
     def test_module_getname(self, space, api):
         w_sys = space.wrap(space.sys)
         p = api.PyModule_GetName(w_sys)

From pypy.commits at gmail.com  Wed Mar 22 04:01:23 2017
From: pypy.commits at gmail.com (mjacob)
Date: Wed, 22 Mar 2017 01:01:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58d22f53.83c3190a.4c2dc.124a@mx.google.com>

Author: Manuel Jacob 
Branch: py3.5
Changeset: r90785:eff72e17f613
Date: 2017-03-22 08:59 +0100
http://bitbucket.org/pypy/pypy/changeset/eff72e17f613/

Log:	hg merge default

diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -19,6 +19,14 @@
 def init_moduleobject(space):
     make_typedescr(Module.typedef, basestruct=PyModuleObject.TO)
 
+ at cpython_api([rffi.CCHARP], PyObject)
+def PyModule_New(space, name):
+    """
+    Return a new module object with the __name__ attribute set to name.
+    Only the module's __doc__ and __name__ attributes are filled in;
+    the caller is responsible for providing a __file__ attribute."""
+    return Module(space, space.newtext(rffi.charp2str(name)))
+
 @cpython_api([PyModuleDef, rffi.INT_real], PyObject)
 def PyModule_Create2(space, module, api_version):
     """Create a new module object, given the definition in module, assuming the
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1326,14 +1326,6 @@
     PyModule_Type."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], PyObject)
-def PyModule_New(space, name):
-    """
-    Return a new module object with the __name__ attribute set to name.
-    Only the module's __doc__ and __name__ attributes are filled in;
-    the caller is responsible for providing a __file__ attribute."""
-    raise NotImplementedError
-
 @cpython_api([PyObject], rffi.CCHARP)
 def PyModule_GetFilename(space, module):
     """Similar to PyModule_GetFilenameObject() but return the filename
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -1,9 +1,16 @@
+from pypy.module.cpyext.modsupport import PyModule_New
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi
 
 
 class TestModuleObject(BaseApiTest):
+    def test_module_new(self, space):
+        with rffi.scoped_str2charp('testname') as buf:
+            w_mod = PyModule_New(space, buf)
+        assert space.eq_w(space.getattr(w_mod, space.newtext('__name__')),
+                          space.newtext('testname'))
+
     def test_module_getname(self, space, api):
         w_sys = space.wrap(space.sys)
         p = api.PyModule_GetName(w_sys)

From pypy.commits at gmail.com  Wed Mar 22 04:02:48 2017
From: pypy.commits at gmail.com (Raemi)
Date: Wed, 22 Mar 2017 01:02:48 -0700 (PDT)
Subject: [pypy-commit] pypy nogil-unsafe-2: workaround for tests passing
 only because of rpython's platform-check cache
Message-ID: <58d22fa8.15ca190a.faf9a.1295@mx.google.com>

Author: Remi Meier 
Branch: nogil-unsafe-2
Changeset: r90786:ad597a412706
Date: 2017-03-22 09:02 +0100
http://bitbucket.org/pypy/pypy/changeset/ad597a412706/

Log:	workaround for tests passing only because of rpython's platform-
	check cache

	apparently the initial platform check also tries to compile all C
	files mentioned in separate_module_files. Since thread.c currently
	does not compile without structdef.h & friends, the platform check
	actually fails. It only worked by chance since rpython apparently
	maintains a '_cache' folder with previous successful compilations.

	This commit prevents all C files to be compiled during said platform
	check. Unsure if this is OK or why it is done before...

diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -65,7 +65,11 @@
         return result
 
     def _compile_o_files(self, cfiles, eci, standalone=True):
-        cfiles = self._all_cfiles(cfiles, eci)
+        # XXX: why does platform-check add all known C files here?
+        # apparently it adds, e.g., thread.c if rthread is used. And then
+        # compiles them already. Is this necessary? (thread.c does currently
+        # not compile on its own...)
+        # cfiles = self._all_cfiles(cfiles, eci)
         compile_args = self._compile_args_from_eci(eci, standalone)
         ofiles = []
         for cfile in cfiles:

From pypy.commits at gmail.com  Wed Mar 22 04:16:24 2017
From: pypy.commits at gmail.com (Raemi)
Date: Wed, 22 Mar 2017 01:16:24 -0700 (PDT)
Subject: [pypy-commit] pypy nogil-unsafe-2: do the previous workaround only
 in the problematic case
Message-ID: <58d232d8.1bd8190a.94c58.144f@mx.google.com>

Author: Remi Meier 
Branch: nogil-unsafe-2
Changeset: r90787:856f0f42a1ee
Date: 2017-03-22 09:15 +0100
http://bitbucket.org/pypy/pypy/changeset/856f0f42a1ee/

Log:	do the previous workaround only in the problematic case

diff --git a/rpython/tool/gcc_cache.py b/rpython/tool/gcc_cache.py
--- a/rpython/tool/gcc_cache.py
+++ b/rpython/tool/gcc_cache.py
@@ -25,7 +25,8 @@
         try:
             if ignore_errors:
                 platform.log_errors = False
-            result = platform.execute(platform.compile(c_files, eci))
+            result = platform.execute(platform.compile(c_files, eci,
+                                                       all_modules=False))
             if result.err:
                 sys.stderr.write(result.err)
         finally:
diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -38,7 +38,6 @@
 RPY_EXTERN void RPyGilLeaveMasterSection(void);
 RPY_EXTERN void RPyGilMasterRequestSafepoint(void);
 
-
 #define RPyGilAcquire _RPyGilAcquire
 #define RPyGilRelease _RPyGilRelease
 #define RPyFetchFastGil _RPyFetchFastGil
diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -50,8 +50,9 @@
             raise TypeError("You should not instantiate Platform class directly")
         self.cc = cc
 
-    def compile(self, cfiles, eci, outputfilename=None, standalone=True):
-        ofiles = self._compile_o_files(cfiles, eci, standalone)
+    def compile(self, cfiles, eci, outputfilename=None, standalone=True,
+                all_modules=True):
+        ofiles = self._compile_o_files(cfiles, eci, standalone, all_modules)
         return self._finish_linking(ofiles, eci, outputfilename, standalone)
 
     def _all_cfiles(self, cfiles, eci):
@@ -64,12 +65,13 @@
                 result.append(cfile)
         return result
 
-    def _compile_o_files(self, cfiles, eci, standalone=True):
-        # XXX: why does platform-check add all known C files here?
-        # apparently it adds, e.g., thread.c if rthread is used. And then
-        # compiles them already. Is this necessary? (thread.c does currently
-        # not compile on its own...)
-        # cfiles = self._all_cfiles(cfiles, eci)
+    def _compile_o_files(self, cfiles, eci, standalone=True, all_modules=True):
+        if all_modules:
+            # XXX: why does platform-check add all known C files here?
+            # apparently it adds, e.g., thread.c if rthread is used. And then
+            # compiles them already. Is this necessary? (thread.c does currently
+            # not compile on its own...)
+            cfiles = self._all_cfiles(cfiles, eci)
         compile_args = self._compile_args_from_eci(eci, standalone)
         ofiles = []
         for cfile in cfiles:

From pypy.commits at gmail.com  Wed Mar 22 04:39:09 2017
From: pypy.commits at gmail.com (Raemi)
Date: Wed, 22 Mar 2017 01:39:09 -0700 (PDT)
Subject: [pypy-commit] pypy nogil-unsafe-2: hacking around until test_thread
 under memory/gc can run
Message-ID: <58d2382d.49a9190a.b4292.151d@mx.google.com>

Author: Remi Meier 
Branch: nogil-unsafe-2
Changeset: r90788:5303f96d7faa
Date: 2017-03-22 09:38 +0100
http://bitbucket.org/pypy/pypy/changeset/5303f96d7faa/

Log:	hacking around until test_thread under memory/gc can run

diff --git a/rpython/translator/c/src/thread.c b/rpython/translator/c/src/thread.c
--- a/rpython/translator/c/src/thread.c
+++ b/rpython/translator/c/src/thread.c
@@ -9,10 +9,49 @@
 #include "common_header.h"
 #endif
 
+#ifdef RPYTHON_LL2CTYPES
+// only for python tests
+
+#define RPY_TLOFS_alt_errno  offsetof(struct pypy_threadlocal_s, alt_errno)
+#define RPY_TLOFS_nursery_free  offsetof(struct pypy_threadlocal_s, nursery_free)
+#define RPY_TLOFS_nursery_top  offsetof(struct pypy_threadlocal_s, nursery_top)
+#define RPY_TLOFS_rpy_errno  offsetof(struct pypy_threadlocal_s, rpy_errno)
+#define RPY_TLOFS_shadowstack  offsetof(struct pypy_threadlocal_s, shadowstack)
+#define RPY_TLOFS_shadowstack_top  offsetof(struct pypy_threadlocal_s, shadowstack_top)
+#define RPY_TLOFS_synclock  offsetof(struct pypy_threadlocal_s, synclock)
+struct pypy_threadlocal_s {
+    int ready;
+    char *stack_end;
+    struct pypy_threadlocal_s *prev, *next;
+    int alt_errno;
+    void* nursery_free;
+    void* nursery_top;
+    int rpy_errno;
+    void* shadowstack;
+    void* shadowstack_top;
+    Signed synclock;
+};
+
+/* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
+#ifndef RPY_EXTERN
+#define RPY_EXTERN RPY_EXPORTED
+#endif
+
+#define USE__THREAD 1
+
+/* RPY_EXTERN __thread struct pypy_threadlocal_s pypy_threadlocal = { 0 }; */
+
+#include "threadlocal.c"
+
+#else
+
 # include "common_header.h"
 # include "structdef.h"
 # include "forwarddecl.h"
 
+#endif
+
+
 #ifdef _WIN32
 #include "src/thread_nt.c"
 #else
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -1,5 +1,8 @@
-#include "common_header.h"
-#include "structdef.h"       /* for struct pypy_threadlocal_s */
+#ifndef RPYTHON_LL2CTYPES
+# include "common_header.h"
+# include "structdef.h"       /* for struct pypy_threadlocal_s */
+#endif
+
 #include 
 #include 
 #include 
@@ -208,7 +211,7 @@
 
 
 /* ------------------------------------------------------------ */
-#ifdef USE___THREAD
+/* #ifdef USE___THREAD XXX */
 /* ------------------------------------------------------------ */
 
 
@@ -235,39 +238,39 @@
 }
 
 
-/* ------------------------------------------------------------ */
-#else
-/* ------------------------------------------------------------ */
+/* /\* ------------------------------------------------------------ *\/ */
+/* #else */
+/* /\* ------------------------------------------------------------ *\/ */
 
 
-/* this is the case where the 'struct pypy_threadlocal_s' is allocated
-   explicitly, with malloc()/free(), and attached to (a single) thread-
-   local key using the API of Windows or pthread. */
+/* /\* this is the case where the 'struct pypy_threadlocal_s' is allocated */
+/*    explicitly, with malloc()/free(), and attached to (a single) thread- */
+/*    local key using the API of Windows or pthread. *\/ */
 
 
-char *_RPython_ThreadLocals_Build(void)
-{
-    void *p = malloc(sizeof(struct pypy_threadlocal_s));
-    if (!p) {
-        fprintf(stderr, "Internal RPython error: "
-                        "out of memory for the thread-local storage");
-        abort();
-    }
-    _RPy_ThreadLocals_Init(p);
-    _RPy_ThreadLocals_Set(p);
-    return (char *)p;
-}
+/* char *_RPython_ThreadLocals_Build(void) */
+/* { */
+/*     void *p = malloc(sizeof(struct pypy_threadlocal_s)); */
+/*     if (!p) { */
+/*         fprintf(stderr, "Internal RPython error: " */
+/*                         "out of memory for the thread-local storage"); */
+/*         abort(); */
+/*     } */
+/*     _RPy_ThreadLocals_Init(p); */
+/*     _RPy_ThreadLocals_Set(p); */
+/*     return (char *)p; */
+/* } */
 
-void RPython_ThreadLocals_ThreadDie(void)
-{
-    void *p = _RPy_ThreadLocals_Get();
-    if (p != NULL) {
-        _RPy_ThreadLocals_Set(NULL);
-        threadloc_unlink(p);   /* includes free(p) */
-    }
-}
+/* void RPython_ThreadLocals_ThreadDie(void) */
+/* { */
+/*     void *p = _RPy_ThreadLocals_Get(); */
+/*     if (p != NULL) { */
+/*         _RPy_ThreadLocals_Set(NULL); */
+/*         threadloc_unlink(p);   /\* includes free(p) *\/ */
+/*     } */
+/* } */
 
 
-/* ------------------------------------------------------------ */
-#endif
-/* ------------------------------------------------------------ */
+/* /\* ------------------------------------------------------------ *\/ */
+/* #endif */
+/* /\* ------------------------------------------------------------ *\/ */

From pypy.commits at gmail.com  Wed Mar 22 08:49:20 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Mar 2017 05:49:20 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-native: pass two own-linux tests by
 adding rvmprof/src to the path and ensure that RPYTHON_VMPROF is defined
Message-ID: <58d272d0.de13190a.390a4.2e9f@mx.google.com>

Author: Richard Plangger 
Branch: vmprof-native
Changeset: r90789:8827c130f035
Date: 2017-03-22 09:48 -0300
http://bitbucket.org/pypy/pypy/changeset/8827c130f035/

Log:	pass two own-linux tests by adding rvmprof/src to the path and
	ensure that RPYTHON_VMPROF is defined

diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -5,10 +5,14 @@
 
 
 cwd = py.path.local(__file__).dirpath()
+rvmp = cwd.join('../../..')
+rvmp = rvmp.join('rpython/rlib/rvmprof/src')
+
 eci = ExternalCompilationInfo(
     includes=[cwd.join('faulthandler.h')],
-    include_dirs=[str(cwd), cdir],
-    separate_module_files=[cwd.join('faulthandler.c')])
+    include_dirs=[str(cwd), cdir, rvmp],
+    separate_module_files=[cwd.join('faulthandler.c')],
+    compile_extra=['-DRPYTHON_VMPROF=1'])
 
 eci_later = eci.merge(ExternalCompilationInfo(
     pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n']))

From pypy.commits at gmail.com  Wed Mar 22 14:04:11 2017
From: pypy.commits at gmail.com (tobweber)
Date: Wed, 22 Mar 2017 11:04:11 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Fix calculation
 of duration when a second jump lies between start and stop but the duration
 is less than a second, i.e.,
 add carry over from the nanoseconds to the seconds component
Message-ID: <58d2bc9b.c4cf190a.cfbf6.4287@mx.google.com>

Author: Tobias Weber 
Branch: c8-overheads-instrumentation
Changeset: r2034:0e47a33eecfb
Date: 2017-03-22 18:22 +0100
http://bitbucket.org/pypy/stmgc/changeset/0e47a33eecfb/

Log:	Fix calculation of duration when a second jump lies between start
	and stop but the duration is less than a second, i.e., add carry
	over from the nanoseconds to the seconds component

diff --git a/c8/stm/timing.h b/c8/stm/timing.h
--- a/c8/stm/timing.h
+++ b/c8/stm/timing.h
@@ -7,11 +7,18 @@
    runtime. */
 #define start_timer() struct timespec start, stop;                             \
                       struct timespec duration = { .tv_sec = 0, .tv_nsec = 0 };\
+                      uint32_t nanosec_diff, sec_diff;                         \
                       continue_timer()
 
 /* Must use start_timer before using this macro. */
-#define get_duration() duration.tv_sec += stop.tv_sec - start.tv_sec;       \
-                       duration.tv_nsec += stop.tv_nsec - start.tv_nsec;
+#define get_duration() nanosec_diff = stop.tv_nsec - start.tv_nsec;         \
+                       sec_diff = stop.tv_sec - start.tv_sec;               \
+                       if (stop.tv_nsec < start.tv_nsec) {                  \
+                           nanosec_diff += 1000000000;                      \
+                           sec_diff -= 1;                                   \
+                       }                                                    \
+                       duration.tv_sec += sec_diff;                         \
+                       duration.tv_nsec += nanosec_diff;
 
 #define pause_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &stop);            \
                       get_duration()

From pypy.commits at gmail.com  Wed Mar 22 14:30:06 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Mar 2017 11:30:06 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-native: merge default
Message-ID: <58d2c2ae.c4052e0a.9655.0ae6@mx.google.com>

Author: Richard Plangger 
Branch: vmprof-native
Changeset: r90790:dd7413118eca
Date: 2017-03-22 11:45 -0300
http://bitbucket.org/pypy/pypy/changeset/dd7413118eca/

Log:	merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -80,5 +80,6 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
+^\.cache$
 
 pypy/module/cppyy/.+/*\.pcm
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -34,3 +34,5 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
+b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -27,14 +27,19 @@
 Building
 ========
 
-build with:
+First switch to or download the correct branch.  The basic choices are
+``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X
+branch (e.g. ``py3.5``).
+
+Build with:
 
 .. code-block:: console
 
     $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
 
-This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
-to use virtualenv with the resulting pypy-c as the interpreter; you can
-find more details about various installation schemes here:
+This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy
+directory.  We suggest to use virtualenv with the resulting
+pypy-c/pypy3-c as the interpreter; you can find more details about
+various installation schemes here:
 
     http://doc.pypy.org/en/latest/install.html
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -29,8 +29,8 @@
     'pypy': {
         'stdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
         'platstdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
-        'purelib': '{base}/lib-{implementation_lower}/{py_version_short}',
-        'platlib': '{base}/lib-{implementation_lower}/{py_version_short}',
+        'purelib': '{base}/site-packages',
+        'platlib': '{base}/site-packages',
         'include': '{base}/include',
         'platinclude': '{base}/include',
         'scripts': '{base}/bin',
diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -36,9 +36,9 @@
 except ImportError:
     def _delitem_if_value_is(d, key, value):
         try:
-            if self.data[key] is value:  # fall-back: there is a potential
+            if d[key] is value:  # fall-back: there is a potential
                 #             race condition in multithreaded programs HERE
-                del self.data[key]
+                del d[key]
         except KeyError:
             pass
 
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -110,22 +110,22 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev \
+    tk-dev libgc-dev python-cffi \
     liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel \
+    gdbm-devel python-cffi\
     xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses \
-    xz-devel  # For lzma on PyPy3.
+    libexpat-devel libffi-devel python-curses python-cffi \
+    xz-devel # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
 On Mac OS X, most of these build-time dependencies are installed alongside
diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst
--- a/pypy/doc/release-v5.7.0.rst
+++ b/pypy/doc/release-v5.7.0.rst
@@ -2,23 +2,27 @@
 PyPy2.7 and PyPy3.5 v5.7 - two in one release
 =============================================
 
-We have released PyPy2.7 and a beta-quality PyPy3.5 v5.7.
+The PyPy team is proud to release both PyPy2.7 v5.7 (an interpreter supporting
+Python v2.7 syntax), and a beta-quality PyPy3.5 v5.7 (an interpreter for Python
+v3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
-PyPy 3.5 (our first in the 3.5 series) includes the upstream stdlib version
+PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
 3.5.3.
 
 We continue to make incremental improvements to our C-API
-compatibility layer (cpyext). PyPy2 can now import and run many c-extension
-packages, among the most notable are numpy, cython, and pandas. Performance may
+compatibility layer (cpyext). PyPy2 can now import and run many C-extension
+packages, among the most notable are Numpy, Cython, and Pandas. Performance may
 be slower than CPython, especially for frequently-called short C functions.
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
 Work proceeds at a good pace on the PyPy3.5
 version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta
-release. Thanks Mozilla !!! While we do not pass all tests, asyncio works and
+release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
 as `these benchmarks show`_ it already gives a nice speed bump.
-We also backported the ``f""`` formatting from 3.6 (as an expection; otherwise
+We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
 "PyPy3.5" supports the Python 3.5 language).
 
 CFFI_ has been updated to 1.10, improving an already great package for
@@ -65,7 +69,7 @@
 We also welcome developers of other `dynamic languages`_ to see what RPython
 can do for them.
 
-This release supports: 
+The PyPy 2.7 release supports: 
 
   * **x86** machines on most common operating systems
     (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
@@ -80,63 +84,60 @@
 .. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
 
 Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.6 released Nov, 2016)
-=============================================================================================
+=====================================================================================
 
 See also issues that were resolved_
 
 * New features and cleanups
 
   * update the format of the PYPYLOG file and improvements to vmprof
-  * improve the consistency of RPython annotation unions
-  * emit more sysconfig values for downstream cextension packages
-  * add PyAnySet_Check, PyModule_GetName, PyWeakref_Check*,
-    _PyImport_{Acquire,Release}Lock, PyGen_Check*, PyOS_AfterFork,
-  * add translation option --keepgoing to continue after the first AnnotationError
+  * emit more sysconfig values for downstream cextension packages including
+    properly setting purelib and platlib to site-packages
+  * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``,
+    ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``,
   * detect and raise on recreation of a PyPy object from a PyObject during
     tp_dealloc
   * refactor and clean up poor handling of unicode exposed in work on py3.5
-  * builtin cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed)
-  * add translation time --disable_entrypoints option for embedding PyPy together
-    with another RPython VM
-  * adapt ``weakref`` according to Python issue #19542, will be in CPython 2.7.14
+  * builtin module cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed)
+  * adapt ``weakref`` according to CPython issue #19542_, will be in CPython 2.7.14
   * support translations with cpyext and the Boehm GC (for special cases like
-    revdb
+    RevDB_
   * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is
     now possible to obtain the address of any readonly object without pinning it
   * refactor the initialization code in translating cpyext
-  * fix ``"".replace("", "x", num)`` to give the same result as CPython
   * use a cffi-style C parser to create rffi objects in cpyext, now the
-    translating python must have cffi available
-  * add a rpython implementation of siphash24, allow choosing hash algorithm
-    randomizing the seed
-  * make ``attach_gdb`` work on Windows (with Visual Studio Debugger)
+    translating Python must have either ``cffi`` or ``pycparser`` available
   * implement ``move_to_end(last=True/False)`` on RPython ordered dicts, make
     available as ``__pypy__.move_to_end`` and, on py3.5,
     ``OrderedDict.move_to_end()``
   * remove completely RPython ``space.wrap`` in a major cleanup, differentiate
     between ``space.newtext`` and ``space.newbytes`` on py3.5
-  * improve shadowstack to where it is now the default in place of asmgcc
+  * any uncaught RPython exception in the interpreter is turned into a
+    SystemError (rather than a segfault)
+  * add translation time --disable_entrypoints option for embedding PyPy together
+    with another RPython VM
+
 
 * Bug Fixes
 
-  * any uncaught RPython exception in the interpreter is turned into a
-    SystemError (rather than a segfault)
+  * fix ``"".replace("", "x", num)`` to give the same result as CPython
   * create log files without the executable bit
-  * disable clock_gettime() on OS/X, since we support 10.11 and it was only
+  * disable ``clock_gettime()`` on OS/X, since we support 10.11 and it was only
     added in 10.12
-  * support HAVE_FSTATVFS which was unintentionally always false
-  * fix user-created C-API heaptype, issue #2434
-  * fix PyDict_Update is not actually the same as dict.update
-  * assign tp_doc on PyTypeObject and tie it to the app-level __doc__ attribute
-    issue #2446
+  * support ``HAVE_FSTATVFS`` which was unintentionally always false
+  * fix user-created C-API heaptype, issue #2434_
+  * fix ``PyDict_Update`` is not actually the same as ``dict.update``
+  * assign ``tp_doc`` on ``PyTypeObject`` and tie it to the app-level ``__doc__`` attribute
+    issue #2446_
   * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``,
     ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release``
-  * improve support for creating c-extension objects from app-level classes,
-    filling more slots especially ``tp_new`` and ``tp_dealloc``
-  * add rstack.stack_almost_full() and use it to avoid stack overflow due to
-    the JIT where possible
-  * fix for ctypes.c_bool returning bool restype issue #2475
+  * improve support for creating C-extension objects from app-level classes,
+    filling more slots, especially ``tp_new`` and ``tp_dealloc``
+  * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_
   * fix in corner cases with the GIL and C-API functions
+  * allow overriding thread.local.__init__ in a subclass, issue #2501_
+  * allow ``PyClass_New`` to be called with NULL as the first arguemnt, issue #2504_
+
 
 * Performance improvements:
 
@@ -158,17 +159,28 @@
     information across failing guards
   * add optimized "zero-copy" path for ``io.FileIO.readinto``
 
+* RPython improvements
+
+  * improve the consistency of RPython annotation unions
+  * add translation option --keepgoing to continue after the first AnnotationError
+  * improve shadowstack to where it is now the default in place of asmgcc
+  * add a rpython implementation of siphash24, allow choosing hash algorithm
+    randomizing the seed
+  * add rstack.stack_almost_full() and use it to avoid stack overflow due to
+    the JIT where possible
+
 Highlights of the PyPy3.5 release (since 5.5 alpha released Oct, 2016)
-=========================================================
+==========================================================================
 
-Development moved from the py3k branch to the py3.5 branch in the pypy bitbucket repo
+Development moved from the py3k branch to the py3.5 branch in the PyPy bitbucket repo.
 
 * New features
 
-  * this first PyPy3.5 release implements much, but not all, of Python 3.5.3
+  * this first PyPy3.5 release implements most of Python 3.5.3, exceptions are listed below
   * PEP 456 allowing secure and interchangable hash algorithms
   * use cryptography_'s cffi backend for SSL
 
+
 * Bug Fixes
 
   * implement fixes for some CPython issues that arose since the last release 
@@ -176,12 +188,21 @@
 
 * Performance improvements:
 
-  * do not create a list whenever descr_new of a bytesobject is called
-  * 
-  * 
-  * 
+  * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called
+
+* The following features of Python 3.5 are not implemented yet in PyPy:
+
+  * PEP 442: Safe object finalization
+  * PEP 489: Multi-phase extension module initialization
 
 .. _resolved: whatsnew-pypy2-5.7.0.html
+.. _19542: https://bugs.python.org/issue19542
+.. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys
+.. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on
+.. _2475: https://bitbucket.org/pypy/pypy/issues/2475
+.. _2501: https://bitbucket.org/pypy/pypy/issues/2501
+.. _2504: https://bitbucket.org/pypy/pypy/issues/2504
+.. _RevDB: https://bitbucket.org/pypy/revdb
 .. _cryptography: https://cryptography.io
 .. _cppyy: cppyy.html
 
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -48,3 +48,4 @@
     w_modules_dict = space.sys.get('modules')
     space.setitem(w_modules_dict, w_name, module)
     space.setitem(w_modules_dict, space.newtext(name + '.lib'), lib)
+    return module
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1469,10 +1469,6 @@
     copy_header_files(cts, trunk_include, use_micronumpy)
 
 
-def _load_from_cffi(space, name, path, initptr):
-    from pypy.module._cffi_backend import cffi1_module
-    cffi1_module.load_cffi1_module(space, name, path, initptr)
-
 @unwrap_spec(path='text', name='text')
 def load_extension_module(space, path, name):
     # note: this is used both to load CPython-API-style C extension
@@ -1505,11 +1501,11 @@
             pass
         else:
             try:
-                _load_from_cffi(space, name, path, initptr)
+                from pypy.module._cffi_backend import cffi1_module
+                return cffi1_module.load_cffi1_module(space, name, path, initptr)
             except:
                 rdynload.dlclose(dll)
                 raise
-            return
     #
     if space.config.objspace.usemodules.cpyext:
         also_look_for = 'init%s' % (basename,)
@@ -1518,8 +1514,7 @@
         except KeyError:
             pass
         else:
-            load_cpyext_module(space, name, path, dll, initptr)
-            return
+            return load_cpyext_module(space, name, path, dll, initptr)
         if look_for is not None:
             look_for += ' or ' + also_look_for
         else:
@@ -1535,9 +1530,10 @@
 
     space.getbuiltinmodule("cpyext")    # mandatory to init cpyext
     state = space.fromcache(State)
-    if state.find_extension(name, path) is not None:
+    w_mod = state.find_extension(name, path)
+    if w_mod is not None:
         rdynload.dlclose(dll)
-        return
+        return w_mod
     old_context = state.package_context
     state.package_context = name, path
     try:
@@ -1546,7 +1542,8 @@
         state.check_and_raise_exception()
     finally:
         state.package_context = old_context
-    state.fixup_extension(name, path)
+    w_mod = state.fixup_extension(name, path)
+    return w_mod
 
 @specialize.ll()
 def generic_cpy_call(space, func, *args):
diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py
--- a/pypy/module/cpyext/classobject.py
+++ b/pypy/module/cpyext/classobject.py
@@ -39,6 +39,8 @@
 
 @cpython_api([PyObject, PyObject, PyObject], PyObject)
 def PyClass_New(space, w_bases, w_dict, w_name):
+    if w_bases is None:
+        w_bases = space.newtuple([])
     w_classobj = space.gettypefor(W_ClassObject)
     return space.call_function(w_classobj,
                                w_name, w_bases, w_dict)
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -10,6 +10,14 @@
 from pypy.module.cpyext.state import State
 from pypy.interpreter.error import oefmt
 
+ at cpython_api([rffi.CCHARP], PyObject)
+def PyModule_New(space, name):
+    """
+    Return a new module object with the __name__ attribute set to name.
+    Only the module's __doc__ and __name__ attributes are filled in;
+    the caller is responsible for providing a __file__ attribute."""
+    return Module(space, space.newtext(rffi.charp2str(name)))
+
 #@cpython_api([rffi.CCHARP], PyObject)
 def PyImport_AddModule(space, name):
     """Return the module object corresponding to a module name.  The name argument
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -165,6 +165,7 @@
         w_dict = w_mod.getdict(space)
         w_copy = space.call_method(w_dict, 'copy')
         self.extensions[path] = w_copy
+        return w_mod
 
 
 def _rawrefcount_perform(space):
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1365,13 +1365,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], PyObject)
-def PyModule_New(space, name):
-    """Return a new module object with the __name__ attribute set to name.  Only
-    the module's __doc__ and __name__ attributes are filled in; the caller is
-    responsible for providing a __file__ attribute."""
-    raise NotImplementedError
-
 @cpython_api([PyObject], rffi.CCHARP)
 def PyModule_GetFilename(space, module):
     """Return the name of the file from which module was loaded using module's
diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py
--- a/pypy/module/cpyext/test/test_classobject.py
+++ b/pypy/module/cpyext/test/test_classobject.py
@@ -77,3 +77,15 @@
         class C:
             pass
         assert module.get_classtype() is type(C)
+
+    def test_pyclass_new_no_bases(self):
+        module = self.import_extension('foo', [
+            ("new_foo", "METH_O",
+             """
+                 return PyClass_New(NULL, PyDict_New(), args);
+             """)])
+        FooClass = module.new_foo("FooClass")
+        class Cls1:
+            pass
+        assert type(FooClass) is type(Cls1)
+        assert FooClass.__bases__ == Cls1.__bases__
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -1,8 +1,15 @@
+from pypy.module.cpyext.modsupport import PyModule_New
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from rpython.rtyper.lltypesystem import rffi
 
 
 class TestModuleObject(BaseApiTest):
+    def test_module_new(self, space):
+        with rffi.scoped_str2charp('testname') as buf:
+            w_mod = PyModule_New(space, buf)
+        assert space.eq_w(space.getattr(w_mod, space.newtext('__name__')),
+                          space.newtext('testname'))
+
     def test_module_getname(self, space, api):
         w_sys = space.wrap(space.sys)
         p = api.PyModule_GetName(w_sys)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -616,7 +616,7 @@
     from pypy.module.cpyext.api import load_extension_module
     log_pyverbose(space, 1, "import %s # from %s\n" %
                   (modulename, filename))
-    load_extension_module(space, filename, modulename)
+    return load_extension_module(space, filename, modulename)
     # NB. cpyext.api.load_extension_module() can also delegate to _cffi_backend
 
 @jit.dont_look_inside
@@ -680,8 +680,8 @@
                         pass
                 return w_mod
             elif find_info.modtype == C_EXTENSION and has_so_extension(space):
-                load_c_extension(space, find_info.filename, space.text_w(w_modulename))
-                return check_sys_modules(space, w_modulename)
+                return load_c_extension(space, find_info.filename,
+                                        space.text_w(w_modulename))
         except OperationError:
             w_mods = space.sys.get('modules')
             space.call_method(w_mods, 'pop', w_modulename, space.w_None)
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -136,8 +136,8 @@
 def load_dynamic(space, w_modulename, filename, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
-    importing.load_c_extension(space, filename, space.text_w(w_modulename))
-    return importing.check_sys_modules(space, w_modulename)
+    return importing.load_c_extension(space, filename,
+                                      space.text_w(w_modulename))
 
 def new_module(space, w_name):
     return Module(space, w_name, add_package=False)
diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py
--- a/pypy/module/thread/os_local.py
+++ b/pypy/module/thread/os_local.py
@@ -1,6 +1,7 @@
 import weakref
 from rpython.rlib import jit
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt
 from pypy.interpreter.executioncontext import ExecutionContext
 from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty,
     descr_get_dict)
@@ -74,18 +75,20 @@
         return w_dict
 
     def descr_local__new__(space, w_subtype, __args__):
+        from pypy.objspace.std.typeobject import _precheck_for_new
+        w_subtype = _precheck_for_new(space, w_subtype)
+        if __args__.arguments_w or __args__.keywords:
+            w_parent_init, _ = space.lookup_in_type_where(w_subtype, '__init__')
+            if w_parent_init is space.w_object:
+                raise oefmt(space.w_TypeError,
+                            "Initialization arguments are not supported")
         local = space.allocate_instance(Local, w_subtype)
         Local.__init__(local, space, __args__)
         return local
 
-    def descr_local__init__(self, space):
-        # No arguments allowed
-        pass
-
 Local.typedef = TypeDef("thread._local",
                         __doc__ = "Thread-local data",
                         __new__ = interp2app(Local.descr_local__new__.im_func),
-                        __init__ = interp2app(Local.descr_local__init__),
                         __dict__ = GetSetProperty(descr_get_dict, cls=Local),
                         )
 
diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py
--- a/pypy/module/thread/test/test_local.py
+++ b/pypy/module/thread/test/test_local.py
@@ -72,6 +72,19 @@
         assert seen1 == [1, 2, 3, 4, 5]
         assert tags == ['???']
 
+    def test_local_init2(self):
+        import thread
+
+        class A(object):
+            def __init__(self, n):
+                assert n == 42
+                self.n = n
+        class X(thread._local, A):
+            pass
+
+        x = X(42)
+        assert x.n == 42
+
     def test_local_setdict(self):
         import thread
         x = thread._local()
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -263,11 +263,18 @@
                     if compiled:
                         w_result = self.import_pyc_file(space, fullname, fname,
                                                         buf, pkgpath)
-                        if w_result is not None:
-                            return w_result
+                        if w_result is None:
+                            continue
                     else:
-                        return self.import_py_file(space, fullname, fname,
+                        w_result = self.import_py_file(space, fullname, fname,
                                                    buf, pkgpath)
+                    if space.sys.get_flag('verbose') >= 1:
+                        w_stderr = space.sys.get('stderr')
+                        message = "import %s # loaded from Zip %s%s%s\n" % (
+                                fullname, self.filename, os.path.sep, fname)
+                        space.call_method(w_stderr, "write",
+                                          space.newtext(message))
+                    return w_result
                 except:
                     w_mods = space.sys.get('modules')
                     space.call_method(w_mods, 'pop', space.newtext(fullname), space.w_None)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1049,6 +1049,8 @@
             else:
                 return d.pop(key, w_default)
         elif self._never_equal_to(space.type(w_key)):
+            if w_default is not None:
+                return w_default
             raise KeyError
         else:
             self.switch_to_object_strategy(w_dict)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -161,7 +161,7 @@
         w_d.initialize_content([(w(1), wb("a")), (w(2), wb("b"))])
         w_l = self.space.call_method(w_d, "keys")
         assert sorted(self.space.listview_int(w_l)) == [1,2]
-        
+
         # make sure that .keys() calls newlist_bytes for string dicts
         def not_allowed(*args):
             assert False, 'should not be called'
@@ -174,7 +174,7 @@
 
         # XXX: it would be nice if the test passed without monkeypatch.undo(),
         # but we need space.newlist_unicode for it
-        monkeypatch.undo() 
+        monkeypatch.undo()
         w_d = self.space.newdict()
         w_d.initialize_content([(w(u"a"), w(1)), (w(u"b"), w(6))])
         w_l = self.space.call_method(w_d, "keys")
@@ -223,6 +223,10 @@
         assert len(dd) == 1
         raises(KeyError, dd.pop, 33)
 
+        assert d.pop("abc", None) is None
+        raises(KeyError, d.pop, "abc")
+        assert len(d) == 2
+
     def test_has_key(self):
         d = {1: 2, 3: 4}
         assert d.has_key(1)
@@ -1466,4 +1470,3 @@
     fakespace = FakeSpace()
     d = fakespace.newdict(module=True)
     assert type(d.get_strategy()) is BytesDictStrategy
-
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,6 +1,6 @@
 # Edit these appropriately before running this script
 maj=5
-min=6
+min=7
 rev=0
 branchname=release-pypy2.7-5.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -429,7 +429,11 @@
                 gc._trace_callback(callback, arg, p + offset)
             llop.threadlocalref_release(lltype.Void)
         _lambda_trace_tlref = lambda: _trace_tlref
-        TRACETLREF = lltype.GcStruct('TRACETLREF')
+        # WAAAH obscurity: can't use a name that may be non-unique,
+        # otherwise the types compare equal, even though we call
+        # register_custom_trace_hook() to register different trace
+        # functions...
+        TRACETLREF = lltype.GcStruct('TRACETLREF%d' % unique_id)
         _tracetlref_obj = lltype.malloc(TRACETLREF, immortal=True)
 
     @staticmethod
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -99,8 +99,11 @@
         eci = eci.merge(ExternalCompilationInfo(includes=['linux/random.h']))
         class CConfig:
             _compilation_info_ = eci
-            GRND_NONBLOCK = rffi_platform.ConstantInteger('GRND_NONBLOCK')
+            GRND_NONBLOCK = rffi_platform.DefinedConstantInteger(
+                'GRND_NONBLOCK')
         globals().update(rffi_platform.configure(CConfig))
+        if GRND_NONBLOCK is None:
+            GRND_NONBLOCK = 0x0001      # from linux/random.h
 
         # On Linux, use the syscall() function because the GNU libc doesn't
         # expose the Linux getrandom() syscall yet.
diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py
--- a/rpython/rlib/test/test_rthread.py
+++ b/rpython/rlib/test/test_rthread.py
@@ -262,15 +262,23 @@
             py.test.skip("no __thread support here")
 
         class FooBar(object):
-            pass
+            def __init__(self, a, b):
+                self.lst = [a, b]
         t = ThreadLocalReference(FooBar)
+        t2 = ThreadLocalReference(FooBar)
 
         def tset():
-            x1 = FooBar()
+            x1 = FooBar(40, 2)
             t.set(x1)
             return weakref.ref(x1)
         tset._dont_inline_ = True
 
+        def t2set():
+            x1 = FooBar(50, 3)
+            t2.set(x1)
+            return weakref.ref(x1)
+        t2set._dont_inline_ = True
+
         class WrFromThread:
             pass
         wr_from_thread = WrFromThread()
@@ -279,22 +287,30 @@
             config = objectmodel.fetch_translated_config()
             assert t.automatic_keepalive(config) is True
             wr = tset()
-            import gc; gc.collect()   # 'x1' should not be collected
-            x2 = t.get()
+            wr2 = t2set()
+            import gc; gc.collect()   # the two 'x1' should not be collected
+            x1 = t.get()
+            assert x1 is not None
+            assert wr() is not None
+            assert wr() is x1
+            assert x1.lst == [40, 2]
+            x2 = t2.get()
             assert x2 is not None
-            assert wr() is not None
-            assert wr() is x2
-            return wr
+            assert wr2() is not None
+            assert wr2() is x2
+            assert x2.lst == [50, 3]
+            return wr, wr2
 
         def thread_entry_point():
-            wr = f()
+            wr, wr2 = f()
             wr_from_thread.wr = wr
+            wr_from_thread.wr2 = wr2
             wr_from_thread.seen = True
 
         def main():
             wr_from_thread.seen = False
             start_new_thread(thread_entry_point, ())
-            wr1 = f()
+            wr1, wr2 = f()
             count = 0
             while True:
                 time.sleep(0.5)
@@ -302,10 +318,15 @@
                     break
                 count += 1
             assert wr_from_thread.seen is True
-            wr2 = wr_from_thread.wr
-            import gc; gc.collect()      # wr2() should be collected here
+            wr_other_1 = wr_from_thread.wr
+            wr_other_2 = wr_from_thread.wr2
+            import gc; gc.collect()      # wr_other_*() should be collected here
             assert wr1() is not None     # this thread, still running
-            assert wr2() is None         # other thread, not running any more
+            assert wr2() is not None     # this thread, still running
+            assert wr_other_1() is None  # other thread, not running any more
+            assert wr_other_2() is None  # other thread, not running any more
+            assert wr1().lst == [40, 2]
+            assert wr2().lst == [50, 3]
             return 42
 
         extra_options = {'no__thread': no__thread, 'shared': True}

From pypy.commits at gmail.com  Wed Mar 22 14:30:11 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Mar 2017 11:30:11 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-native: dlclose must be called later,
 it worked for cpython because the process already holds a reference
 to libunwind and dlclose does not gargabe collect
Message-ID: <58d2c2b3.1a482e0a.b4a8.0afd@mx.google.com>

Author: Richard Plangger 
Branch: vmprof-native
Changeset: r90791:e72336569fc5
Date: 2017-03-22 15:29 -0300
http://bitbucket.org/pypy/pypy/changeset/e72336569fc5/

Log:	dlclose must be called later, it worked for cpython because the
	process already holds a reference to libunwind and dlclose does not
	gargabe collect

diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -451,6 +451,7 @@
 #endif
 
 static const char * vmprof_error = NULL;
+static void * libhandle = NULL;
 
 
 #ifdef VMPROF_LINUX
@@ -469,10 +470,7 @@
 #define UL_PREFIX ""
 #endif
 
-extern void * __vmprof_eval_vmprof_addr;
-
 int vmp_native_enable(void) {
-    void * libhandle;
     vmp_native_traces_enabled = 1;
 
     if (!unw_get_reg) {
@@ -500,9 +498,6 @@
         if (!(unw_getcontext = dlsym(libhandle, U_PREFIX PREFIX "_getcontext"))) {
             goto bail_out;
         }
-        if (dlclose(libhandle)) {
-            goto bail_out;
-        }
     }
 
 #if defined(__unix__)
@@ -518,6 +513,14 @@
 }
 
 void vmp_native_disable(void) {
+
+    if (libhandle != NULL) {
+        if (dlclose(libhandle)) {
+            vmprof_error = dlerror();
+            fprintf(stderr, "could close libunwind at runtime. error: %s\n", vmprof_error);
+        }
+    }
+
     vmp_native_traces_enabled = 0;
     if (vmp_ranges != NULL) {
         free(vmp_ranges);

From pypy.commits at gmail.com  Wed Mar 22 14:35:05 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 22 Mar 2017 11:35:05 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-native: correct error message
Message-ID: <58d2c3d9.02d5190a.8257b.4099@mx.google.com>

Author: Richard Plangger 
Branch: vmprof-native
Changeset: r90792:994394fc922f
Date: 2017-03-22 15:33 -0300
http://bitbucket.org/pypy/pypy/changeset/994394fc922f/

Log:	correct error message

diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -517,7 +517,7 @@
     if (libhandle != NULL) {
         if (dlclose(libhandle)) {
             vmprof_error = dlerror();
-            fprintf(stderr, "could close libunwind at runtime. error: %s\n", vmprof_error);
+            fprintf(stderr, "could not close libunwind at runtime. error: %s\n", vmprof_error);
         }
     }
 

From pypy.commits at gmail.com  Thu Mar 23 05:53:43 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 23 Mar 2017 02:53:43 -0700 (PDT)
Subject: [pypy-commit] pypy nogil-unsafe-2: add threadsan target and make
 tracebacks threadlocal
Message-ID: <58d39b27.0829190a.81b2e.5d50@mx.google.com>

Author: Remi Meier 
Branch: nogil-unsafe-2
Changeset: r90793:fb471557c36c
Date: 2017-03-23 10:53 +0100
http://bitbucket.org/pypy/pypy/changeset/fb471557c36c/

Log:	add threadsan target and make tracebacks threadlocal

	Add support for gcc's thread sanitizer by adding a makefile target.
	Avoid some data races by making the traceback buffer thread-local.

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -198,6 +198,9 @@
     BoolOption("lldebug0",
                "If true, makes an lldebug0 build", default=False,
                cmdline="--lldebug0"),
+    BoolOption("threadsan",
+               "If true, makes an thread-sanitizer build", default=False,
+               cmdline="--threadsan"),
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -397,6 +397,8 @@
             extra_opts += ["lldebug"]
         elif self.config.translation.lldebug0:
             extra_opts += ["lldebug0"]
+        elif self.config.translation.threadsan:
+            extra_opts += ["threadsan"]
         self.translator.platform.execute_makefile(self.targetdir,
                                                   extra_opts)
         if shared:
@@ -433,6 +435,9 @@
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(DEFAULT_TARGET)'),
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
             ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'),
+            ('threadsan', '',
+             '$(MAKE) CFLAGS="-g -O1 $(CFLAGS) -fno-omit-frame-pointer -fsanitize=thread" '
+             + 'LDFLAGS="-g $(LDFLAGS) -fsanitize=thread" $(DEFAULT_TARGET)'),
             ]
         if self.has_profopt():
             rules.append(
diff --git a/rpython/translator/c/src/debug_traceback.c b/rpython/translator/c/src/debug_traceback.c
--- a/rpython/translator/c/src/debug_traceback.c
+++ b/rpython/translator/c/src/debug_traceback.c
@@ -6,8 +6,8 @@
 #include 
 #include 
 
-int pypydtcount = 0;
-struct pypydtentry_s pypy_debug_tracebacks[PYPY_DEBUG_TRACEBACK_DEPTH];
+__thread int pypydtcount = 0;
+__thread struct pypydtentry_s pypy_debug_tracebacks[PYPY_DEBUG_TRACEBACK_DEPTH];
 
 void pypy_debug_traceback_print(void)
 {
diff --git a/rpython/translator/c/src/debug_traceback.h b/rpython/translator/c/src/debug_traceback.h
--- a/rpython/translator/c/src/debug_traceback.h
+++ b/rpython/translator/c/src/debug_traceback.h
@@ -60,8 +60,8 @@
   void *exctype;
 };
 
-RPY_EXTERN int pypydtcount;
-RPY_EXTERN struct pypydtentry_s pypy_debug_tracebacks[PYPY_DEBUG_TRACEBACK_DEPTH];
+RPY_EXTERN __thread int pypydtcount;
+RPY_EXTERN __thread struct pypydtentry_s pypy_debug_tracebacks[PYPY_DEBUG_TRACEBACK_DEPTH];
 
 RPY_EXTERN void pypy_debug_traceback_print(void);
 RPY_EXTERN void pypy_debug_catch_fatal_exception(void);
diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -76,17 +76,20 @@
  */
 
 #define _RPyGilAcquire() do {                                           \
-        assert((RPY_THREADLOCALREF_GET(synclock) & 0b001) == 0b0);      \
+        assert((__sync_fetch_and_add(                                   \
+                    &RPY_THREADLOCALREF_GET(synclock), 0)               \
+                & 0b001) == 0b0);                                       \
     if (!__sync_bool_compare_and_swap(                                  \
-                &RPY_THREADLOCALREF_GET(synclock), 0b100L, 0b101L))     \
+            &RPY_THREADLOCALREF_GET(synclock), 0b100L, 0b101L))         \
         RPyGilAcquireSlowPath();                                        \
-        } while (0)
+            } while (0)
 
-#define _RPyGilRelease() do {                                       \
-        assert((RPY_THREADLOCALREF_GET(synclock) & 0b101) == 0b101);  \
-    if (!__sync_bool_compare_and_swap(                              \
-                &RPY_THREADLOCALREF_GET(synclock), 0b101L, 0b100L)) \
-        RPyGilReleaseSlowPath();                                    \
+#define _RPyGilRelease() do {                                           \
+        assert((__sync_fetch_and_add(                                   \
+                    &RPY_THREADLOCALREF_GET(synclock), 0) & 0b101) == 0b101); \
+    if (!__sync_bool_compare_and_swap(                                  \
+            &RPY_THREADLOCALREF_GET(synclock), 0b101L, 0b100L))         \
+        RPyGilReleaseSlowPath();                                        \
         } while (0)
 
 static inline long *_RPyFetchFastGil(void) {
@@ -94,12 +97,12 @@
 //    return &rpy_fastgil;
 }
 
-#define RPyGilYieldThread() do { \
-    assert(RPY_THREADLOCALREF_GET(synclock) & 1L); \
-    if (RPY_THREADLOCALREF_GET(synclock) == 0b111L) { \
-        RPyGilYieldThreadSlowPath(); \
-    } \
-    } while (0)
+#define RPyGilYieldThread() do {                                        \
+        assert(__sync_fetch_and_add(&RPY_THREADLOCALREF_GET(synclock), 0) & 1L); \
+    if (RPY_THREADLOCALREF_GET(synclock) == 0b111L) {                   \
+        RPyGilYieldThreadSlowPath();                                    \
+            }                                                           \
+            } while (0)
 
 typedef unsigned char rpy_spinlock_t;
 static inline void rpy_spinlock_acquire(rpy_spinlock_t *p)
diff --git a/rpython/translator/c/src/thread_gil.c b/rpython/translator/c/src/thread_gil.c
--- a/rpython/translator/c/src/thread_gil.c
+++ b/rpython/translator/c/src/thread_gil.c
@@ -97,6 +97,7 @@
     RPyGilAcquire();
 }
 
+__attribute__((no_sanitize_thread))
 void RPyGilMasterRequestSafepoint(void)
 {
     pthread_mutex_lock(&sync_mutex);
@@ -111,10 +112,13 @@
         if (t == NULL)
             break;
 
-      retry:
-        switch (t->synclock) {
+      retry:;
+        /* this read and the setting of nursery_top make thread sanitizer
+         * unhappy */
+        long synclock = t->synclock;
+        switch (synclock) {
         default:
-            fprintf(stderr, "ERROR: found synclock=%ld\n", t->synclock);
+            fprintf(stderr, "ERROR: found synclock=%ld\n", synclock);
             abort();
         case 0b000L:
             /* new thread, no need to explicitly request safepoint */
diff --git a/rpython/translator/c/test/test_nogil.py b/rpython/translator/c/test/test_nogil.py
--- a/rpython/translator/c/test/test_nogil.py
+++ b/rpython/translator/c/test/test_nogil.py
@@ -22,6 +22,7 @@
         t.config.translation.gcrootfinder = self.gcrootfinder
         t.config.translation.thread = True
         t.config.translation.no__thread = no__thread
+        # t.config.translation.threadsan = True
         t.buildannotator().build_types(entry_point, [s_list_of_strings])
         t.buildrtyper().specialize()
         #
@@ -46,7 +47,7 @@
             pass
         state = State()
 
-        def thread():
+        def worker():
             rthread.gc_thread_start()
             x = None
             for i in range(100000000):
@@ -74,16 +75,16 @@
             state.counter = TS
 
             for _ in range(TS):
-                rthread.start_new_thread(thread, ())
+                rthread.start_new_thread(worker, ())
 
-            i = 0
             while True:
-                x = X(None, i)
                 time.sleep(0.1)
-                assert x.i == i
+                state.lock.acquire(True)
                 if state.counter == 0:
+                    state.lock.release()
                     break
-                i += 1
+                state.lock.release()
+
             os.write(1, "all threads done\n")
             return 0
 
@@ -152,8 +153,11 @@
 
             while True:
                 time.sleep(0.1)
+                state.lock.acquire(True)
                 if state.counter == 0:
+                    state.lock.release()
                     break
+                state.lock.release()
             os.write(1, "all threads done\n")
             return 0
 

From pypy.commits at gmail.com  Thu Mar 23 06:31:14 2017
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 23 Mar 2017 03:31:14 -0700 (PDT)
Subject: [pypy-commit] pypy nogil-unsafe-2: work around annoying GCC warning
Message-ID: <58d3a3f2.12532e0a.95bd.2647@mx.google.com>

Author: Remi Meier 
Branch: nogil-unsafe-2
Changeset: r90794:fdfdff0a82cc
Date: 2017-03-23 11:30 +0100
http://bitbucket.org/pypy/pypy/changeset/fdfdff0a82cc/

Log:	work around annoying GCC warning

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -814,7 +814,7 @@
                                        c_string_constant(op.args[1].value))
 
     def OP_DEBUG_ASSERT_NOT_NONE(self, op):
-        return 'RPyAssert(%s != NULL, "ll_assert_not_none() failed");' % (
+        return 'RPyAssert((void*)(%s) != NULL, "ll_assert_not_none() failed");' % (
                     self.expr(op.args[0]),)
 
     def OP_DEBUG_FATALERROR(self, op):

From pypy.commits at gmail.com  Thu Mar 23 07:13:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Mar 2017 04:13:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in timfel/pypy (pull request
 #529)
Message-ID: <58d3adc8.1a0e2e0a.b630.2bd2@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90796:5be35c6caec2
Date: 2017-03-23 11:12 +0000
http://bitbucket.org/pypy/pypy/changeset/5be35c6caec2/

Log:	Merged in timfel/pypy (pull request #529)

	Add an assertion, because the annotator doesn't seem to understand
	that index0 is >=0 here and we get an extra guard in the jit

diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py
--- a/rpython/rlib/rstrategies/rstrategies.py
+++ b/rpython/rlib/rstrategies/rstrategies.py
@@ -1,7 +1,7 @@
 
 import weakref, sys
 from rpython.rlib.rstrategies import logger
-from rpython.rlib import jit, objectmodel, rerased
+from rpython.rlib import jit, objectmodel, rerased, rarithmetic
 from rpython.rlib.objectmodel import specialize, not_rpython
 
 def make_accessors(strategy='strategy', storage='storage'):
@@ -443,6 +443,7 @@
 
     def store(self, w_self, index0, wrapped_value):
         self.check_index_store(w_self, index0)
+        assert index0 >= 0
         if self._check_can_handle(wrapped_value):
             unwrapped = self._unwrap(wrapped_value)
             self.get_storage(w_self)[index0] = unwrapped
@@ -451,6 +452,7 @@
 
     def fetch(self, w_self, index0):
         self.check_index_fetch(w_self, index0)
+        assert index0 >= 0
         unwrapped = self.get_storage(w_self)[index0]
         return self._wrap(unwrapped)
 
@@ -517,7 +519,7 @@
         self.check_index(w_self, start)
         self.check_index(w_self, end)
     def check_index(self, w_self, index0):
-        if index0 < 0 or index0 >= self.size(w_self):
+        if not rarithmetic.int_between(0, index0, self.size(w_self)):
             raise IndexError
 
 class UnsafeIndexingMixin(object):

From pypy.commits at gmail.com  Thu Mar 23 07:13:19 2017
From: pypy.commits at gmail.com (timfel)
Date: Thu, 23 Mar 2017 04:13:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: add an assertion,
 because the annotator doesn't seem to understand that index0 is >=0
 here and we get an extra guard in the jit
Message-ID: <58d3adcf.07452e0a.d92d.2b30@mx.google.com>

Author: Tim Felgentreff 
Branch: 
Changeset: r90795:f343b8876f3e
Date: 2017-03-22 14:39 +0100
http://bitbucket.org/pypy/pypy/changeset/f343b8876f3e/

Log:	add an assertion, because the annotator doesn't seem to understand
	that index0 is >=0 here and we get an extra guard in the jit

diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py
--- a/rpython/rlib/rstrategies/rstrategies.py
+++ b/rpython/rlib/rstrategies/rstrategies.py
@@ -1,7 +1,7 @@
 
 import weakref, sys
 from rpython.rlib.rstrategies import logger
-from rpython.rlib import jit, objectmodel, rerased
+from rpython.rlib import jit, objectmodel, rerased, rarithmetic
 from rpython.rlib.objectmodel import specialize, not_rpython
 
 def make_accessors(strategy='strategy', storage='storage'):
@@ -443,6 +443,7 @@
 
     def store(self, w_self, index0, wrapped_value):
         self.check_index_store(w_self, index0)
+        assert index0 >= 0
         if self._check_can_handle(wrapped_value):
             unwrapped = self._unwrap(wrapped_value)
             self.get_storage(w_self)[index0] = unwrapped
@@ -451,6 +452,7 @@
 
     def fetch(self, w_self, index0):
         self.check_index_fetch(w_self, index0)
+        assert index0 >= 0
         unwrapped = self.get_storage(w_self)[index0]
         return self._wrap(unwrapped)
 
@@ -517,7 +519,7 @@
         self.check_index(w_self, start)
         self.check_index(w_self, end)
     def check_index(self, w_self, index0):
-        if index0 < 0 or index0 >= self.size(w_self):
+        if not rarithmetic.int_between(0, index0, self.size(w_self)):
             raise IndexError
 
 class UnsafeIndexingMixin(object):

From pypy.commits at gmail.com  Thu Mar 23 07:19:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Mar 2017 04:19:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Aaaaaaaaah this skip would skip *the
 whole file*!!
Message-ID: <58d3af5f.09132e0a.b0d9.2a55@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90797:011e84c44f76
Date: 2017-03-23 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/011e84c44f76/

Log:	Aaaaaaaaah this skip would skip *the whole file*!!

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4637,8 +4637,8 @@
 
         self.meta_interp(f, [10])
 
-    @py.test.skip("loops!")
     def test_finalizer_bug(self):
+        py.test.skip("loops!")
         from rpython.rlib import rgc
         driver = JitDriver(greens=[], reds=[])
         class Fin(object):

From pypy.commits at gmail.com  Thu Mar 23 12:51:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Mar 2017 09:51:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Disable the insert key (and equivalents)
 in PyPy 3 (possibly until someone complains)
Message-ID: <58d3fd22.87212e0a.a7a4.0d7e@mx.google.com>

Author: Armin Rigo 
Branch: py3.5
Changeset: r90798:63f85f548a14
Date: 2017-03-23 17:51 +0100
http://bitbucket.org/pypy/pypy/changeset/63f85f548a14/

Log:	Disable the insert key (and equivalents) in PyPy 3 (possibly until
	someone complains)

diff --git a/lib_pypy/pyrepl/commands.py b/lib_pypy/pyrepl/commands.py
--- a/lib_pypy/pyrepl/commands.py
+++ b/lib_pypy/pyrepl/commands.py
@@ -384,4 +384,7 @@
 class quoted_insert(Command):
     kills_digit_arg = 0
     def do(self):
-        self.reader.push_input_trans(QITrans())
+        # XXX in Python 3, processing insert/C-q/C-v keys crashes
+        # because of a mixture of str and bytes.  Disable these keys.
+        pass
+        #self.reader.push_input_trans(QITrans())

From pypy.commits at gmail.com  Thu Mar 23 14:50:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 23 Mar 2017 11:50:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2512: translate with
 x86-without-sse2
Message-ID: <58d4190f.09132e0a.e2e5b.138c@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90799:23fd2966aada
Date: 2017-03-23 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/23fd2966aada/

Log:	Issue #2512: translate with x86-without-sse2

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -60,6 +60,7 @@
 
 class VectorAssemblerMixin(object):
     _mixin_ = True
+    element_ones = []    # overridden in assembler.py
 
     def setup_once_vector(self):
         pass
@@ -342,6 +343,7 @@
         else:
             assert lhsloc is xmm0
             maskloc = X86_64_XMM_SCRATCH_REG
+            assert len(self.element_ones) > 0
             self.mc.MOVAPD(maskloc, heap(self.element_ones[get_scale(size)]))
             self.mc.PXOR(resloc, resloc)
             # note that resloc contains true false for each element by the last compare operation

From pypy.commits at gmail.com  Fri Mar 24 07:34:09 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 24 Mar 2017 04:34:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix one translation error on win32,
 leave a hint about the next one
Message-ID: <58d50431.84452e0a.9f580.1d9b@mx.google.com>

Author: mattip 
Branch: py3.5
Changeset: r90800:04c4a170cbee
Date: 2017-03-24 14:33 +0300
http://bitbucket.org/pypy/pypy/changeset/04c4a170cbee/

Log:	fix one translation error on win32, leave a hint about the next one

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -577,6 +577,8 @@
 
     def wrap_windowserror(space, e, w_filename=None):
         XXX    # WindowsError no longer exists in Py3.5
+               # instead, OSError has a kwarg winerror that overrides
+               # any errno supplied
         from rpython.rlib import rwin32
 
         winerror = e.winerror
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -855,9 +855,7 @@
         # started through main() instead of wmain()
         rwin32._wgetenv(u"")
         for key, value in rwin32._wenviron_items():
-            if isinstance(key, str):
-                key = key.upper()
-            space.setitem(w_env, space.newtext(key), space.newtext(value))
+            space.setitem(w_env, space.newunicode(key), space.newunicode(value))
 
     @unwrap_spec(name=unicode, value=unicode)
     def putenv(space, name, value):

From pypy.commits at gmail.com  Fri Mar 24 11:05:48 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 24 Mar 2017 08:05:48 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-injection: fix merge,
 fix deprecated wrap()
Message-ID: <58d535cc.1459190a.c07a8.265a@mx.google.com>

Author: Matti Picus 
Branch: cpyext-injection
Changeset: r90802:979ca8d739c9
Date: 2017-03-24 18:04 +0300
http://bitbucket.org/pypy/pypy/changeset/979ca8d739c9/

Log:	fix merge, fix deprecated wrap()

diff --git a/pypy/module/cpyext/injection/_test_module.py b/pypy/module/cpyext/injection/_test_module.py
--- a/pypy/module/cpyext/injection/_test_module.py
+++ b/pypy/module/cpyext/injection/_test_module.py
@@ -22,11 +22,11 @@
 def injected_getitem(space, w_self, index):
     if index > 0:
         intval = space.int_w(w_self)
-        return space.wrap(index * intval)
+        return space.newint(index * intval)
     else:
         org = space.fromcache(Original)
         return space.call_function(org.w_original_getitem, w_self,
-                                   space.wrap(index))
+                                   space.newint(index))
 
 class W_MyThing(W_IntObject):
     def getclass(self, space):
@@ -57,10 +57,10 @@
 def injected_make(space, arg):
     if arg == 15:
         org = space.fromcache(Original)
-        return space.call_function(org.w_original_make, space.wrap(arg))
+        return space.call_function(org.w_original_make, space.newint(arg))
     if arg == 25:
         org = space.fromcache(Original)
-        return space.wrap(W_MyThing(arg))
+        return W_MyThing(arg).spacebind(space)
     return space.w_Ellipsis
 
 
@@ -74,16 +74,16 @@
     org = space.fromcache(Original)
     org.w_original_getitem = dict_w['__getitem__']
     for key, value in injected_methods.items():
-        dict_w[key] = space.wrap(value)
+        dict_w[key] = value.spacebind(space)
 
 def inject_global(space, w_func, name):
     assert name == 'make'
     org = space.fromcache(Original)
     org.w_original_make = w_func
-    return space.wrap(interp_injected_make)
+    return interp_injected_make.spacebind(space)
 
 def inject_module(space, w_mod, name):
     assert name == 'injection'
     org = space.fromcache(Original)
-    w_type = space.getattr(w_mod, space.wrap('test_mytype'))
+    w_type = space.getattr(w_mod, space.newtext('test_mytype'))
     org.w_original_type = w_type
diff --git a/pypy/module/cpyext/injection/numpy.py b/pypy/module/cpyext/injection/numpy.py
--- a/pypy/module/cpyext/injection/numpy.py
+++ b/pypy/module/cpyext/injection/numpy.py
@@ -29,7 +29,7 @@
     def __init__(self, space):
         self.injected_methods_w = []
         for key, value in injected_methods.items():
-            self.injected_methods_w.append((key, space.wrap(value)))
+            self.injected_methods_w.append((key, value.spacebind(space)))
 
 class W_ArrayObject(W_Root):
     def getclass(self, space):
@@ -150,7 +150,7 @@
     w_type = space.appexec([w_mod], """(mod):
         return mod.typeinfo['DOUBLE'][-1]
     """)
-    w_array_type = space.getattr(w_mod, space.wrap('ndarray'))
+    w_array_type = space.getattr(w_mod, space.newtext('ndarray'))
     assert isinstance(w_array_type, W_TypeObject)
     assert isinstance(w_type, W_TypeObject)
     org.w_float64_type = w_type
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -506,7 +506,6 @@
         W_TypeObject.__init__(self, space, name,
             bases_w or [space.w_object], dict_w, force_new_layout=new_layout,
             is_heaptype=flag_heaptype)
-            self.layout.typedef = newtypedef
         self.flag_cpytype = True
         # if a sequence or a mapping, then set the flag to force it
         if pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_item:

From pypy.commits at gmail.com  Fri Mar 24 12:07:35 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 24 Mar 2017 09:07:35 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Create PyBuffer: a base class for all
 pypy3-style buffers
Message-ID: <58d54447.97cb190a.71b21.27b3@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90803:325cb226de2d
Date: 2017-03-24 16:06 +0000
http://bitbucket.org/pypy/pypy/changeset/325cb226de2d/

Log:	Create PyBuffer: a base class for all pypy3-style buffers

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/buffer.py
@@ -0,0 +1,4 @@
+from rpython.rlib.buffer import Buffer
+
+class PyBuffer(Buffer):
+    _immutable_ = True
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -2,12 +2,12 @@
 # A convenient read-write buffer.  Located here for want of a better place.
 #
 
-from rpython.rlib.buffer import Buffer
+from pypy.interpreter.buffer import PyBuffer
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
 
 
-class ByteBuffer(Buffer):
+class ByteBuffer(PyBuffer):
     _immutable_ = True
 
     def __init__(self, len):
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -5,13 +5,13 @@
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
 from pypy.module._cffi_backend import ctypestruct
 
-from rpython.rlib.buffer import Buffer
+from pypy.interpreter.buffer import PyBuffer
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 
-class LLBuffer(Buffer):
+class LLBuffer(PyBuffer):
     _immutable_ = True
 
     def __init__(self, raw_cdata, size):
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,5 +1,4 @@
 from rpython.rlib import jit, rgc
-from rpython.rlib.buffer import Buffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
 from rpython.rlib.unroll import unrolling_iterable
@@ -7,6 +6,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
+from pypy.interpreter.buffer import PyBuffer
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import (
@@ -848,7 +848,7 @@
     v.typecode = k
 unroll_typecodes = unrolling_iterable(types.keys())
 
-class ArrayBuffer(Buffer):
+class ArrayBuffer(PyBuffer):
     _immutable_ = True
 
     def __init__(self, array, readonly):
@@ -895,7 +895,7 @@
                 return rffi.charpsize2str(rffi.ptradd(data, start), size)
             finally:
                 self.array._charbuf_stop()
-        return Buffer.getslice(self, start, stop, step, size)
+        return PyBuffer.getslice(self, start, stop, step, size)
 
     def get_raw_address(self):
         return self.array._charbuf_start()
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef
-from rpython.rlib.buffer import Buffer
+from pypy.interpreter.buffer import PyBuffer
 from pypy.conftest import option
 
 class AppTestMemoryView:
@@ -292,7 +292,7 @@
         assert (m[0], m[1], m[2], m[3]) == expected
         a.free()
 
-class MockBuffer(Buffer):
+class MockBuffer(PyBuffer):
     def __init__(self, space, w_arr, w_dim, w_fmt, \
                  w_itemsize, w_strides, w_shape):
         self.space = space

From pypy.commits at gmail.com  Fri Mar 24 12:40:48 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 24 Mar 2017 09:40:48 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Replace Buffer with PyBuffer in more
 places
Message-ID: <58d54c10.1a4c2e0a.23ad1.2b00@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90804:409febd767af
Date: 2017-03-24 16:40 +0000
http://bitbucket.org/pypy/pypy/changeset/409febd767af/

Log:	Replace Buffer with PyBuffer in more places

diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, generic_new_descr, GetSetProperty)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from rpython.rlib.buffer import Buffer
+from pypy.interpreter.buffer import PyBuffer
 from rpython.rlib.rStringIO import RStringIO
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.objectmodel import import_from_mixin
@@ -12,7 +12,7 @@
 import sys
 
 
-class BytesIOBuffer(Buffer):
+class BytesIOBuffer(PyBuffer):
     _immutable_ = True
 
     def __init__(self, w_bytesio):
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,10 +1,11 @@
-from rpython.rlib.buffer import Buffer
 from rpython.rtyper.lltypesystem import rffi
 
+from pypy.interpreter.bufffer import PyBuffer
+
 # XXX not the most efficient implementation
 
 
-class RawFFIBuffer(Buffer):
+class RawFFIBuffer(PyBuffer):
     _immutable_ = True
 
     def __init__(self, datainstance):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -19,9 +19,9 @@
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import userslot
+from pypy.interpreter.buffer import PyBuffer
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
-from rpython.rlib.buffer import Buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize, not_rpython
 from rpython.tool.sourcetools import func_renamer
@@ -313,7 +313,7 @@
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.newint(res)
 
-class CPyBuffer(Buffer):
+class CPyBuffer(PyBuffer):
     # Similar to Py_buffer
     _immutable_ = True
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,8 +1,8 @@
+from pypy.interpreter.buffer import Buffer
 from pypy.interpreter.error import oefmt
 from rpython.rlib import jit, rgc
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.listsort import make_timsort_class
-from rpython.rlib.buffer import Buffer
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
@@ -21,7 +21,7 @@
 
 TimSort = make_timsort_class()
 class StrideSort(TimSort):
-    ''' 
+    '''
     argsort (return the indices to sort) a list of strides
     '''
     def __init__(self, rangelist, strides, order):
@@ -380,14 +380,14 @@
 
     def get_buffer(self, space, flags):
         errtype = space.w_ValueError # should be BufferError, numpy does this instead
-        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and
                 not self.flags & NPY.ARRAY_C_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not C-contiguous")
-        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and
                 not self.flags & NPY.ARRAY_F_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not Fortran contiguous")
         if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
-                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
                      self.flags & NPY.ARRAY_C_CONTIGUOUS)):
            raise oefmt(errtype, "ndarray is not contiguous")
         if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
@@ -527,7 +527,7 @@
         try:
             length = support.product_check(shape)
             self.size = ovfcheck(length * dtype.elsize)
-        except OverflowError: 
+        except OverflowError:
             raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big.")
         if storage == lltype.nullptr(RAW_STORAGE):
             if dtype.num == NPY.OBJECT:
@@ -702,7 +702,7 @@
         free_raw_storage(self.storage)
 
 
-class ArrayBuffer(Buffer):
+class ArrayBuffer(PyBuffer):
     _immutable_ = True
 
     def __init__(self, impl, readonly):
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,8 +2,8 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.interpreter.buffer import PyBuffer
 from rpython.rlib import rmmap, rarithmetic, objectmodel
-from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
 
@@ -311,7 +311,7 @@
         return OperationError(space.w_SystemError, space.newtext('%s' % e))
 
 
-class MMapBuffer(Buffer):
+class MMapBuffer(PyBuffer):
     _immutable_ = True
 
     def __init__(self, space, mmap, readonly):
@@ -331,7 +331,7 @@
         if step == 1:
             return self.mmap.getslice(start, size)
         else:
-            return Buffer.getslice(self, start, stop, step, size)
+            return PyBuffer.getslice(self, start, stop, step, size)
 
     def setitem(self, index, char):
         self.check_valid_writeable()
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -3,7 +3,6 @@
 import sys
 from rpython.rlib.objectmodel import (
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
-from rpython.rlib.buffer import Buffer
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rstring import StringBuilder, ByteListBuilder
 from rpython.rlib.debug import check_list_of_chars, check_nonneg
@@ -18,6 +17,7 @@
     getbytevalue, makebytesdata_w, newbytesdata_w)
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.buffer import PyBuffer
 from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
 from pypy.objspace.std.stringmethods import _descr_getslice_slowpath
@@ -1276,7 +1276,7 @@
         start += step
 
 
-class BytearrayBuffer(Buffer):
+class BytearrayBuffer(PyBuffer):
     _immutable_ = True
     readonly = False
 
@@ -1306,7 +1306,7 @@
             if start != 0 or stop != len(data):
                 data = data[start:stop]
             return "".join(data)
-        return Buffer.getslice(self, start, stop, step, size)
+        return PyBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -5,8 +5,6 @@
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.rstring import (
     find, rfind, count, endswith, replace, rsplit, split, startswith)
-from rpython.rlib.buffer import Buffer
-
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, unwrap_spec
 from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop

From pypy.commits at gmail.com  Sat Mar 25 11:50:29 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 25 Mar 2017 08:50:29 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: copy rpython.rlib.buffer to
 pypy.interpreter.buffer
Message-ID: <58d691c5.0fd2190a.ac863.1a47@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90805:3d091950a819
Date: 2017-03-24 17:24 +0000
http://bitbucket.org/pypy/pypy/changeset/3d091950a819/

Log:	copy rpython.rlib.buffer to pypy.interpreter.buffer

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,7 +3,6 @@
 from rpython.rlib.cache import Cache
 from rpython.tool.uid import HUGEVAL_BYTES
 from rpython.rlib import jit, types
-from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.objectmodel import (we_are_translated, newlist_hint,
      compute_unique_id, specialize, not_rpython)
@@ -11,6 +10,7 @@
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
 
+from pypy.interpreter.buffer import StringBuffer
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     make_finalizer_queue)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -1,4 +1,170 @@
-from rpython.rlib.buffer import Buffer
 
-class PyBuffer(Buffer):
+class Buffer(object):
+    """Abstract base class for buffers."""
+    _attrs_ = ['readonly']
     _immutable_ = True
+
+    def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
+        raise NotImplementedError
+
+    def __len__(self):
+        res = self.getlength()
+        assert res >= 0
+        return res
+
+    def as_str(self):
+        "Returns an interp-level string with the whole content of the buffer."
+        # May be overridden.
+        return self.getslice(0, self.getlength(), 1, self.getlength())
+
+    def as_str_and_offset_maybe(self):
+        """
+        If the buffer is backed by a string, return a pair (string, offset),
+        where offset is the offset inside the string where the buffer start.
+        Else, return (None, 0).
+        """
+        return None, 0
+
+    def getitem(self, index):
+        "Returns the index'th character in the buffer."
+        raise NotImplementedError   # Must be overriden.  No bounds checks.
+
+    def __getitem__(self, i):
+        return self.getitem(i)
+
+    def getslice(self, start, stop, step, size):
+        # May be overridden.  No bounds checks.
+        return ''.join([self.getitem(i) for i in range(start, stop, step)])
+
+    def __getslice__(self, start, stop):
+        return self.getslice(start, stop, 1, stop - start)
+
+    def setitem(self, index, char):
+        "Write a character into the buffer."
+        raise NotImplementedError   # Must be overriden.  No bounds checks.
+
+    def __setitem__(self, i, char):
+        return self.setitem(i, char)
+
+    def setslice(self, start, string):
+        # May be overridden.  No bounds checks.
+        for i in range(len(string)):
+            self.setitem(start + i, string[i])
+
+    def get_raw_address(self):
+        raise ValueError("no raw buffer")
+
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
+
+    def releasebuffer(self):
+        pass
+
+class StringBuffer(Buffer):
+    _attrs_ = ['readonly', 'value']
+    _immutable_ = True
+
+    def __init__(self, value):
+        self.value = value
+        self.readonly = 1
+
+    def getlength(self):
+        return len(self.value)
+
+    def as_str(self):
+        return self.value
+
+    def as_str_and_offset_maybe(self):
+        return self.value, 0
+
+    def getitem(self, index):
+        return self.value[index]
+
+    def getslice(self, start, stop, step, size):
+        if size == 0:
+            return ""
+        if step == 1:
+            assert 0 <= start <= stop
+            if start == 0 and stop == len(self.value):
+                return self.value
+            return self.value[start:stop]
+        return Buffer.getslice(self, start, stop, step, size)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        # may still raise ValueError on some GCs
+        return rffi.get_raw_address_of_string(self.value)
+
+class SubBuffer(Buffer):
+    _attrs_ = ['buffer', 'offset', 'size', 'readonly']
+    _immutable_ = True
+
+    def __init__(self, buffer, offset, size):
+        self.readonly = buffer.readonly
+        if isinstance(buffer, SubBuffer):     # don't nest them
+            # we want a view (offset, size) over a view
+            # (buffer.offset, buffer.size) over buffer.buffer.
+            # Note that either '.size' can be -1 to mean 'up to the end'.
+            at_most = buffer.getlength() - offset
+            if size > at_most or size < 0:
+                if at_most < 0:
+                    at_most = 0
+                size = at_most
+            offset += buffer.offset
+            buffer = buffer.buffer
+        #
+        self.buffer = buffer
+        self.offset = offset
+        self.size = size
+
+    def getlength(self):
+        at_most = self.buffer.getlength() - self.offset
+        if 0 <= self.size <= at_most:
+            return self.size
+        elif at_most >= 0:
+            return at_most
+        else:
+            return 0
+
+    def as_str_and_offset_maybe(self):
+        string, offset = self.buffer.as_str_and_offset_maybe()
+        if string is not None:
+            return string, offset + self.offset
+        return None, 0
+
+    def getitem(self, index):
+        return self.buffer.getitem(self.offset + index)
+
+    def getslice(self, start, stop, step, size):
+        if start == stop:
+            return ''     # otherwise, adding self.offset might make them
+                          # out of bounds
+        return self.buffer.getslice(self.offset + start, self.offset + stop,
+                                    step, size)
+
+    def setitem(self, index, char):
+        self.buffer.setitem(self.offset + index, char)
+
+    def setslice(self, start, string):
+        if len(string) == 0:
+            return        # otherwise, adding self.offset might make 'start'
+                          # out of bounds
+        self.buffer.setslice(self.offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        ptr = self.buffer.get_raw_address()
+        return rffi.ptradd(ptr, self.offset)
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -2,12 +2,12 @@
 # A convenient read-write buffer.  Located here for want of a better place.
 #
 
-from pypy.interpreter.buffer import PyBuffer
+from pypy.interpreter.buffer import Buffer
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
 
 
-class ByteBuffer(PyBuffer):
+class ByteBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, len):
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -5,13 +5,13 @@
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
 from pypy.module._cffi_backend import ctypestruct
 
-from pypy.interpreter.buffer import PyBuffer
+from pypy.interpreter.buffer import Buffer
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 
-class LLBuffer(PyBuffer):
+class LLBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, raw_cdata, size):
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rlib.buffer import Buffer, SubBuffer
+from pypy.interpreter.buffer import Buffer, SubBuffer
 from rpython.rlib.rgc import (
     nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
 from rpython.rlib.rstring import StringBuilder
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, generic_new_descr, GetSetProperty)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.buffer import PyBuffer
+from pypy.interpreter.buffer import Buffer
 from rpython.rlib.rStringIO import RStringIO
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.objectmodel import import_from_mixin
@@ -12,7 +12,7 @@
 import sys
 
 
-class BytesIOBuffer(PyBuffer):
+class BytesIOBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, w_bytesio):
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,11 +1,11 @@
 from rpython.rtyper.lltypesystem import rffi
 
-from pypy.interpreter.bufffer import PyBuffer
+from pypy.interpreter.bufffer import Buffer
 
 # XXX not the most efficient implementation
 
 
-class RawFFIBuffer(PyBuffer):
+class RawFFIBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, datainstance):
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -6,7 +6,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
-from pypy.interpreter.buffer import PyBuffer
+from pypy.interpreter.buffer import Buffer
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import (
@@ -848,7 +848,7 @@
     v.typecode = k
 unroll_typecodes = unrolling_iterable(types.keys())
 
-class ArrayBuffer(PyBuffer):
+class ArrayBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, array, readonly):
@@ -895,7 +895,7 @@
                 return rffi.charpsize2str(rffi.ptradd(data, start), size)
             finally:
                 self.array._charbuf_stop()
-        return PyBuffer.getslice(self, start, stop, step, size)
+        return Buffer.getslice(self, start, stop, step, size)
 
     def get_raw_address(self):
         return self.array._charbuf_start()
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -19,7 +19,7 @@
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import userslot
-from pypy.interpreter.buffer import PyBuffer
+from pypy.interpreter.buffer import Buffer
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
 from rpython.rlib.unroll import unrolling_iterable
@@ -313,7 +313,7 @@
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.newint(res)
 
-class CPyBuffer(PyBuffer):
+class CPyBuffer(Buffer):
     # Similar to Py_buffer
     _immutable_ = True
 
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -3,7 +3,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rlib.buffer import StringBuffer
+from pypy.interpreter.buffer import StringBuffer
 from pypy.module.cpyext.pyobject import from_ref
 from pypy.module.cpyext.memoryobject import PyMemoryViewObject
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -702,7 +702,7 @@
         free_raw_storage(self.storage)
 
 
-class ArrayBuffer(PyBuffer):
+class ArrayBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, impl, readonly):
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from rpython.rlib.buffer import SubBuffer
+from pypy.interpreter.buffer import SubBuffer
 from rpython.rlib.rstring import strip_spaces
 from rpython.rlib.rawstorage import RAW_STORAGE_PTR
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -91,7 +91,7 @@
             w_base = w_object
             if read_only:
                 w_base = None
-            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data,
                                 dtype, w_base=w_base, strides=strides,
                                 start=offset), read_only
         if w_data is None:
@@ -104,11 +104,11 @@
         #print 'create view from shape',shape,'dtype',dtype,'data',data
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
-                   "__array_interface__ strides not fully supported yet") 
+                   "__array_interface__ strides not fully supported yet")
         arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
-        
+
     except OperationError as e:
         if e.match(space, space.w_AttributeError):
             return None, False
@@ -120,7 +120,7 @@
         return descr
     msg = "invalid PEP 3118 format string: '%s'" % c_format
     space.warn(space.newtext(msg), space.w_RuntimeWarning)
-    return None 
+    return None
 
 def _array_from_buffer_3118(space, w_object, dtype):
     try:
@@ -139,12 +139,12 @@
             raise oefmt(space.w_NotImplementedError,
                 "creating an array from a memoryview while specifying dtype "
                 "not supported")
-        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))):
             msg = ("Item size computed from the PEP 3118 buffer format "
                   "string does not match the actual item size.")
             space.warn(space.newtext(msg), space.w_RuntimeWarning)
             return w_object
-        dtype = descr 
+        dtype = descr
     elif not dtype:
         dtype = descriptor.get_dtype_cache(space).w_stringdtype
         dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
@@ -181,7 +181,7 @@
             raise e
     writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
     w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
-               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
+               storage_bytes=buflen, dtype=dtype, w_base=w_object,
                writable=writable, strides=strides)
     if w_ret:
         return w_ret
@@ -212,7 +212,7 @@
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
         if w_array is None:
-            if (    not space.isinstance_w(w_object, space.w_bytes) and 
+            if (    not space.isinstance_w(w_object, space.w_bytes) and
                     not space.isinstance_w(w_object, space.w_unicode) and
                     not isinstance(w_object, W_GenericBox)):
                 # use buffer interface
@@ -551,7 +551,7 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.call_method(w_buffer, '__buffer__', 
+        w_buffer = space.call_method(w_buffer, '__buffer__',
                                     space.newint(space.BUF_FULL_RO))
         buf = _getbuffer(space, w_buffer)
 
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.interpreter.buffer import PyBuffer
+from pypy.interpreter.buffer import Buffer
 from rpython.rlib import rmmap, rarithmetic, objectmodel
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
@@ -311,7 +311,7 @@
         return OperationError(space.w_SystemError, space.newtext('%s' % e))
 
 
-class MMapBuffer(PyBuffer):
+class MMapBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, space, mmap, readonly):
@@ -331,7 +331,7 @@
         if step == 1:
             return self.mmap.getslice(start, size)
         else:
-            return PyBuffer.getslice(self, start, stop, step, size)
+            return Buffer.getslice(self, start, stop, step, size)
 
     def setitem(self, index, char):
         self.check_valid_writeable()
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -1,9 +1,9 @@
 from rpython.rlib import jit
-from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstruct.error import StructError, StructOverflowError
 from rpython.rlib.rstruct.formatiterator import CalcSizeFormatIterator
 
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SubBuffer
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -1,19 +1,20 @@
 from rpython.annotator.model import SomeInstance, s_None
-from pypy.interpreter import argument, gateway
-from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.objspace.std.sliceobject import W_SliceObject
-from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.objectmodel import (instantiate, we_are_translated, specialize,
     not_rpython)
 from rpython.rlib.nonconst import NonConstant
 from rpython.rlib.rarithmetic import r_uint, r_singlefloat
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.lltypesystem import lltype
-from pypy.tool.option import make_config
 from rpython.tool.sourcetools import compile2, func_with_new_name
 from rpython.translator.translator import TranslationContext
 
+from pypy.tool.option import make_config
+from pypy.interpreter import argument, gateway
+from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache
+from pypy.interpreter.buffer import StringBuffer
+from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.objspace.std.sliceobject import W_SliceObject
+
 
 class W_MyObject(W_Root):
     typedef = None
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -17,7 +17,7 @@
     getbytevalue, makebytesdata_w, newbytesdata_w)
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.buffer import PyBuffer
+from pypy.interpreter.buffer import Buffer
 from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
 from pypy.objspace.std.stringmethods import _descr_getslice_slowpath
@@ -1276,7 +1276,7 @@
         start += step
 
 
-class BytearrayBuffer(PyBuffer):
+class BytearrayBuffer(Buffer):
     _immutable_ = True
     readonly = False
 
@@ -1306,7 +1306,7 @@
             if start != 0 or stop != len(data):
                 data = data[start:stop]
             return "".join(data)
-        return PyBuffer.getslice(self, start, stop, step, size)
+        return Buffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -4,10 +4,10 @@
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin, newlist_hint,
     resizelist_hint)
-from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder
 
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import StringBuffer
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import (
     WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -3,10 +3,10 @@
 """
 import operator
 
-from rpython.rlib.buffer import Buffer, SubBuffer, StringBuffer
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib.rstruct.error import StructError
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import Buffer, SubBuffer, StringBuffer
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.buffer import PyBuffer
+from pypy.interpreter.buffer import Buffer
 from pypy.conftest import option
 
 class AppTestMemoryView:
@@ -292,7 +292,7 @@
         assert (m[0], m[1], m[2], m[3]) == expected
         a.free()
 
-class MockBuffer(PyBuffer):
+class MockBuffer(Buffer):
     def __init__(self, space, w_arr, w_dim, w_fmt, \
                  w_itemsize, w_strides, w_shape):
         self.space = space
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -3,7 +3,6 @@
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin,
     enforceargs)
-from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
@@ -13,6 +12,7 @@
 
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import StringBuffer
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef

From pypy.commits at gmail.com  Sat Mar 25 11:50:32 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 25 Mar 2017 08:50:32 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Move ByteBuffer to
 pypy.interpreter.buffer
Message-ID: <58d691c8.1a142e0a.479c5.5183@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90806:3c2915c596d5
Date: 2017-03-25 15:49 +0000
http://bitbucket.org/pypy/pypy/changeset/3c2915c596d5/

Log:	Move ByteBuffer to pypy.interpreter.buffer

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -1,3 +1,4 @@
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
 
 class Buffer(object):
     """Abstract base class for buffers."""
@@ -73,6 +74,25 @@
     def releasebuffer(self):
         pass
 
+class ByteBuffer(Buffer):
+    _immutable_ = True
+
+    def __init__(self, len):
+        self.data = ['\x00'] * len
+        self.readonly = False
+
+    def getlength(self):
+        return len(self.data)
+
+    def getitem(self, index):
+        return self.data[index]
+
+    def setitem(self, index, char):
+        self.data[index] = char
+
+    def get_raw_address(self):
+        return nonmoving_raw_ptr_for_resizable_list(self.data)
+
 class StringBuffer(Buffer):
     _attrs_ = ['readonly', 'value']
     _immutable_ = True
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -2,29 +2,8 @@
 # A convenient read-write buffer.  Located here for want of a better place.
 #
 
-from pypy.interpreter.buffer import Buffer
+from pypy.interpreter.buffer import ByteBuffer
 from pypy.interpreter.gateway import unwrap_spec
-from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
-
-
-class ByteBuffer(Buffer):
-    _immutable_ = True
-
-    def __init__(self, len):
-        self.data = ['\x00'] * len
-        self.readonly = False
-
-    def getlength(self):
-        return len(self.data)
-
-    def getitem(self, index):
-        return self.data[index]
-
-    def setitem(self, index, char):
-        self.data[index] = char
-
-    def get_raw_address(self):
-        return nonmoving_raw_ptr_for_resizable_list(self.data)
 
 @unwrap_spec(length=int)
 def bytebuffer(space, length):

From pypy.commits at gmail.com  Sat Mar 25 12:13:21 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 25 Mar 2017 09:13:21 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: fix typo
Message-ID: <58d69721.02502e0a.db2d6.5221@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90807:c93aaa884ed3
Date: 2017-03-25 16:12 +0000
http://bitbucket.org/pypy/pypy/changeset/c93aaa884ed3/

Log:	fix typo

diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,6 +1,6 @@
 from rpython.rtyper.lltypesystem import rffi
 
-from pypy.interpreter.bufffer import Buffer
+from pypy.interpreter.buffer import Buffer
 
 # XXX not the most efficient implementation
 

From pypy.commits at gmail.com  Sat Mar 25 12:45:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 25 Mar 2017 09:45:51 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Fix dimensionality check in
 memoryview.cast()
Message-ID: <58d69ebf.0b572e0a.f016e.54b1@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90808:ea010dd6def5
Date: 2017-03-25 16:45 +0000
http://bitbucket.org/pypy/pypy/changeset/ea010dd6def5/

Log:	Fix dimensionality check in memoryview.cast()

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -552,13 +552,12 @@
                 raise oefmt(space.w_ValueError, \
                         "memoryview: number of dimensions must not exceed %d",
                         ndim)
-            # yes access ndim as field
-            if self.ndim > 1 and buf.getndim() != 1:
+            if ndim > 1 and buf.getndim() != 1:
                 raise oefmt(space.w_TypeError,
                             "memoryview: cast must be 1D -> ND or ND -> 1D")
 
+        origfmt = self.getformat()
         mv = W_MemoryView(buf, self.format, self.itemsize)
-        origfmt = mv.getformat()
         mv._cast_to_1D(space, origfmt, fmt, itemsize)
         if w_shape:
             fview = space.fixedview(w_shape)

From pypy.commits at gmail.com  Sat Mar 25 15:58:43 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 Mar 2017 12:58:43 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: document that by default wheel
 needs to be updated
Message-ID: <58d6cbf3.cf4d2e0a.be6b5.5909@mx.google.com>

Author: Matti Picus 
Branch: extradoc
Changeset: r875:3021c4b0dfb9
Date: 2017-03-25 22:58 +0300
http://bitbucket.org/pypy/pypy.org/changeset/3021c4b0dfb9/

Log:	document that by default wheel needs to be updated

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -234,6 +234,7 @@
 For example, without using a virtualenv:
      
 
       $ ./pypy-xxx/bin/pypy -m ensurepip
+$ ./pypy-xxx/bin/pypy -mpip install -U wheel # to update wheel
 $ ./pypy-xxx/bin/pip install cython numpy
 
      
 
      (See the general installation documentation for more.)
      
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -249,6 +249,7 @@
 For example, without using a virtualenv::
 
     $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pypy -mpip install -U wheel # to update wheel
     $ ./pypy-xxx/bin/pip install cython numpy
 
 (See the general `installation documentation`_ for more.)

From pypy.commits at gmail.com  Sat Mar 25 16:13:35 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 25 Mar 2017 13:13:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: document the need to upgrade wheel
Message-ID: <58d6cf6f.0da5190a.ccc9.5a2b@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90809:9d2b5856f4c3
Date: 2017-03-25 23:12 +0300
http://bitbucket.org/pypy/pypy/changeset/9d2b5856f4c3/

Log:	document the need to upgrade wheel

diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -57,6 +57,7 @@
 .. code-block:: console
 
     $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install -U pip wheel # to upgrade to the latest versions
     $ ./pypy-xxx/bin/pip install pygments  # for example
 
 Third party libraries will be installed in ``pypy-xxx/site-packages``, and
@@ -77,7 +78,17 @@
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
 
-Note that bin/python is now a symlink to bin/pypy.
+	# in any case activate it
+	$ source my-pypy-env/bin/activate
+
+Note that my-pypy-env/bin/python is now a symlink to my-pypy-env/bin/pypy
+so you should be able to run pypy simply by typing::
+
+    $ python
+
+You should still upgrade pip and wheel to the latest versions via::
+
+    $ my-pypy-env/bin/pip install -U pip wheel
 
 .. _pip: http://pypi.python.org/pypi/pip
 .. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html

From pypy.commits at gmail.com  Sat Mar 25 22:24:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 25 Mar 2017 19:24:25 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: begin refactoring mv._cast_to_1D()
Message-ID: <58d72659.1a482e0a.c00e0.6361@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90810:eab83a20276c
Date: 2017-03-25 17:07 +0000
http://bitbucket.org/pypy/pypy/changeset/eab83a20276c/

Log:	begin refactoring mv._cast_to_1D()

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -557,8 +557,7 @@
                             "memoryview: cast must be 1D -> ND or ND -> 1D")
 
         origfmt = self.getformat()
-        mv = W_MemoryView(buf, self.format, self.itemsize)
-        mv._cast_to_1D(space, origfmt, fmt, itemsize)
+        mv = self._cast_to_1D(space, buf, origfmt, fmt, itemsize)
         if w_shape:
             fview = space.fixedview(w_shape)
             shape = [space.int_w(w_obj) for w_obj in fview]
@@ -595,8 +594,7 @@
 
         self.flags = flags
 
-    def _cast_to_1D(self, space, origfmt, fmt, itemsize):
-        buf = self.buf
+    def _cast_to_1D(self, space, buf, origfmt, fmt, itemsize):
         if itemsize < 0:
             raise oefmt(space.w_ValueError, "memoryview: destination" \
                     " format must be a native single character format prefixed" \
@@ -616,14 +614,14 @@
         if not newfmt:
             raise oefmt(space.w_RuntimeError,
                     "memoryview: internal error")
-        self.format = newfmt
-        self.itemsize = itemsize
-        self.ndim = 1
-        self.shape = [buf.getlength() // itemsize]
-        self.strides = [itemsize]
+        mv = W_MemoryView(buf, newfmt, itemsize)
+        mv.ndim = 1
+        mv.shape = [buf.getlength() // itemsize]
+        mv.strides = [itemsize]
         # XX suboffsets
 
-        self._init_flags()
+        mv._init_flags()
+        return mv
 
     def get_native_fmtstr(self, fmt):
         lenfmt = len(fmt)
@@ -645,8 +643,10 @@
                  'Q','n','N','f','d','?','P']
         for c in unrolling_iterable(chars):
             if c == format:
-                if nat: return '@'+c
-                else: return c
+                if nat:
+                    return '@'+c
+                else:
+                    return c
 
         return None
 

From pypy.commits at gmail.com  Sat Mar 25 22:24:27 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 25 Mar 2017 19:24:27 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Add a base class for C-style buffers:
 BinaryBuffer
Message-ID: <58d7265b.58092e0a.bd1d1.61c5@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90811:08a3f30f31dc
Date: 2017-03-26 03:23 +0100
http://bitbucket.org/pypy/pypy/changeset/08a3f30f31dc/

Log:	Add a base class for C-style buffers: BinaryBuffer

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -1,4 +1,6 @@
 from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
+from rpython.rlib.signature import signature
+from rpython.rlib import types
 
 class Buffer(object):
     """Abstract base class for buffers."""
@@ -16,8 +18,7 @@
 
     def as_str(self):
         "Returns an interp-level string with the whole content of the buffer."
-        # May be overridden.
-        return self.getslice(0, self.getlength(), 1, self.getlength())
+        raise NotImplementedError
 
     def as_str_and_offset_maybe(self):
         """
@@ -57,6 +58,34 @@
         raise ValueError("no raw buffer")
 
     def getformat(self):
+        raise NotImplementedError
+
+    def getitemsize(self):
+        raise NotImplementedError
+
+    def getndim(self):
+        raise NotImplementedError
+
+    def getshape(self):
+        raise NotImplementedError
+
+    def getstrides(self):
+        raise NotImplementedError
+
+    def releasebuffer(self):
+        pass
+
+class BinaryBuffer(Buffer):
+    """Base class for buffers of bytes"""
+    _attrs_ = ['readonly']
+    _immutable_ = True
+
+    def as_str(self):
+        "Returns an interp-level string with the whole content of the buffer."
+        # May be overridden.
+        return self.getslice(0, self.getlength(), 1, self.getlength())
+
+    def getformat(self):
         return 'B'
 
     def getitemsize(self):
@@ -71,10 +100,9 @@
     def getstrides(self):
         return [1]
 
-    def releasebuffer(self):
-        pass
 
-class ByteBuffer(Buffer):
+
+class ByteBuffer(BinaryBuffer):
     _immutable_ = True
 
     def __init__(self, len):
@@ -93,7 +121,7 @@
     def get_raw_address(self):
         return nonmoving_raw_ptr_for_resizable_list(self.data)
 
-class StringBuffer(Buffer):
+class StringBuffer(BinaryBuffer):
     _attrs_ = ['readonly', 'value']
     _immutable_ = True
 
@@ -128,10 +156,11 @@
         # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
-class SubBuffer(Buffer):
+class SubBuffer(BinaryBuffer):
     _attrs_ = ['buffer', 'offset', 'size', 'readonly']
     _immutable_ = True
 
+    #@signature(types.any(), types.instance(BinaryBuffer), types.int(), types.int(), returns=types.none())
     def __init__(self, buffer, offset, size):
         self.readonly = buffer.readonly
         if isinstance(buffer, SubBuffer):     # don't nest them
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -5,13 +5,13 @@
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
 from pypy.module._cffi_backend import ctypestruct
 
-from pypy.interpreter.buffer import Buffer
+from pypy.interpreter.buffer import BinaryBuffer
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 
-class LLBuffer(Buffer):
+class LLBuffer(BinaryBuffer):
     _immutable_ = True
 
     def __init__(self, raw_cdata, size):
@@ -34,7 +34,7 @@
     def getslice(self, start, stop, step, size):
         if step == 1:
             return rffi.charpsize2str(rffi.ptradd(self.raw_cdata, start), size)
-        return Buffer.getslice(self, start, stop, step, size)
+        return BinaryBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         raw_cdata = rffi.ptradd(self.raw_cdata, start)
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.interpreter.buffer import Buffer, SubBuffer
+from pypy.interpreter.buffer import BinaryBuffer, SubBuffer
 from rpython.rlib.rgc import (
     nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
 from rpython.rlib.rstring import StringBuilder
@@ -155,7 +155,7 @@
     readinto1 = interp2app(W_BufferedIOBase.readinto1_w),
 )
 
-class RawBuffer(Buffer):
+class RawBuffer(BinaryBuffer):
     _immutable_ = True
 
     def __init__(self, n):
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, generic_new_descr, GetSetProperty)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.buffer import Buffer
+from pypy.interpreter.buffer import BinaryBuffer
 from rpython.rlib.rStringIO import RStringIO
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.objectmodel import import_from_mixin
@@ -12,7 +12,7 @@
 import sys
 
 
-class BytesIOBuffer(Buffer):
+class BytesIOBuffer(BinaryBuffer):
     _immutable_ = True
 
     def __init__(self, w_bytesio):
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,11 +1,11 @@
 from rpython.rtyper.lltypesystem import rffi
 
-from pypy.interpreter.buffer import Buffer
+from pypy.interpreter.buffer import BinaryBuffer
 
 # XXX not the most efficient implementation
 
 
-class RawFFIBuffer(Buffer):
+class RawFFIBuffer(BinaryBuffer):
     _immutable_ = True
 
     def __init__(self, datainstance):
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.interpreter.buffer import Buffer
+from pypy.interpreter.buffer import BinaryBuffer
 from rpython.rlib import rmmap, rarithmetic, objectmodel
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
@@ -311,7 +311,7 @@
         return OperationError(space.w_SystemError, space.newtext('%s' % e))
 
 
-class MMapBuffer(Buffer):
+class MMapBuffer(BinaryBuffer):
     _immutable_ = True
 
     def __init__(self, space, mmap, readonly):
@@ -331,7 +331,7 @@
         if step == 1:
             return self.mmap.getslice(start, size)
         else:
-            return Buffer.getslice(self, start, stop, step, size)
+            return BinaryBuffer.getslice(self, start, stop, step, size)
 
     def setitem(self, index, char):
         self.check_valid_writeable()
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -17,7 +17,7 @@
     getbytevalue, makebytesdata_w, newbytesdata_w)
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.buffer import Buffer
+from pypy.interpreter.buffer import BinaryBuffer
 from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
 from pypy.objspace.std.stringmethods import _descr_getslice_slowpath
@@ -1276,7 +1276,7 @@
         start += step
 
 
-class BytearrayBuffer(Buffer):
+class BytearrayBuffer(BinaryBuffer):
     _immutable_ = True
     readonly = False
 
@@ -1306,7 +1306,7 @@
             if start != 0 or stop != len(data):
                 data = data[start:stop]
             return "".join(data)
-        return Buffer.getslice(self, start, stop, step, size)
+        return BinaryBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.

From pypy.commits at gmail.com  Sun Mar 26 14:22:26 2017
From: pypy.commits at gmail.com (mjacob)
Date: Sun, 26 Mar 2017 11:22:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement main part of PEP 489
 (Multi-phase extension module initialization).
Message-ID: <58d806e2.0da5190a.ccc9.7f8b@mx.google.com>

Author: Manuel Jacob 
Branch: py3.5
Changeset: r90812:76f032290171
Date: 2017-03-26 20:22 +0200
http://bitbucket.org/pypy/pypy/changeset/76f032290171/

Log:	Implement main part of PEP 489 (Multi-phase extension module
	initialization).

	This also removes the cpyext.load_module() app-level function, which
	is more difficult to support with the new multi-phase initialization
	and equivalent in functionality to imp.load_dynamic(). I've started
	a thread on the mailing list whether this function is worth re-
	adding.

diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -4,7 +4,6 @@
 
 class Module(MixedModule):
     interpleveldefs = {
-        'load_module': 'api.load_extension_module',
     }
 
     appleveldefs = {
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -565,7 +565,7 @@
     'PyUnicode_FromFormat', 'PyUnicode_FromFormatV', 'PyUnicode_AsWideCharString',
     'PyUnicode_GetSize', 'PyUnicode_GetLength',
     'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant',
-    'PyModule_GetDef',
+    'PyModule_GetDef', 'PyModuleDef_Init',
     'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack',
     '_PyArg_Parse_SizeT', '_PyArg_ParseTuple_SizeT',
     '_PyArg_ParseTupleAndKeywords_SizeT', '_PyArg_VaParse_SizeT',
@@ -1489,8 +1489,7 @@
     copy_header_files(cts, trunk_include, use_micronumpy)
 
 
- at unwrap_spec(path='fsencode', name='text')
-def load_extension_module(space, path, name):
+def create_extension_module(space, w_spec):
     # note: this is used both to load CPython-API-style C extension
     # modules (cpyext) and to load CFFI-style extension modules
     # (_cffi_backend).  Any of the two can be disabled at translation
@@ -1498,6 +1497,9 @@
     # order of things here.
     from rpython.rlib import rdynload
 
+    name = space.text_w(space.getattr(w_spec, space.newtext("name")))
+    path = space.text_w(space.getattr(w_spec, space.newtext("origin")))
+
     if os.sep not in path:
         path = os.curdir + os.sep + path      # force a '/' in the path
     basename = name.split('.')[-1]
@@ -1535,7 +1537,7 @@
         except KeyError:
             pass
         else:
-            return load_cpyext_module(space, name, path, dll, initptr)
+            return create_cpyext_module(space, w_spec, name, path, dll, initptr)
         if look_for is not None:
             look_for += ' or ' + also_look_for
         else:
@@ -1549,8 +1551,9 @@
 
 initfunctype = lltype.Ptr(lltype.FuncType([], PyObject))
 
-def load_cpyext_module(space, name, path, dll, initptr):
+def create_cpyext_module(space, w_spec, name, path, dll, initptr):
     from rpython.rlib import rdynload
+    from pypy.module.cpyext.pyobject import get_w_obj_and_decref
 
     space.getbuiltinmodule("cpyext")    # mandatory to init cpyext
     state = space.fromcache(State)
@@ -1562,25 +1565,54 @@
     state.package_context = name, path
     try:
         initfunc = rffi.cast(initfunctype, initptr)
-        w_mod = generic_cpy_call(space, initfunc)
+        initret = generic_cpy_call_dont_convert_result(space, initfunc)
         state.check_and_raise_exception()
+        if not initret.c_ob_type:
+            raise oefmt(space.w_SystemError,
+                        "init function of %s returned uninitialized object",
+                        name)
+        # This should probably compare by identity with PyModuleDef_Type from
+        # modsupport.c, but I didn't find a way to do that.
+        tp_name_nonconst = rffi.cast(rffi.CCHARP, initret.c_ob_type.c_tp_name)
+        if rffi.charp2str(tp_name_nonconst) == "moduledef":
+            from pypy.module.cpyext.modsupport import \
+                    create_module_from_def_and_spec
+            return create_module_from_def_and_spec(space, initret, w_spec,
+                                                   name)
     finally:
         state.package_context = old_context
+    w_mod = get_w_obj_and_decref(space, initret)
     state.fixup_extension(w_mod, name, path)
     return w_mod
 
+def exec_extension_module(space, w_mod):
+    from pypy.module.cpyext.modsupport import exec_def
+    if not space.config.objspace.usemodules.cpyext:
+        return
+    if not isinstance(w_mod, Module):
+        return
+    space.getbuiltinmodule("cpyext")
+    mod_as_pyobj = rawrefcount.from_obj(PyObject, w_mod)
+    if mod_as_pyobj:
+        return exec_def(space, w_mod, mod_as_pyobj)
+
 @specialize.ll()
 def generic_cpy_call(space, func, *args):
     FT = lltype.typeOf(func).TO
-    return make_generic_cpy_call(FT, False)(space, func, *args)
+    return make_generic_cpy_call(FT, False, True)(space, func, *args)
 
 @specialize.ll()
 def generic_cpy_call_expect_null(space, func, *args):
     FT = lltype.typeOf(func).TO
-    return make_generic_cpy_call(FT, True)(space, func, *args)
+    return make_generic_cpy_call(FT, True, True)(space, func, *args)
+
+ at specialize.ll()
+def generic_cpy_call_dont_convert_result(space, func, *args):
+    FT = lltype.typeOf(func).TO
+    return make_generic_cpy_call(FT, False, False)(space, func, *args)
 
 @specialize.memo()
-def make_generic_cpy_call(FT, expect_null):
+def make_generic_cpy_call(FT, expect_null, convert_result):
     from pypy.module.cpyext.pyobject import make_ref, from_ref
     from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
     from pypy.module.cpyext.pyobject import get_w_obj_and_decref
@@ -1634,8 +1666,9 @@
             keepalive_until_here(*keepalives)
 
         if is_PyObject(RESULT_TYPE):
-            if not is_pyobj(result):
+            if not convert_result or not is_pyobj(result):
                 ret = result
+                has_result = bool(ret)
             else:
                 # The object reference returned from a C function
                 # that is called from Python must be an owned reference
@@ -1644,10 +1677,10 @@
                     ret = get_w_obj_and_decref(space, result)
                 else:
                     ret = None
+                has_result = ret is not None
 
             # Check for exception consistency
             has_error = PyErr_Occurred(space) is not None
-            has_result = ret is not None
             if has_error and has_result:
                 raise oefmt(space.w_SystemError,
                             "An exception was set, but function returned a "
diff --git a/pypy/module/cpyext/include/moduleobject.h b/pypy/module/cpyext/include/moduleobject.h
--- a/pypy/module/cpyext/include/moduleobject.h
+++ b/pypy/module/cpyext/include/moduleobject.h
@@ -8,6 +8,8 @@
 
 #include "cpyext_moduleobject.h"
 
+PyAPI_FUNC(PyObject *) PyModuleDef_Init(struct PyModuleDef*);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cts,
-    parse_dir, bootstrap_function)
+    parse_dir, bootstrap_function, generic_cpy_call)
 from pypy.module.cpyext.pyobject import PyObject, as_pyobj, make_typedescr
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
@@ -14,6 +14,7 @@
 cts.parse_header(parse_dir / 'cpyext_moduleobject.h')
 PyModuleDef = cts.gettype('PyModuleDef *')
 PyModuleObject = cts.gettype('PyModuleObject *')
+PyModuleDef_Slot = cts.gettype('PyModuleDef_Slot')
 
 @bootstrap_function
 def init_moduleobject(space):
@@ -64,6 +65,89 @@
     return w_mod
 
 
+createfunctype = lltype.Ptr(lltype.FuncType([PyObject, PyModuleDef], PyObject))
+execfunctype = lltype.Ptr(lltype.FuncType([PyObject], rffi.INT_real))
+
+
+def create_module_from_def_and_spec(space, moddef, w_spec, name):
+    moddef = rffi.cast(PyModuleDef, moddef)
+    if moddef.c_m_size < 0:
+        raise oefmt(space.w_SystemError,
+                    "module %s: m_size may not be negative for multi-phase "
+                    "initialization", name)
+    createf = lltype.nullptr(rffi.VOIDP.TO)
+    has_execution_slots = False
+    cur_slot = rffi.cast(rffi.CArrayPtr(PyModuleDef_Slot), moddef.c_m_slots)
+    if cur_slot:
+        while True:
+            slot = rffi.cast(lltype.Signed, cur_slot[0].c_slot)
+            if slot == 0:
+                break
+            elif slot == 1:
+                if createf:
+                    raise oefmt(space.w_SystemError,
+                                "module %s has multiple create slots", name)
+                createf = cur_slot[0].c_value
+            elif slot < 0 or slot > 2:
+                raise oefmt(space.w_SystemError,
+                            "module %s uses unknown slot ID %d", name, slot)
+            else:
+                has_execution_slots = True
+            cur_slot = rffi.ptradd(cur_slot, 1)
+    if createf:
+        createf = rffi.cast(createfunctype, createf)
+        w_mod = generic_cpy_call(space, createf, w_spec, moddef)
+    else:
+        w_mod = Module(space, space.newtext(name))
+    if isinstance(w_mod, Module):
+        mod = rffi.cast(PyModuleObject, as_pyobj(space, w_mod))
+        #mod.c_md_state = None
+        mod.c_md_def = moddef
+    else:
+        if moddef.c_m_size > 0 or moddef.c_m_traverse or moddef.c_m_clear or \
+           moddef.c_m_free:
+            raise oefmt(space.w_SystemError,
+                        "module %s is not a module object, but requests "
+                        "module state", name)
+        if has_execution_slots:
+            raise oefmt(space.w_SystemError,
+                        "module %s specifies execution slots, but did not "
+                        "create a ModuleType instance", name)
+    dict_w = {}
+    convert_method_defs(space, dict_w, moddef.c_m_methods, None, w_mod, name)
+    for key, w_value in dict_w.items():
+        space.setattr(w_mod, space.newtext(key), w_value)
+    if moddef.c_m_doc:
+        doc = rffi.charp2str(rffi.cast(rffi.CCHARP, moddef.c_m_doc))
+        space.setattr(w_mod, space.newtext('__doc__'), space.newtext(doc))
+    return w_mod
+
+
+def exec_def(space, w_mod, mod_as_pyobj):
+    from pypy.module.cpyext.pyerrors import PyErr_Occurred
+    mod = rffi.cast(PyModuleObject, mod_as_pyobj)
+    moddef = mod.c_md_def
+    cur_slot = rffi.cast(rffi.CArrayPtr(PyModuleDef_Slot), moddef.c_m_slots)
+    while cur_slot and rffi.cast(lltype.Signed, cur_slot[0].c_slot):
+        if rffi.cast(lltype.Signed, cur_slot[0].c_slot) == 2:
+            execf = rffi.cast(execfunctype, cur_slot[0].c_value)
+            res = generic_cpy_call(space, execf, w_mod)
+            has_error = PyErr_Occurred(space) is not None
+            if rffi.cast(lltype.Signed, res):
+                if has_error:
+                    state = space.fromcache(State)
+                    state.check_and_raise_exception()
+                else:
+                    raise oefmt(space.w_SystemError,
+                                "execution of module %S failed without "
+                                "setting an exception", w_mod.w_name)
+            if has_error:
+                raise oefmt(space.w_SystemError,
+                            "execution of module %S raised unreported "
+                            "exception", w_mod.w_name)
+        cur_slot = rffi.ptradd(cur_slot, 1)
+
+
 def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None):
     w_name = space.newtext_or_none(name)
     methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods)
diff --git a/pypy/module/cpyext/src/modsupport.c b/pypy/module/cpyext/src/modsupport.c
--- a/pypy/module/cpyext/src/modsupport.c
+++ b/pypy/module/cpyext/src/modsupport.c
@@ -602,3 +602,26 @@
     }
     return ((PyModuleObject *)m)->md_def;
 }
+
+PyTypeObject PyModuleDef_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    "moduledef",                                /* tp_name */
+    sizeof(struct PyModuleDef),                 /* tp_size */
+    0,                                          /* tp_itemsize */
+};
+
+static Py_ssize_t max_module_number;
+
+PyObject*
+PyModuleDef_Init(struct PyModuleDef* def)
+{
+    if (PyType_Ready(&PyModuleDef_Type) < 0)
+         return NULL;
+    if (def->m_base.m_index == 0) {
+        max_module_number++;
+        Py_REFCNT(def) = 1;
+        Py_TYPE(def) = &PyModuleDef_Type;
+        def->m_base.m_index = max_module_number;
+    }
+    return (PyObject*)def;
+}
diff --git a/pypy/module/cpyext/test/multiphase.c b/pypy/module/cpyext/test/multiphase.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/multiphase.c
@@ -0,0 +1,28 @@
+#include "Python.h"
+
+static struct PyModuleDef multiphase_def;
+
+static PyObject *check_getdef_same(PyObject *self, PyObject *args) {
+    return PyBool_FromLong(PyModule_GetDef(self) == &multiphase_def);
+}
+
+static PyMethodDef methods[] = {
+    {"check_getdef_same", check_getdef_same, METH_NOARGS},
+    {NULL}
+};
+
+static PyModuleDef multiphase_def = {
+    PyModuleDef_HEAD_INIT, /* m_base */
+    "multiphase",          /* m_name */
+    "example docstring",   /* m_doc */
+    0,                     /* m_size */
+    methods,               /* m_methods */
+    NULL,                  /* m_slots */
+    NULL,                  /* m_traverse */
+    NULL,                  /* m_clear */
+    NULL,                  /* m_free */
+};
+
+PyMODINIT_FUNC PyInit_multiphase(void) {
+    return PyModuleDef_Init(&multiphase_def);
+}
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -40,9 +40,11 @@
 
     def load_module(self, mod, name):
         space = self.space
-        api.load_extension_module(space, mod, name)
-        return space.getitem(
-            space.sys.get('modules'), space.wrap(name))
+        w_path = space.newtext(mod)
+        w_name = space.newtext(name)
+        return space.appexec([w_name, w_path], '''(name, path):
+            import imp
+            return imp.load_dynamic(name, path)''')
 
 
 def get_cpyext_info(space):
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -25,12 +25,85 @@
         module = self.import_extension('foo', [
             ("check_getdef_same", "METH_NOARGS",
              """
-                 return PyBool_FromLong(PyModule_GetDef(mod_global) == &moduledef);
+                 return PyBool_FromLong(PyModule_GetDef(self) == &moduledef);
              """
             )], prologue="""
             static struct PyModuleDef moduledef;
-            static PyObject *mod_global;
-            """, more_init="""
-               mod_global = mod;
             """)
         assert module.check_getdef_same()
+
+
+class AppTestMultiPhase(AppTestCpythonExtensionBase):
+    def test_basic(self):
+        from types import ModuleType
+        module = self.import_module(name='multiphase')
+        assert isinstance(module, ModuleType)
+        assert module.__name__ == 'multiphase'
+        assert module.__doc__ == "example docstring"
+
+    def test_getdef(self):
+        from types import ModuleType
+        module = self.import_module(name='multiphase')
+        assert module.check_getdef_same()
+
+    def test_slots(self):
+        from types import ModuleType
+        body = """
+        static PyModuleDef multiphase_def;
+
+        static PyObject* multiphase_create(PyObject *spec, PyModuleDef *def) {
+            PyObject *module = PyModule_New("altname");
+            PyObject_SetAttrString(module, "create_spec", spec);
+            PyObject_SetAttrString(module, "create_def_eq",
+                                   PyBool_FromLong(def == &multiphase_def));
+            return module;
+        }
+
+        static int multiphase_exec(PyObject* module) {
+            Py_INCREF(Py_True);
+            PyObject_SetAttrString(module, "exec_called", Py_True);
+            return 0;
+        }
+
+        static PyModuleDef_Slot multiphase_slots[] = {
+            {Py_mod_create, multiphase_create},
+            {Py_mod_exec, multiphase_exec},
+            {0, NULL}
+        };
+
+        static PyModuleDef multiphase_def = {
+            PyModuleDef_HEAD_INIT,                      /* m_base */
+            "multiphase",                               /* m_name */
+            "example docstring",                        /* m_doc */
+            0,                                          /* m_size */
+            NULL,                                       /* m_methods */
+            multiphase_slots,                           /* m_slots */
+            NULL,                                       /* m_traverse */
+            NULL,                                       /* m_clear */
+            NULL,                                       /* m_free */
+        };
+        """
+        init = """
+        return PyModuleDef_Init(&multiphase_def);
+        """
+        module = self.import_module(name='multiphase', body=body, init=init)
+        assert module.create_spec
+        assert module.create_spec is module.__spec__
+        assert module.create_def_eq
+        assert module.exec_called
+
+    def test_forget_init(self):
+        from types import ModuleType
+        body = """
+        static PyModuleDef multiphase_def = {
+            PyModuleDef_HEAD_INIT,                      /* m_base */
+            "multiphase",                               /* m_name */
+            "example docstring",                        /* m_doc */
+            0,                                          /* m_size */
+        };
+        """
+        init = """
+        return (PyObject *) &multiphase_def;
+        """
+        raises(SystemError, self.import_module, name='multiphase', body=body,
+               init=init)
diff --git a/pypy/module/imp/__init__.py b/pypy/module/imp/__init__.py
--- a/pypy/module/imp/__init__.py
+++ b/pypy/module/imp/__init__.py
@@ -16,8 +16,8 @@
         'init_frozen':     'interp_imp.init_frozen',
         'is_builtin':      'interp_imp.is_builtin',
         'is_frozen':       'interp_imp.is_frozen',
+        'exec_dynamic':    'interp_imp.exec_dynamic',
         'exec_builtin':    'interp_imp.exec_builtin',
-        'exec_dynamic':    'interp_imp.exec_builtin',
         'get_frozen_object': 'interp_imp.get_frozen_object',
         'is_frozen_package': 'interp_imp.is_frozen_package',
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -126,11 +126,6 @@
         space.sys.setmodule(w_mod)
     return w_mod
 
-def load_c_extension(space, filename, modulename):
-    from pypy.module.cpyext.api import load_extension_module
-    return load_extension_module(space, filename, modulename)
-    # NB. cpyext.api.load_extension_module() can also delegate to _cffi_backend
-
 # __________________________________________________________________
 #
 # import lock, to prevent two threads from running module-level code in
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -49,11 +49,9 @@
 def create_dynamic(space, w_spec, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
-    w_modulename = space.getattr(w_spec, space.newtext("name"))
-    w_path = space.getattr(w_spec, space.newtext("origin"))
-    filename = space.fsencode_w(w_path)
-    return importing.load_c_extension(space, filename,
-                                      space.text_w(w_modulename))
+    from pypy.module.cpyext.api import create_extension_module
+    # NB. cpyext.api.create_extension_module() can also delegate to _cffi_backend
+    return create_extension_module(space, w_spec)
 
 def create_builtin(space, w_spec):
     w_name = space.getattr(w_spec, space.newtext("name"))
@@ -66,8 +64,12 @@
     reuse = space.finditem(space.sys.get('modules'), w_name) is not None
     return space.getbuiltinmodule(name, force_init=True, reuse=reuse)
 
+def exec_dynamic(space, w_mod):
+    from pypy.module.cpyext.api import exec_extension_module
+    exec_extension_module(space, w_mod)
+
 def exec_builtin(space, w_mod):
-    return  # Until we really support ModuleDef
+    return
 
 def init_frozen(space, w_name):
     return None

From pypy.commits at gmail.com  Sun Mar 26 14:23:36 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 Mar 2017 11:23:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: clean up tests,
 tp_new is set by PyType_Ready
Message-ID: <58d80728.0d50190a.5fe6e.8469@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90813:8db3b9a6b0a2
Date: 2017-03-26 20:57 +0300
http://bitbucket.org/pypy/pypy/changeset/8db3b9a6b0a2/

Log:	clean up tests, tp_new is set by PyType_Ready

diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -344,7 +344,6 @@
 #endif
     if (m == NULL)
         INITERROR;
-    PyMyArrayType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&PyMyArrayType) < 0)
         INITERROR;
     Py_INCREF(&PyMyArrayType);
diff --git a/pypy/module/cpyext/test/comparisons.c b/pypy/module/cpyext/test/comparisons.c
--- a/pypy/module/cpyext/test/comparisons.c
+++ b/pypy/module/cpyext/test/comparisons.c
@@ -112,8 +112,6 @@
         return;
     if (PyType_Ready(&OldCmpType) < 0)
         return;
-    CmpType.tp_new = PyType_GenericNew;
-    OldCmpType.tp_new = PyType_GenericNew;
     m = Py_InitModule("comparisons", NULL);
     if (m == NULL)
         return;
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -36,14 +36,14 @@
     Py_ssize_t foo_ssizet;
 } fooobject;
 
-static PyTypeObject footype;
+static PyTypeObject fooType;
 
 static fooobject *
 newfooobject(void)
 {
     fooobject *foop;
 
-    foop = PyObject_New(fooobject, &footype);
+    foop = PyObject_New(fooobject, &fooType);
     if (foop == NULL)
         return NULL;
 
@@ -194,7 +194,7 @@
 
 PyDoc_STRVAR(foo_doc, "foo is for testing.");
 
-static PyTypeObject footype = {
+static PyTypeObject fooType = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
     sizeof(fooobject),       /*tp_size*/
@@ -706,13 +706,11 @@
     if (module == NULL)
         INITERROR;
 
-    footype.tp_new = PyType_GenericNew;
-
     UnicodeSubtype.tp_base = &PyUnicode_Type;
     UnicodeSubtype2.tp_base = &UnicodeSubtype;
     MetaType.tp_base = &PyType_Type;
 
-    if (PyType_Ready(&footype) < 0)
+    if (PyType_Ready(&fooType) < 0)
         INITERROR;
     if (PyType_Ready(&UnicodeSubtype) < 0)
         INITERROR;
@@ -725,8 +723,6 @@
     if (PyType_Ready(&SimplePropertyType) < 0)
         INITERROR;
 
-    SimplePropertyType.tp_new = PyType_GenericNew;
-    InitErrType.tp_new = PyType_GenericNew;
 
     Py_TYPE(&CustomType) = &MetaType;
     if (PyType_Ready(&CustomType) < 0)
@@ -748,7 +744,7 @@
     d = PyModule_GetDict(module);
     if (d == NULL)
         INITERROR;
-    if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0)
+    if (PyDict_SetItemString(d, "fooType", (PyObject *)&fooType) < 0)
         INITERROR;
     if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0)
         INITERROR;
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -13,12 +13,12 @@
         assert 'foo' in sys.modules
         assert "copy" in dir(module.fooType)
         obj = module.new()
-        print(obj.foo)
+        #print(obj.foo)
         assert obj.foo == 42
-        print("Obj has type", type(obj))
+        #print("Obj has type", type(obj))
         assert type(obj) is module.fooType
-        print("type of obj has type", type(type(obj)))
-        print("type of type of obj has type", type(type(type(obj))))
+        #print("type of obj has type", type(type(obj)))
+        #print("type of type of obj has type", type(type(type(obj))))
         assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):

From pypy.commits at gmail.com  Sun Mar 26 14:23:38 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 Mar 2017 11:23:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: add comment about leaking reference in
 test, message appears after test teardown
Message-ID: <58d8072a.02502e0a.db2d6.7df4@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90814:796aa1a166cb
Date: 2017-03-26 20:58 +0300
http://bitbucket.org/pypy/pypy/changeset/796aa1a166cb/

Log:	add comment about leaking reference in test, message appears after
	test teardown

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -166,6 +166,10 @@
                           " on too long format string"
         finally:
             warnings.resetwarnings()
+        # calling get_buffer_info on x creates a memory leak,
+        # which is detected as an error at test teardown:
+        # Exception TypeError: "'NoneType' object is not callable"
+        #         in  ignored
 
     def test_releasebuffer(self):
         if not self.runappdirect:

From pypy.commits at gmail.com  Sun Mar 26 15:14:00 2017
From: pypy.commits at gmail.com (mjacob)
Date: Sun, 26 Mar 2017 12:14:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove this test involving old style
 classes, which were removed in Python 3.0.
Message-ID: <58d812f8.09132e0a.61c6d.823a@mx.google.com>

Author: Manuel Jacob 
Branch: py3.5
Changeset: r90816:362dac153db1
Date: 2017-03-26 20:33 +0200
http://bitbucket.org/pypy/pypy/changeset/362dac153db1/

Log:	Remove this test involving old style classes, which were removed in
	Python 3.0.

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1197,25 +1197,3 @@
             pass
         bases = module.foo(C)
         assert bases == (A, B)
-
-    def test_multiple_inheritance_old_style_base(self):
-        module = self.import_extension('foo', [
-           ("foo", "METH_O",
-            '''
-                PyTypeObject *tp;
-                tp = (PyTypeObject*)args;
-                Py_INCREF(tp->tp_bases);
-                return tp->tp_bases;
-            '''
-            )])
-        # used to segfault after some iterations
-        for i in range(11):
-            print i
-            class A(object):
-                pass
-            class B:
-                pass
-            class C(A, B):
-                pass
-            bases = module.foo(C)
-            assert bases == (A, B)

From pypy.commits at gmail.com  Sun Mar 26 15:13:58 2017
From: pypy.commits at gmail.com (mjacob)
Date: Sun, 26 Mar 2017 12:13:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove this test testing PyClass_New,
 which was removed in Python 3.x.
Message-ID: <58d812f6.d8202e0a.57423.82da@mx.google.com>

Author: Manuel Jacob 
Branch: py3.5
Changeset: r90815:0456fd144e9f
Date: 2017-03-26 20:26 +0200
http://bitbucket.org/pypy/pypy/changeset/0456fd144e9f/

Log:	Remove this test testing PyClass_New, which was removed in Python
	3.x.

diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py
--- a/pypy/module/cpyext/test/test_classobject.py
+++ b/pypy/module/cpyext/test/test_classobject.py
@@ -27,15 +27,3 @@
         InstanceMethod.testmethod.attribute = "test"
         assert testfunction.attribute == "test"
         raises(AttributeError, setattr, inst.testmethod, "attribute", "test")
-
-    def test_pyclass_new_no_bases(self):
-        module = self.import_extension('foo', [
-            ("new_foo", "METH_O",
-             """
-                 return PyClass_New(NULL, PyDict_New(), args);
-             """)])
-        FooClass = module.new_foo("FooClass")
-        class Cls1:
-            pass
-        assert type(FooClass) is type(Cls1)
-        assert FooClass.__bases__ == Cls1.__bases__

From pypy.commits at gmail.com  Sun Mar 26 15:50:48 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 Mar 2017 12:50:48 -0700 (PDT)
Subject: [pypy-commit] pypy issue2522: add failing test for multiple bases
Message-ID: <58d81b98.1f002e0a.8e31e.8881@mx.google.com>

Author: Matti Picus 
Branch: issue2522
Changeset: r90818:65ab47447a58
Date: 2017-03-26 21:52 +0300
http://bitbucket.org/pypy/pypy/changeset/65ab47447a58/

Log:	add failing test for multiple bases

diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -160,6 +160,28 @@
     return PyObject_GenericSetAttr((PyObject *)self, name, value);
 }
 
+static PyObject *
+new_fooType(PyTypeObject * t, PyObject *args, PyObject *kwds)
+{
+    PyObject * o;
+    /* copied from numpy scalartypes.c for inherited classes */
+    if (t->tp_bases && (PyTuple_GET_SIZE(t->tp_bases) > 1))
+    {
+        PyTypeObject *sup; 
+        /* We are inheriting from a Python type as well so 
+           give it first dibs on conversion */
+        sup = (PyTypeObject *)PyTuple_GET_ITEM(t->tp_bases, 1);
+        /* Prevent recursion */ 
+        if (new_fooType != sup->tp_new) 
+        {
+            o = sup->tp_new(t, args, kwds);
+            return o;
+        }
+    }
+    o = t->tp_alloc(t, 0);
+    return o;   
+};
+
 static PyMemberDef foo_members[] = {
     {"int_member", T_INT, offsetof(fooobject, foo), 0,
      "A helpful docstring."},
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -974,6 +974,8 @@
             pass
         class foo(f2, f1):
             pass
+
+        x = foo(1)
         assert bar.__base__ is f2
         # On cpython, the size changes.
         if '__pypy__' in sys.builtin_module_names:

From pypy.commits at gmail.com  Sun Mar 26 15:50:50 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 Mar 2017 12:50:50 -0700 (PDT)
Subject: [pypy-commit] pypy issue2522: fix for -A,
 fix issue setting tp_new to 0x1
Message-ID: <58d81b9a.0d152e0a.59711.7d2d@mx.google.com>

Author: Matti Picus 
Branch: issue2522
Changeset: r90819:c121ac248c57
Date: 2017-03-26 22:42 +0300
http://bitbucket.org/pypy/pypy/changeset/c121ac248c57/

Log:	fix for -A, fix issue setting tp_new to 0x1

diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -732,6 +732,7 @@
     UnicodeSubtype2.tp_base = &UnicodeSubtype;
     MetaType.tp_base = &PyType_Type;
 
+    fooType.tp_new = &new_fooType;
     InitErrType.tp_new = PyType_GenericNew;
 
     if (PyType_Ready(&fooType) < 0)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -975,7 +975,7 @@
         class foo(f2, f1):
             pass
 
-        x = foo(1)
+        x = foo()
         assert bar.__base__ is f2
         # On cpython, the size changes.
         if '__pypy__' in sys.builtin_module_names:
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -770,10 +770,6 @@
         if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
             pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
-    if space.is_w(w_type, space.w_object):
-        # will be filled later on with the correct value
-        # may not be 0
-        pto.c_tp_new = cts.cast('newfunc', 1)
     update_all_slots(space, w_type, pto)
     if not pto.c_tp_new:
         base_object_pyo = make_ref(space, space.w_object)

From pypy.commits at gmail.com  Sun Mar 26 15:50:45 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 Mar 2017 12:50:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: revert parts of 8db3b9a6b0a2 for -A
 tests
Message-ID: <58d81b95.0c2b190a.13b28.7242@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90817:be8d848b5214
Date: 2017-03-26 21:51 +0300
http://bitbucket.org/pypy/pypy/changeset/be8d848b5214/

Log:	revert parts of 8db3b9a6b0a2 for -A tests

diff --git a/pypy/module/cpyext/test/comparisons.c b/pypy/module/cpyext/test/comparisons.c
--- a/pypy/module/cpyext/test/comparisons.c
+++ b/pypy/module/cpyext/test/comparisons.c
@@ -64,7 +64,7 @@
     0,                                              /* tp_dictoffset */
     0,                                              /* tp_init */
     0,                                              /* tp_alloc */
-    0,                                              /* tp_new */
+    PyType_GenericNew,                              /* tp_new */
     0                                               /* tp_free */
 };
 
@@ -108,6 +108,8 @@
 {
     PyObject *m, *d;
 
+    OldCmpType.tp_new = &PyType_GenericNew;
+
     if (PyType_Ready(&CmpType) < 0)
         return;
     if (PyType_Ready(&OldCmpType) < 0)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -592,7 +592,7 @@
 
     0,          /*tp_init*/
     0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
-    0,          /*tp_new*/
+    PyType_GenericNew, /*tp_new*/
     0,          /*tp_free  Low-level free-memory routine */
     0,          /*tp_is_gc For PyObject_IS_GC */
     0,          /*tp_bases*/
@@ -710,6 +710,8 @@
     UnicodeSubtype2.tp_base = &UnicodeSubtype;
     MetaType.tp_base = &PyType_Type;
 
+    InitErrType.tp_new = PyType_GenericNew;
+
     if (PyType_Ready(&fooType) < 0)
         INITERROR;
     if (PyType_Ready(&UnicodeSubtype) < 0)

From pypy.commits at gmail.com  Mon Mar 27 09:33:24 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Mon, 27 Mar 2017 06:33:24 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: merge default into
 sirtom67/float_complex.
Message-ID: <58d914a4.904d190a.c2d59.0d05@mx.google.com>

Author: Tom Krauss 
Branch: sirtom67/float_complex
Changeset: r2922:ce482ab27242
Date: 2017-03-27 08:32 -0500
http://bitbucket.org/cffi/cffi/changeset/ce482ab27242/

Log:	merge default into sirtom67/float_complex.

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1180,7 +1180,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.10.0.tar.gz
 
-   - MD5: ...
+   - MD5: 2b5fa41182ed0edaf929a789e602a070
 
-   - SHA: ...
+   - SHA: 8484aba03d1e64367d3110c0e36c1ed052b43f12
 
-   - SHA256: ...
+   - SHA256: b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -9,18 +9,16 @@
 * Issue #295: use calloc() directly instead of
   PyObject_Malloc()+memset() to handle ffi.new() with a default
   allocator.  Speeds up ``ffi.new(large-array)`` where most of the time
-  you never touch most of the array.  (But avoid doing that too often:
-  on 32-bit PyPy it will quickly exhaust the address space.  If possible,
-  use instead explicit calls to calloc() and free().)
+  you never touch most of the array.
 
 * Some OS/X build fixes ("only with Xcode but without CLT").
 
 * Improve a couple of error messages: when getting mismatched versions
   of cffi and its backend; and when calling functions which cannot be
   called with libffi because an argument is a struct that is "too
-  complicated" (not a struct *pointer*).
+  complicated" (and not a struct *pointer*, which always works).
 
-* Add support for some obscure compilers (non-msvc, non-gcc, non-icc,
+* Add support for some unusual compilers (non-msvc, non-gcc, non-icc,
   non-clang)
 
 * Implemented the remaining cases for ``ffi.from_buffer``.  Now all
@@ -33,15 +31,15 @@
 
 * The C type ``_Bool`` or ``bool`` now converts to a Python boolean
   when reading, instead of the content of the byte as an integer.  The
-  change here is mostly what occurs if the byte happens to contain a
+  potential incompatibility here is what occurs if the byte contains a
   value different from 0 and 1.  Previously, it would just return it;
   with this change, CFFI raises an exception in this case.  But this
   case means "undefined behavior" in C; if you really have to interface
-  with a library relying on this, don't use ``_Bool`` in the CFFI side.
+  with a library relying on this, don't use ``bool`` in the CFFI side.
   Also, it is still valid to use a byte string as initializer for a
-  ``_Bool[]``, but now it must only contain ``\x00`` or ``\x01``.  As an
-  aside, ``ffi.string()`` no longer works on ``_Bool[]`` (but it never
-  made much sense, as this function stops on the first zero).
+  ``bool[]``, but now it must only contain ``\x00`` or ``\x01``.  As an
+  aside, ``ffi.string()`` no longer works on ``bool[]`` (but it never
+  made much sense, as this function stops at the first zero).
 
 * ``ffi.buffer`` is now the name of cffi's buffer type, and
   ``ffi.buffer()`` works like before but is the constructor of that type.
@@ -61,6 +59,11 @@
   "memory pressure", causing the GC to run too infrequently if you call
   ``ffi.new()`` very often and/or with large arrays.  Fixed in PyPy 5.7.
 
+* Support in ``ffi.cdef()`` for numeric expressions with ``+`` or
+  ``-``.  Assumes that there is no overflow; it should be fixed first
+  before we add more general support for arbitrary arithmetic on
+  constants.
+
 
 v1.9
 ====
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1552,7 +1552,8 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function bar() called, but no code was attached "
+        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
+        b"but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")

From pypy.commits at gmail.com  Mon Mar 27 09:33:19 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Mon, 27 Mar 2017 06:33:19 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: merge default in again.
Message-ID: <58d9149f.1b582e0a.32fd3.0d6d@mx.google.com>

Author: Tom Krauss 
Branch: sirtom67/float_complex
Changeset: r2920:8e884c0520fe
Date: 2017-03-19 18:33 -0500
http://bitbucket.org/cffi/cffi/changeset/8e884c0520fe/

Log:	merge default in again.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -803,6 +803,16 @@
                            "the actual array length in this context"
                            % exprnode.coord.line)
         #
+        if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and
+                exprnode.op == '+'):
+            return (self._parse_constant(exprnode.left) +
+                    self._parse_constant(exprnode.right))
+        #
+        if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and
+                exprnode.op == '-'):
+            return (self._parse_constant(exprnode.left) -
+                    self._parse_constant(exprnode.right))
+        #
         raise FFIError(":%d: unsupported expression: expected a "
                        "simple numeric constant" % exprnode.coord.line)
 
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1230,7 +1230,8 @@
     def test_ffi_buffer_comparisons(self):
         ffi = FFI(backend=self.Backend())
         ba = bytearray(range(100, 110))
-        assert ba == memoryview(ba)    # justification for the following
+        if sys.version_info >= (2, 7):
+            assert ba == memoryview(ba)    # justification for the following
         a = ffi.new("uint8_t[]", list(ba))
         c = ffi.new("uint8_t[]", [99] + list(ba))
         try:
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -386,13 +386,14 @@
 def test_enum():
     ffi = FFI()
     ffi.cdef("""
-        enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1};
+        enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1};
         """)
     C = ffi.dlopen(None)
     assert C.POS == 1
     assert C.TWO == 2
     assert C.NIL == 0
     assert C.NEG == -1
+    assert C.OP == 2
 
 def test_stdcall():
     ffi = FFI()
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2063,7 +2063,7 @@
             struct foo s = { 40, 200 };
             return s;
         }
-        struct foo g(int a, ...) { }
+        struct foo g(int a, ...) { return f(); }
     """)
     assert lib.f().x == 200
     e = py.test.raises(NotImplementedError, lib.g, 0)
@@ -2092,7 +2092,7 @@
             s.b = 200;
             return s;
         }
-        struct foo g(int a, ...) { }
+        struct foo g(int a, ...) { return f(); }
     """)
     assert lib.f().b == 200
     e = py.test.raises(NotImplementedError, lib.g, 0)
@@ -2118,7 +2118,7 @@
             struct foo s = { 11 };
             return s;
         }
-        struct foo g(int a, ...) { }
+        struct foo g(int a, ...) { return f(); }
     """)
     assert lib.f().x == 11
     e = py.test.raises(NotImplementedError, lib.g, 0)
@@ -2130,6 +2130,8 @@
         "set_source() and not taking a final '...' argument)")
 
 def test_call_with_zero_length_field():
+    if sys.platform == 'win32':
+        py.test.skip("zero-length field not supported by MSVC")
     ffi = FFI()
     ffi.cdef("""
         struct foo { int a; int x[0]; };
@@ -2142,7 +2144,7 @@
             struct foo s = { 42 };
             return s;
         }
-        struct foo g(int a, ...) { }
+        struct foo g(int a, ...) { return f(); }
     """)
     assert lib.f().a == 42
     e = py.test.raises(NotImplementedError, lib.g, 0)
@@ -2166,7 +2168,7 @@
             union foo s = { 42 };
             return s;
         }
-        union foo g(int a, ...) { }
+        union foo g(int a, ...) { return f(); }
     """)
     assert lib.f().a == 42
     e = py.test.raises(NotImplementedError, lib.g, 0)

From pypy.commits at gmail.com  Mon Mar 27 09:33:21 2017
From: pypy.commits at gmail.com (sirtom67)
Date: Mon, 27 Mar 2017 06:33:21 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: tests pass. Had to
 #include  - might want to make that optional
Message-ID: <58d914a1.4f5c190a.f0702.0ccb@mx.google.com>

Author: Tom Krauss 
Branch: sirtom67/float_complex
Changeset: r2921:8e77b23da941
Date: 2017-03-27 08:27 -0500
http://bitbucket.org/cffi/cffi/changeset/8e77b23da941/

Log:	tests pass. Had to #include  - might want to make that
	optional

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2964,9 +2964,20 @@
     if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) {
         Py_complex value = read_raw_complex_data(cd->c_data, cd->c_type->ct_size);
         PyObject *op = PyComplex_FromCComplex(value);
-        PyComplexObject *opc = (PyComplexObject *) op;
         return op;
     }
+    // floats can also be converted to complex
+    if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
+        Py_complex value;
+        if (!(cd->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
+            value.real = read_raw_float_data(cd->c_data, cd->c_type->ct_size);
+        }
+        else {
+            value.real = (double)read_raw_longdouble_data(cd->c_data);
+        }
+        value.imag = 0.0;
+        return PyComplex_FromCComplex(value);
+    }
     PyErr_Format(PyExc_TypeError, "complex() not supported on cdata '%s'",
                  cd->c_type->ct_name);
     return NULL;
diff --git a/c/parse_c_type.c b/c/parse_c_type.c
--- a/c/parse_c_type.c
+++ b/c/parse_c_type.c
@@ -797,7 +797,6 @@
 int parse_c_type_from(struct _cffi_parse_info_s *info, size_t *output_index,
                       const char *input)
 {
-    printf("parse_c_type_from\n");
     int result;
     token_t token;
 
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -16,6 +16,7 @@
 #endif
 
 #include 
+#include 
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -99,6 +100,29 @@
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
 
+#define _cffi_from_c_float__Complex(x)  PyComplex_FromDoubles(crealf(x), cimagf(x))
+#define _cffi_from_c_double__Complex(x) PyComplex_FromDoubles(creal(x), cimag(x))
+
+/* inefficient - converts twice! */
+#define _cffi_to_c_float__Complex(op)                                    \
+   (  ((float)(PyComplex_AsCComplex(op).real))  +                       \
+    I*((float)(PyComplex_AsCComplex(op).imag))     )
+/* not safe!
+//#define _cffi_to_c_float__Complex(op)                                  \
+//    (  ((float)((PyComplexObject *)(op))->cval.real)  +                \
+//     I*((float)((PyComplexObject *)(op))->cval.imag)     )
+*/
+
+/* inefficient - converts twice! */
+#define _cffi_to_c_double__Complex(op)                                   \
+   (  (PyComplex_AsCComplex(op).real)  +                                \
+    I*(PyComplex_AsCComplex(op).imag)     )
+/* not safe!
+//#define _cffi_to_c_double__Complex(op)                                   \
+//    (  (((PyComplexObject *)(op))->cval.real)  +                         \
+//     I*(((PyComplexObject *)(op))->cval.imag)     )
+*/
+
 #define _cffi_from_c_int(x, type)                                        \
     (((type)-1) > 0 ? /* unsigned */                                     \
         (sizeof(type) < sizeof(long) ?                                   \
@@ -110,9 +134,6 @@
             PyInt_FromLong((long)x) :                                    \
             PyLong_FromLongLong((long long)x)))
 
-#define _cffi_from_c_float__Complex(x)  PyComplex_FromDoubles(crealf(x), cimagf(x))
-#define _cffi_from_c_double__Complex(x) PyComplex_FromDoubles(creal(x), cimag(x))
-
 #define _cffi_to_c_int(o, type)                                          \
     ((type)(                                                             \
      sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -806,6 +806,7 @@
 cffimod_header = r'''
 #include 
 #include 
+#include 
 
 /* this block of #ifs should be kept exactly identical between
    c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
@@ -873,6 +874,29 @@
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
 
+#define _cffi_from_c_float__Complex(x)  PyComplex_FromDoubles(crealf(x), cimagf(x))
+#define _cffi_from_c_double__Complex(x) PyComplex_FromDoubles(creal(x), cimag(x))
+
+/* inefficient - converts twice! */
+#define _cffi_to_c_float__Complex(op)                                    \
+    (  ((float)(PyComplex_AsCComplex(op).real))  +                       \
+     I*((float)(PyComplex_AsCComplex(op).imag))     )
+/* not safe!
+//#define _cffi_to_c_float__Complex(op)                                  \
+//    (  ((float)((PyComplexObject *)(op))->cval.real)  +                \
+//     I*((float)((PyComplexObject *)(op))->cval.imag)     )
+*/
+
+/* inefficient - converts twice! */
+#define _cffi_to_c_double__Complex(op)                                   \
+    (  (PyComplex_AsCComplex(op).real)  +                                \
+     I*(PyComplex_AsCComplex(op).imag)     )
+/* not safe!
+//#define _cffi_to_c_double__Complex(op)                                   \
+//    (  (((PyComplexObject *)(op))->cval.real)  +                         \
+//     I*(((PyComplexObject *)(op))->cval.imag)     )
+*/
+
 #define _cffi_from_c_int_const(x)                                        \
     (((x) > 0) ?                                                         \
         ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ?      \
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -241,14 +241,15 @@
         F = tp.is_float_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
-        assert F == (typename in ('float', 'double', 'long double'))
+        assert F == (typename in ('float', 'double', 'long double', 'float _Complex', 'double _Complex'))
         assert I + F + C == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
-            typename == '_Bool' or typename == 'long double'):
+            typename == '_Bool' or typename == 'long double'
+            or '_Complex' in typename):  # omit _Complex since ffi does not yet support
             pass
         else:
             typenames.append(typename)
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1704,6 +1704,8 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
+            'double _Complex',
+            'float _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/testing/cffi1/test_realize_c_type.py b/testing/cffi1/test_realize_c_type.py
--- a/testing/cffi1/test_realize_c_type.py
+++ b/testing/cffi1/test_realize_c_type.py
@@ -45,8 +45,14 @@
 
 def test_all_primitives():
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
-        check(name, name)
+        if '_Complex' not in name:
+            check(name, name)
 
+def test_complex_primitives():
+    py.test.xfail("ffi does not support complex yet")
+    for name in cffi_opcode.PRIMITIVE_TO_INDEX:
+        if '_Complex' in name:
+            check(name, name)
 
 def check_func(input, expected_output=None):
     import _cffi_backend
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -221,7 +221,7 @@
         F = tp.is_float_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
-        assert F == (typename in ('float', 'double', 'long double'))
+        assert F == (typename in ('float', 'double', 'long double', 'float _Complex', 'double _Complex'))
         assert I + F + C == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():

From pypy.commits at gmail.com  Mon Mar 27 12:38:15 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 Mar 2017 09:38:15 -0700 (PDT)
Subject: [pypy-commit] pypy issue2522: close branch to be merged
Message-ID: <58d93ff7.1a142e0a.f41a5.15c4@mx.google.com>

Author: Matti Picus 
Branch: issue2522
Changeset: r90820:7fec9e03d6fa
Date: 2017-03-27 19:09 +0300
http://bitbucket.org/pypy/pypy/changeset/7fec9e03d6fa/

Log:	close branch to be merged


From pypy.commits at gmail.com  Mon Mar 27 12:38:18 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 Mar 2017 09:38:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge issue2522 which fixes tp_new on
 w_object, used in multiple inheritance
Message-ID: <58d93ffa.84202e0a.54657.1506@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90821:daea5c82c284
Date: 2017-03-27 19:10 +0300
http://bitbucket.org/pypy/pypy/changeset/daea5c82c284/

Log:	merge issue2522 which fixes tp_new on w_object, used in multiple
	inheritance

diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -160,6 +160,28 @@
     return PyObject_GenericSetAttr((PyObject *)self, name, value);
 }
 
+static PyObject *
+new_fooType(PyTypeObject * t, PyObject *args, PyObject *kwds)
+{
+    PyObject * o;
+    /* copied from numpy scalartypes.c for inherited classes */
+    if (t->tp_bases && (PyTuple_GET_SIZE(t->tp_bases) > 1))
+    {
+        PyTypeObject *sup; 
+        /* We are inheriting from a Python type as well so 
+           give it first dibs on conversion */
+        sup = (PyTypeObject *)PyTuple_GET_ITEM(t->tp_bases, 1);
+        /* Prevent recursion */ 
+        if (new_fooType != sup->tp_new) 
+        {
+            o = sup->tp_new(t, args, kwds);
+            return o;
+        }
+    }
+    o = t->tp_alloc(t, 0);
+    return o;   
+};
+
 static PyMemberDef foo_members[] = {
     {"int_member", T_INT, offsetof(fooobject, foo), 0,
      "A helpful docstring."},
@@ -710,6 +732,7 @@
     UnicodeSubtype2.tp_base = &UnicodeSubtype;
     MetaType.tp_base = &PyType_Type;
 
+    fooType.tp_new = &new_fooType;
     InitErrType.tp_new = PyType_GenericNew;
 
     if (PyType_Ready(&fooType) < 0)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -974,6 +974,8 @@
             pass
         class foo(f2, f1):
             pass
+
+        x = foo()
         assert bar.__base__ is f2
         # On cpython, the size changes.
         if '__pypy__' in sys.builtin_module_names:
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -770,10 +770,6 @@
         if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
             pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
-    if space.is_w(w_type, space.w_object):
-        # will be filled later on with the correct value
-        # may not be 0
-        pto.c_tp_new = cts.cast('newfunc', 1)
     update_all_slots(space, w_type, pto)
     if not pto.c_tp_new:
         base_object_pyo = make_ref(space, space.w_object)

From pypy.commits at gmail.com  Mon Mar 27 12:38:22 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 Mar 2017 09:38:22 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-injection: merge default into branch
Message-ID: <58d93ffe.12582e0a.15ce1.15b9@mx.google.com>

Author: Matti Picus 
Branch: cpyext-injection
Changeset: r90823:ff3d7db1e4c1
Date: 2017-03-27 19:11 +0300
http://bitbucket.org/pypy/pypy/changeset/ff3d7db1e4c1/

Log:	merge default into branch

diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -57,6 +57,7 @@
 .. code-block:: console
 
     $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install -U pip wheel # to upgrade to the latest versions
     $ ./pypy-xxx/bin/pip install pygments  # for example
 
 Third party libraries will be installed in ``pypy-xxx/site-packages``, and
@@ -77,7 +78,17 @@
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
 
-Note that bin/python is now a symlink to bin/pypy.
+	# in any case activate it
+	$ source my-pypy-env/bin/activate
+
+Note that my-pypy-env/bin/python is now a symlink to my-pypy-env/bin/pypy
+so you should be able to run pypy simply by typing::
+
+    $ python
+
+You should still upgrade pip and wheel to the latest versions via::
+
+    $ my-pypy-env/bin/pip install -U pip wheel
 
 .. _pip: http://pypi.python.org/pypi/pip
 .. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,7 +2,18 @@
 What's new in PyPy2.7 5.8+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.7
+.. this is a revision shortly after release-pypy2.7-v5.7.0
 .. startrev: 44f31f6dd39f
 
+Add cpyext interfaces for ``PyModule_New``
 
+Correctly handle `dict.pop`` where the ``pop``ping 
+key is not the same type as the ``dict``'s and ``pop`` 
+is called with a default (will be part of release 5.7.1)
+
+.. branch: issue2522
+
+Fix missing tp_new on w_object called through multiple inheritance
+(will be part of release 5.7.1)
+
+
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -344,7 +344,6 @@
 #endif
     if (m == NULL)
         INITERROR;
-    PyMyArrayType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&PyMyArrayType) < 0)
         INITERROR;
     Py_INCREF(&PyMyArrayType);
diff --git a/pypy/module/cpyext/test/comparisons.c b/pypy/module/cpyext/test/comparisons.c
--- a/pypy/module/cpyext/test/comparisons.c
+++ b/pypy/module/cpyext/test/comparisons.c
@@ -64,7 +64,7 @@
     0,                                              /* tp_dictoffset */
     0,                                              /* tp_init */
     0,                                              /* tp_alloc */
-    0,                                              /* tp_new */
+    PyType_GenericNew,                              /* tp_new */
     0                                               /* tp_free */
 };
 
@@ -108,12 +108,12 @@
 {
     PyObject *m, *d;
 
+    OldCmpType.tp_new = &PyType_GenericNew;
+
     if (PyType_Ready(&CmpType) < 0)
         return;
     if (PyType_Ready(&OldCmpType) < 0)
         return;
-    CmpType.tp_new = PyType_GenericNew;
-    OldCmpType.tp_new = PyType_GenericNew;
     m = Py_InitModule("comparisons", NULL);
     if (m == NULL)
         return;
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -36,14 +36,14 @@
     Py_ssize_t foo_ssizet;
 } fooobject;
 
-static PyTypeObject footype;
+static PyTypeObject fooType;
 
 static fooobject *
 newfooobject(void)
 {
     fooobject *foop;
 
-    foop = PyObject_New(fooobject, &footype);
+    foop = PyObject_New(fooobject, &fooType);
     if (foop == NULL)
         return NULL;
 
@@ -160,6 +160,28 @@
     return PyObject_GenericSetAttr((PyObject *)self, name, value);
 }
 
+static PyObject *
+new_fooType(PyTypeObject * t, PyObject *args, PyObject *kwds)
+{
+    PyObject * o;
+    /* copied from numpy scalartypes.c for inherited classes */
+    if (t->tp_bases && (PyTuple_GET_SIZE(t->tp_bases) > 1))
+    {
+        PyTypeObject *sup; 
+        /* We are inheriting from a Python type as well so 
+           give it first dibs on conversion */
+        sup = (PyTypeObject *)PyTuple_GET_ITEM(t->tp_bases, 1);
+        /* Prevent recursion */ 
+        if (new_fooType != sup->tp_new) 
+        {
+            o = sup->tp_new(t, args, kwds);
+            return o;
+        }
+    }
+    o = t->tp_alloc(t, 0);
+    return o;   
+};
+
 static PyMemberDef foo_members[] = {
     {"int_member", T_INT, offsetof(fooobject, foo), 0,
      "A helpful docstring."},
@@ -194,7 +216,7 @@
 
 PyDoc_STRVAR(foo_doc, "foo is for testing.");
 
-static PyTypeObject footype = {
+static PyTypeObject fooType = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
     sizeof(fooobject),       /*tp_size*/
@@ -592,7 +614,7 @@
 
     0,          /*tp_init*/
     0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
-    0,          /*tp_new*/
+    PyType_GenericNew, /*tp_new*/
     0,          /*tp_free  Low-level free-memory routine */
     0,          /*tp_is_gc For PyObject_IS_GC */
     0,          /*tp_bases*/
@@ -706,13 +728,14 @@
     if (module == NULL)
         INITERROR;
 
-    footype.tp_new = PyType_GenericNew;
-
     UnicodeSubtype.tp_base = &PyUnicode_Type;
     UnicodeSubtype2.tp_base = &UnicodeSubtype;
     MetaType.tp_base = &PyType_Type;
 
-    if (PyType_Ready(&footype) < 0)
+    fooType.tp_new = &new_fooType;
+    InitErrType.tp_new = PyType_GenericNew;
+
+    if (PyType_Ready(&fooType) < 0)
         INITERROR;
     if (PyType_Ready(&UnicodeSubtype) < 0)
         INITERROR;
@@ -725,8 +748,6 @@
     if (PyType_Ready(&SimplePropertyType) < 0)
         INITERROR;
 
-    SimplePropertyType.tp_new = PyType_GenericNew;
-    InitErrType.tp_new = PyType_GenericNew;
 
     Py_TYPE(&CustomType) = &MetaType;
     if (PyType_Ready(&CustomType) < 0)
@@ -748,7 +769,7 @@
     d = PyModule_GetDict(module);
     if (d == NULL)
         INITERROR;
-    if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0)
+    if (PyDict_SetItemString(d, "fooType", (PyObject *)&fooType) < 0)
         INITERROR;
     if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0)
         INITERROR;
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -166,6 +166,10 @@
                           " on too long format string"
         finally:
             warnings.resetwarnings()
+        # calling get_buffer_info on x creates a memory leak,
+        # which is detected as an error at test teardown:
+        # Exception TypeError: "'NoneType' object is not callable"
+        #         in  ignored
 
     def test_releasebuffer(self):
         if not self.runappdirect:
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -13,12 +13,12 @@
         assert 'foo' in sys.modules
         assert "copy" in dir(module.fooType)
         obj = module.new()
-        print(obj.foo)
+        #print(obj.foo)
         assert obj.foo == 42
-        print("Obj has type", type(obj))
+        #print("Obj has type", type(obj))
         assert type(obj) is module.fooType
-        print("type of obj has type", type(type(obj)))
-        print("type of type of obj has type", type(type(type(obj))))
+        #print("type of obj has type", type(type(obj)))
+        #print("type of type of obj has type", type(type(type(obj))))
         assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
@@ -974,6 +974,8 @@
             pass
         class foo(f2, f1):
             pass
+
+        x = foo()
         assert bar.__base__ is f2
         # On cpython, the size changes.
         if '__pypy__' in sys.builtin_module_names:
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -776,10 +776,6 @@
         if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
             pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
-    if space.is_w(w_type, space.w_object):
-        # will be filled later on with the correct value
-        # may not be 0
-        pto.c_tp_new = cts.cast('newfunc', 1)
     update_all_slots(space, w_type, pto)
     if not pto.c_tp_new:
         base_object_pyo = make_ref(space, space.w_object)

From pypy.commits at gmail.com  Mon Mar 27 12:38:20 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 Mar 2017 09:38:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: document changes since releaes 5.7.0
Message-ID: <58d93ffc.194b2e0a.bc348.16e2@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90822:27b41a204775
Date: 2017-03-27 19:11 +0300
http://bitbucket.org/pypy/pypy/changeset/27b41a204775/

Log:	document changes since releaes 5.7.0

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,7 +2,18 @@
 What's new in PyPy2.7 5.8+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.7
+.. this is a revision shortly after release-pypy2.7-v5.7.0
 .. startrev: 44f31f6dd39f
 
+Add cpyext interfaces for ``PyModule_New``
 
+Correctly handle `dict.pop`` where the ``pop``ping 
+key is not the same type as the ``dict``'s and ``pop`` 
+is called with a default (will be part of release 5.7.1)
+
+.. branch: issue2522
+
+Fix missing tp_new on w_object called through multiple inheritance
+(will be part of release 5.7.1)
+
+

From pypy.commits at gmail.com  Mon Mar 27 12:38:25 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Mon, 27 Mar 2017 09:38:25 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Fixes #2508 -- correctly
 handle dict.pop where the popping key is not the same type as the dict's
 and pop is called with a default
Message-ID: <58d94001.02502e0a.44891.15c9@mx.google.com>

Author: Alex Gaynor 
Branch: release-pypy2.7-5.x
Changeset: r90824:1aab6418ec7d
Date: 2017-03-27 19:17 +0300
http://bitbucket.org/pypy/pypy/changeset/1aab6418ec7d/

Log:	Fixes #2508 -- correctly handle dict.pop where the popping key is
	not the same type as the dict's and pop is called with a default
	(grafted from a1b0ce5e4915a6e46c9007f81a991cbaf179f027)

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1049,6 +1049,8 @@
             else:
                 return d.pop(key, w_default)
         elif self._never_equal_to(space.type(w_key)):
+            if w_default is not None:
+                return w_default
             raise KeyError
         else:
             self.switch_to_object_strategy(w_dict)
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -161,7 +161,7 @@
         w_d.initialize_content([(w(1), wb("a")), (w(2), wb("b"))])
         w_l = self.space.call_method(w_d, "keys")
         assert sorted(self.space.listview_int(w_l)) == [1,2]
-        
+
         # make sure that .keys() calls newlist_bytes for string dicts
         def not_allowed(*args):
             assert False, 'should not be called'
@@ -174,7 +174,7 @@
 
         # XXX: it would be nice if the test passed without monkeypatch.undo(),
         # but we need space.newlist_unicode for it
-        monkeypatch.undo() 
+        monkeypatch.undo()
         w_d = self.space.newdict()
         w_d.initialize_content([(w(u"a"), w(1)), (w(u"b"), w(6))])
         w_l = self.space.call_method(w_d, "keys")
@@ -223,6 +223,10 @@
         assert len(dd) == 1
         raises(KeyError, dd.pop, 33)
 
+        assert d.pop("abc", None) is None
+        raises(KeyError, d.pop, "abc")
+        assert len(d) == 2
+
     def test_has_key(self):
         d = {1: 2, 3: 4}
         assert d.has_key(1)
@@ -1466,4 +1470,3 @@
     fakespace = FakeSpace()
     d = fakespace.newdict(module=True)
     assert type(d.get_strategy()) is BytesDictStrategy
-

From pypy.commits at gmail.com  Mon Mar 27 12:38:27 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 Mar 2017 09:38:27 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: add failing test for
 multiple bases
Message-ID: <58d94003.d357190a.13852.146a@mx.google.com>

Author: Matti Picus 
Branch: release-pypy2.7-5.x
Changeset: r90825:99bc8099921e
Date: 2017-03-27 19:17 +0300
http://bitbucket.org/pypy/pypy/changeset/99bc8099921e/

Log:	add failing test for multiple bases (grafted from
	65ab47447a582ca2353f043d84d9d8225554d3bc)

diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -160,6 +160,28 @@
     return PyObject_GenericSetAttr((PyObject *)self, name, value);
 }
 
+static PyObject *
+new_fooType(PyTypeObject * t, PyObject *args, PyObject *kwds)
+{
+    PyObject * o;
+    /* copied from numpy scalartypes.c for inherited classes */
+    if (t->tp_bases && (PyTuple_GET_SIZE(t->tp_bases) > 1))
+    {
+        PyTypeObject *sup; 
+        /* We are inheriting from a Python type as well so 
+           give it first dibs on conversion */
+        sup = (PyTypeObject *)PyTuple_GET_ITEM(t->tp_bases, 1);
+        /* Prevent recursion */ 
+        if (new_fooType != sup->tp_new) 
+        {
+            o = sup->tp_new(t, args, kwds);
+            return o;
+        }
+    }
+    o = t->tp_alloc(t, 0);
+    return o;   
+};
+
 static PyMemberDef foo_members[] = {
     {"int_member", T_INT, offsetof(fooobject, foo), 0,
      "A helpful docstring."},
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -974,6 +974,8 @@
             pass
         class foo(f2, f1):
             pass
+
+        x = foo(1)
         assert bar.__base__ is f2
         # On cpython, the size changes.
         if '__pypy__' in sys.builtin_module_names:

From pypy.commits at gmail.com  Mon Mar 27 12:38:29 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 27 Mar 2017 09:38:29 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: fix for -A,
 fix issue setting tp_new to 0x1
Message-ID: <58d94005.0d50190a.ca227.164f@mx.google.com>

Author: Matti Picus 
Branch: release-pypy2.7-5.x
Changeset: r90826:0c25bd7d5d7f
Date: 2017-03-27 19:29 +0300
http://bitbucket.org/pypy/pypy/changeset/0c25bd7d5d7f/

Log:	fix for -A, fix issue setting tp_new to 0x1 (grafted from
	c121ac248c57)

diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -734,6 +734,9 @@
     UnicodeSubtype2.tp_base = &UnicodeSubtype;
     MetaType.tp_base = &PyType_Type;
 
+    footype.tp_new = &new_fooType;
+    InitErrType.tp_new = PyType_GenericNew;
+
     if (PyType_Ready(&footype) < 0)
         INITERROR;
     if (PyType_Ready(&UnicodeSubtype) < 0)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -975,7 +975,7 @@
         class foo(f2, f1):
             pass
 
-        x = foo(1)
+        x = foo()
         assert bar.__base__ is f2
         # On cpython, the size changes.
         if '__pypy__' in sys.builtin_module_names:
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -770,10 +770,6 @@
         if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize:
             pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize
 
-    if space.is_w(w_type, space.w_object):
-        # will be filled later on with the correct value
-        # may not be 0
-        pto.c_tp_new = cts.cast('newfunc', 1)
     update_all_slots(space, w_type, pto)
     if not pto.c_tp_new:
         base_object_pyo = make_ref(space, space.w_object)

From pypy.commits at gmail.com  Mon Mar 27 19:46:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 27 Mar 2017 16:46:07 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Tyr to use composition instead of
 inheritance to model the relationship between
Message-ID: <58d9a43f.1a4c2e0a.1ec39.2414@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90827:3d947926d247
Date: 2017-03-28 00:45 +0100
http://bitbucket.org/pypy/pypy/changeset/3d947926d247/

Log:	Tyr to use composition instead of inheritance to model the
	relationship between Py_buffers and C-style buffers.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -10,7 +10,7 @@
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
 
-from pypy.interpreter.buffer import StringBuffer
+from pypy.interpreter.buffer import SimpleBuffer, StringBuffer
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     make_finalizer_queue)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
@@ -1519,10 +1519,11 @@
             # most API in CPython 3.x no longer do.
             if self.isinstance_w(w_obj, self.w_bytes):
                 return StringBuffer(w_obj.bytes_w(self))
-            if self.isinstance_w(w_obj, self.w_unicode):  # NB. CPython forbids
-                return StringBuffer(w_obj.text_w(self))   # surrogates here
+            if self.isinstance_w(w_obj, self.w_unicode):
+                # NB. CPython forbids surrogates here
+                return StringBuffer(w_obj.text_w(self))
             try:
-                return w_obj.buffer_w(self, self.BUF_SIMPLE)
+                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_binary()
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or buffer", w_obj)
         elif code == 's#':
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -57,6 +57,10 @@
     def get_raw_address(self):
         raise ValueError("no raw buffer")
 
+    def as_binary(self):
+        # Inefficient. May be overridden.
+        return StringBuffer(self.as_str())
+
     def getformat(self):
         raise NotImplementedError
 
@@ -75,15 +79,34 @@
     def releasebuffer(self):
         pass
 
-class BinaryBuffer(Buffer):
-    """Base class for buffers of bytes"""
-    _attrs_ = ['readonly']
+class SimpleBuffer(Buffer):
+    _attrs_ = ['readonly', 'data']
     _immutable_ = True
 
+    def __init__(self, data):
+        self.data = data
+        self.readonly = self.data.readonly
+
+    def getlength(self):
+        return self.data.getlength()
+
     def as_str(self):
-        "Returns an interp-level string with the whole content of the buffer."
-        # May be overridden.
-        return self.getslice(0, self.getlength(), 1, self.getlength())
+        return self.data.as_str()
+
+    def as_str_and_offset_maybe(self):
+        return self.data.as_str_and_offset_maybe()
+
+    def getitem(self, index):
+        return self.data.getitem(index)
+
+    def setitem(self, index, value):
+        return self.data.setitem(index, value)
+
+    def get_raw_address(self):
+        return self.data.get_raw_address()
+
+    def as_binary(self):
+        return self.data
 
     def getformat(self):
         return 'B'
@@ -100,6 +123,41 @@
     def getstrides(self):
         return [1]
 
+class BinaryBuffer(Buffer):
+    """Base class for buffers of bytes"""
+    _attrs_ = ['readonly']
+    _immutable_ = True
+
+    def as_str(self):
+        "Returns an interp-level string with the whole content of the buffer."
+        # May be overridden.
+        return self.getslice(0, self.getlength(), 1, self.getlength())
+
+    def getslice(self, start, stop, step, size):
+        # May be overridden.  No bounds checks.
+        return ''.join([self.getitem(i) for i in range(start, stop, step)])
+
+
+    def setslice(self, start, string):
+        # May be overridden.  No bounds checks.
+        for i in range(len(string)):
+            self.setitem(start + i, string[i])
+
+
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
 
 
 class ByteBuffer(BinaryBuffer):
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -2,9 +2,9 @@
 # A convenient read-write buffer.  Located here for want of a better place.
 #
 
-from pypy.interpreter.buffer import ByteBuffer
+from pypy.interpreter.buffer import SimpleBuffer, ByteBuffer
 from pypy.interpreter.gateway import unwrap_spec
 
 @unwrap_spec(length=int)
 def bytebuffer(space, length):
-    return space.newbuffer(ByteBuffer(length))
+    return space.newbuffer(SimpleBuffer(ByteBuffer(length)))
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -5,7 +5,7 @@
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
 from pypy.module._cffi_backend import ctypestruct
 
-from pypy.interpreter.buffer import BinaryBuffer
+from pypy.interpreter.buffer import SimpleBuffer, BinaryBuffer
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
@@ -47,7 +47,7 @@
         self.keepalive = keepalive
 
     def buffer_w(self, space, flags):
-        return self.buffer
+        return SimpleBuffer(self.buffer)
 
     def descr_len(self, space):
         return space.newint(self.buffer.getlength())
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.interpreter.buffer import BinaryBuffer, SubBuffer
+from pypy.interpreter.buffer import SimpleBuffer, BinaryBuffer, SubBuffer
 from rpython.rlib.rgc import (
     nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
 from rpython.rlib.rstring import StringBuilder
@@ -579,7 +579,7 @@
     def _raw_read(self, space, buffer, start, length):
         length = intmask(length)
         start = intmask(start)
-        w_buf = space.newbuffer(SubBuffer(buffer, start, length))
+        w_buf = space.newbuffer(SimpleBuffer(SubBuffer(buffer, start, length)))
         while True:
             try:
                 w_size = space.call_method(self.w_raw, "readinto", w_buf)
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.typedef import (
     TypeDef, generic_new_descr, GetSetProperty)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.buffer import BinaryBuffer
+from pypy.interpreter.buffer import SimpleBuffer, BinaryBuffer
 from rpython.rlib.rStringIO import RStringIO
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.objectmodel import import_from_mixin
@@ -125,7 +125,7 @@
 
     def getbuffer_w(self, space):
         self._check_closed(space)
-        return W_MemoryView(BytesIOBuffer(self))
+        return W_MemoryView(SimpleBuffer(BytesIOBuffer(self)))
 
     def getvalue_w(self, space):
         self._check_closed(space)
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -1,5 +1,6 @@
 import sys
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SimpleBuffer
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import interp_attrproperty
@@ -380,7 +381,7 @@
         self._ll_buffer = self.ll_buffer
 
     def buffer_w(self, space, flags):
-        return RawFFIBuffer(self)
+        return SimpleBuffer(RawFFIBuffer(self))
 
     def getrawsize(self):
         raise NotImplementedError("abstract base class")
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -858,6 +858,9 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def as_str(self):
+        return self.getslice(0, self.getlength(), 1, self.getlength())
+
     def getformat(self):
         return self.array.typecode
 
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -17,7 +17,7 @@
     getbytevalue, makebytesdata_w, newbytesdata_w)
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.buffer import BinaryBuffer
+from pypy.interpreter.buffer import SimpleBuffer, BinaryBuffer
 from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
 from pypy.objspace.std.stringmethods import _descr_getslice_slowpath
@@ -52,7 +52,7 @@
                            ''.join(self._data[self._offset:]))
 
     def buffer_w(self, space, flags):
-        return BytearrayBuffer(self)
+        return SimpleBuffer(BytearrayBuffer(self))
 
     def bytearray_list_of_chars_w(self, space):
         return self.getdata()
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -7,7 +7,7 @@
 from rpython.rlib.rstring import StringBuilder
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.buffer import StringBuffer
+from pypy.interpreter.buffer import SimpleBuffer, StringBuffer
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import (
     WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
@@ -419,7 +419,7 @@
 
     def buffer_w(self, space, flags):
         space.check_buf_flags(flags, True)
-        return StringBuffer(self._value)
+        return SimpleBuffer(StringBuffer(self._value))
 
     def readbuf_w(self, space):
         return StringBuffer(self._value)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -682,11 +682,7 @@
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
-        if memory_view_c_contiguous(space, self.flags):
-            return _array_to_hexstring(space, self.buf, 0, 1, self.getlength())
-        else:
-            bytes = self.as_str()
-            return _array_to_hexstring(space, StringBuffer(bytes), 0, 1, len(bytes))
+        return _array_to_hexstring(space, self.buf.as_binary(), 0, 1, self.getlength())
 
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'

From pypy.commits at gmail.com  Tue Mar 28 05:10:16 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 28 Mar 2017 02:10:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: add docstrings to struct module
Message-ID: <58da2878.43142e0a.daf5c.37cf@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: py3.5
Changeset: r90828:69340a250c18
Date: 2017-03-28 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/69340a250c18/

Log:	add docstrings to struct module

diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -40,6 +40,7 @@
         return space.text_w(w_input)
 
 def calcsize(space, w_format):
+    """Return size of C struct described by format string fmt."""
     format = text_or_bytes_w(space, w_format)
     return space.newint(_calcsize(space, format))
 
@@ -60,6 +61,7 @@
 
 
 def pack(space, w_format, args_w):
+    """Return string containing values v1, v2, ... packed according to fmt."""
     format = text_or_bytes_w(space, w_format)
     return do_pack(space, format, args_w)
 
@@ -69,6 +71,9 @@
 
 @unwrap_spec(offset=int)
 def pack_into(space, w_format, w_buffer, offset, args_w):
+    """ Pack the values v1, v2, ... according to fmt.
+Write the packed bytes into the writable buffer buf starting at offset
+    """
     format = text_or_bytes_w(space, w_format)
     return do_pack_into(space, format, w_buffer, offset, args_w)
 
@@ -108,6 +113,8 @@
 
 @unwrap_spec(offset=int)
 def unpack_from(space, w_format, w_buffer, offset=0):
+    """Unpack the buffer, containing packed C structure data, according to
+fmt, starting at offset. Requires len(buffer[offset:]) >= calcsize(fmt)."""
     format = text_or_bytes_w(space, w_format)
     return do_unpack_from(space, format, w_buffer, offset)
 

From pypy.commits at gmail.com  Tue Mar 28 05:19:59 2017
From: pypy.commits at gmail.com (mjacob)
Date: Tue, 28 Mar 2017 02:19:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Remove this test involving old style
 classes, which were removed in Python 3.0.
Message-ID: <58da2abf.904d190a.c2d59.3682@mx.google.com>

Author: Manuel Jacob 
Branch: 
Changeset: r90829:2b7815180e1b
Date: 2017-03-26 20:33 +0200
http://bitbucket.org/pypy/pypy/changeset/2b7815180e1b/

Log:	Remove this test involving old style classes, which were removed in
	Python 3.0.

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1223,25 +1223,3 @@
             pass
         bases = module.foo(C)
         assert bases == (A, B)
-
-    def test_multiple_inheritance_old_style_base(self):
-        module = self.import_extension('foo', [
-           ("foo", "METH_O",
-            '''
-                PyTypeObject *tp;
-                tp = (PyTypeObject*)args;
-                Py_INCREF(tp->tp_bases);
-                return tp->tp_bases;
-            '''
-            )])
-        # used to segfault after some iterations
-        for i in range(11):
-            print i
-            class A(object):
-                pass
-            class B:
-                pass
-            class C(A, B):
-                pass
-            bases = module.foo(C)
-            assert bases == (A, B)

From pypy.commits at gmail.com  Tue Mar 28 05:23:32 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 28 Mar 2017 02:23:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backed out changeset 2b7815180e1b
Message-ID: <58da2b94.c9d8190a.8f18f.360a@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r90830:7eb36df99c53
Date: 2017-03-28 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/7eb36df99c53/

Log:	Backed out changeset 2b7815180e1b

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1223,3 +1223,25 @@
             pass
         bases = module.foo(C)
         assert bases == (A, B)
+
+    def test_multiple_inheritance_old_style_base(self):
+        module = self.import_extension('foo', [
+           ("foo", "METH_O",
+            '''
+                PyTypeObject *tp;
+                tp = (PyTypeObject*)args;
+                Py_INCREF(tp->tp_bases);
+                return tp->tp_bases;
+            '''
+            )])
+        # used to segfault after some iterations
+        for i in range(11):
+            print i
+            class A(object):
+                pass
+            class B:
+                pass
+            class C(A, B):
+                pass
+            bases = module.foo(C)
+            assert bases == (A, B)

From pypy.commits at gmail.com  Tue Mar 28 05:23:34 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 28 Mar 2017 02:23:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: add docstrings to struct module
Message-ID: <58da2b96.d5522e0a.7f26e.38e1@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r90831:0777b28a3618
Date: 2017-03-28 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/0777b28a3618/

Log:	add docstrings to struct module

diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -34,10 +34,12 @@
 
 @unwrap_spec(format='text')
 def calcsize(space, format):
+    """Return size of C struct described by format string fmt."""
     return space.newint(_calcsize(space, format))
 
 
 def _pack(space, format, args_w):
+    """Return string containing values v1, v2, ... packed according to fmt."""
     if jit.isconstant(format):
         size = _calcsize(space, format)
     else:
@@ -60,6 +62,9 @@
 # XXX inefficient
 @unwrap_spec(format='text', offset=int)
 def pack_into(space, format, w_buffer, offset, args_w):
+    """ Pack the values v1, v2, ... according to fmt.
+Write the packed bytes into the writable buffer buf starting at offset
+    """
     res = _pack(space, format, args_w)
     buf = space.getarg_w('w*', w_buffer)
     if offset < 0:
@@ -91,6 +96,8 @@
 
 @unwrap_spec(format='text', offset=int)
 def unpack_from(space, format, w_buffer, offset=0):
+    """Unpack the buffer, containing packed C structure data, according to
+fmt, starting at offset. Requires len(buffer[offset:]) >= calcsize(fmt)."""
     size = _calcsize(space, format)
     buf = space.getarg_w('z*', w_buffer)
     if buf is None:

From pypy.commits at gmail.com  Tue Mar 28 07:00:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 04:00:05 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix for detect_sse4a() (which is not
 used outside this file)
Message-ID: <58da4235.0d50190a.ca227.3c03@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90832:73ea3144f995
Date: 2017-03-28 12:59 +0200
http://bitbucket.org/pypy/pypy/changeset/73ea3144f995/

Log:	Fix for detect_sse4a() (which is not used outside this file)

diff --git a/rpython/jit/backend/x86/detect_feature.py b/rpython/jit/backend/x86/detect_feature.py
--- a/rpython/jit/backend/x86/detect_feature.py
+++ b/rpython/jit/backend/x86/detect_feature.py
@@ -21,7 +21,7 @@
     return bool(code & (1<<25)) and bool(code & (1<<26))
 
 def cpu_id(eax = 1, ret_edx = True, ret_ecx = False):
-    asm = ["\xB8", chr(eax), "\x00\x00\x00", # MOV EAX, $eax
+    asm = ["\xB8", struct.pack("

Author: Nate Bragg 
Branch: lstrip_to_empty_string
Changeset: r90833:835b7eebd891
Date: 2017-03-28 07:07 -0400
http://bitbucket.org/pypy/pypy/changeset/835b7eebd891/

Log:	Failing test showing that lstrip doesn`t consume a whole string.

diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -459,6 +459,8 @@
             return const('    ').strip(' ')
         def left2():
             return const('a  ').strip(' ')
+        def leftall():
+            return const('!!').lstrip(const('!'))
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('ab')
         res = self.interpret(left, [])
@@ -469,6 +471,8 @@
         assert self.ll_to_string(res) == const('')
         res = self.interpret(left2, [])
         assert self.ll_to_string(res) == const('a')
+        res = self.interpret(leftall, [])
+        assert self.ll_to_string(res) == const('')
 
     def test_strip_multiple_chars(self):
         const = self.const
@@ -482,6 +486,8 @@
             return const(' \t\t   ').strip('\t ')
         def left2():
             return const('a  ').strip(' \t')
+        def leftall():
+            return const('!ab!').lstrip(const('!ab'))
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('b')
         res = self.interpret(left, [])
@@ -492,6 +498,8 @@
         assert self.ll_to_string(res) == const('')
         res = self.interpret(left2, [])
         assert self.ll_to_string(res) == const('a')
+        res = self.interpret(leftall, [])
+        assert self.ll_to_string(res) == const('')
 
     def test_upper(self):
         const = self.const

From pypy.commits at gmail.com  Tue Mar 28 07:26:26 2017
From: pypy.commits at gmail.com (Nate Bragg)
Date: Tue, 28 Mar 2017 04:26:26 -0700 (PDT)
Subject: [pypy-commit] pypy lstrip_to_empty_string: If possible,
 make lstrip consume the whole string.
Message-ID: <58da4862.41072e0a.280a1.3c7f@mx.google.com>

Author: Nate Bragg 
Branch: lstrip_to_empty_string
Changeset: r90834:2275619cea33
Date: 2017-03-28 07:10 -0400
http://bitbucket.org/pypy/pypy/changeset/2275619cea33/

Log:	If possible, make lstrip consume the whole string.

	Previously, it would not consume the final character even if it
	matched, because rpos pointed to the final index.

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -435,7 +435,7 @@
         lpos = 0
         rpos = s_len - 1
         if left:
-            while lpos < rpos and s.chars[lpos] == ch:
+            while lpos <= rpos and s.chars[lpos] == ch:
                 lpos += 1
         if right:
             while lpos < rpos + 1 and s.chars[rpos] == ch:
@@ -456,7 +456,7 @@
         lpos = 0
         rpos = s_len - 1
         if left:
-            while lpos < rpos and s.chars[lpos].isspace():
+            while lpos <= rpos and s.chars[lpos].isspace():
                 lpos += 1
         if right:
             while lpos < rpos + 1 and s.chars[rpos].isspace():
@@ -477,7 +477,7 @@
         lpos = 0
         rpos = s_len - 1
         if left:
-            while lpos < rpos and LLHelpers.ll_contains(s2, s.chars[lpos]):
+            while lpos <= rpos and LLHelpers.ll_contains(s2, s.chars[lpos]):
                 lpos += 1
         if right:
             while lpos < rpos + 1 and LLHelpers.ll_contains(s2, s.chars[rpos]):

From pypy.commits at gmail.com  Tue Mar 28 07:26:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 04:26:28 -0700 (PDT)
Subject: [pypy-commit] pypy lstrip_to_empty_string: ready to merge
Message-ID: <58da4864.5125190a.c3ff7.3ad7@mx.google.com>

Author: Armin Rigo 
Branch: lstrip_to_empty_string
Changeset: r90835:2ee78799ab18
Date: 2017-03-28 13:22 +0200
http://bitbucket.org/pypy/pypy/changeset/2ee78799ab18/

Log:	ready to merge


From pypy.commits at gmail.com  Tue Mar 28 07:26:30 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 04:26:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge lstrip_to_empty_string
Message-ID: <58da4866.1bd8190a.f2fe8.411d@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90836:2c39618c2b77
Date: 2017-03-28 13:22 +0200
http://bitbucket.org/pypy/pypy/changeset/2c39618c2b77/

Log:	hg merge lstrip_to_empty_string

	PR #531. Thanks Nate!

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -435,7 +435,7 @@
         lpos = 0
         rpos = s_len - 1
         if left:
-            while lpos < rpos and s.chars[lpos] == ch:
+            while lpos <= rpos and s.chars[lpos] == ch:
                 lpos += 1
         if right:
             while lpos < rpos + 1 and s.chars[rpos] == ch:
@@ -456,7 +456,7 @@
         lpos = 0
         rpos = s_len - 1
         if left:
-            while lpos < rpos and s.chars[lpos].isspace():
+            while lpos <= rpos and s.chars[lpos].isspace():
                 lpos += 1
         if right:
             while lpos < rpos + 1 and s.chars[rpos].isspace():
@@ -477,7 +477,7 @@
         lpos = 0
         rpos = s_len - 1
         if left:
-            while lpos < rpos and LLHelpers.ll_contains(s2, s.chars[lpos]):
+            while lpos <= rpos and LLHelpers.ll_contains(s2, s.chars[lpos]):
                 lpos += 1
         if right:
             while lpos < rpos + 1 and LLHelpers.ll_contains(s2, s.chars[rpos]):
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -459,6 +459,8 @@
             return const('    ').strip(' ')
         def left2():
             return const('a  ').strip(' ')
+        def leftall():
+            return const('!!').lstrip(const('!'))
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('ab')
         res = self.interpret(left, [])
@@ -469,6 +471,8 @@
         assert self.ll_to_string(res) == const('')
         res = self.interpret(left2, [])
         assert self.ll_to_string(res) == const('a')
+        res = self.interpret(leftall, [])
+        assert self.ll_to_string(res) == const('')
 
     def test_strip_multiple_chars(self):
         const = self.const
@@ -482,6 +486,8 @@
             return const(' \t\t   ').strip('\t ')
         def left2():
             return const('a  ').strip(' \t')
+        def leftall():
+            return const('!ab!').lstrip(const('!ab'))
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('b')
         res = self.interpret(left, [])
@@ -492,6 +498,8 @@
         assert self.ll_to_string(res) == const('')
         res = self.interpret(left2, [])
         assert self.ll_to_string(res) == const('a')
+        res = self.interpret(leftall, [])
+        assert self.ll_to_string(res) == const('')
 
     def test_upper(self):
         const = self.const

From pypy.commits at gmail.com  Tue Mar 28 07:26:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 04:26:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: uniformization
Message-ID: <58da4869.1c10190a.667bb.3bba@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90837:48d983a9e92a
Date: 2017-03-28 13:25 +0200
http://bitbucket.org/pypy/pypy/changeset/48d983a9e92a/

Log:	uniformization

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -438,7 +438,7 @@
             while lpos <= rpos and s.chars[lpos] == ch:
                 lpos += 1
         if right:
-            while lpos < rpos + 1 and s.chars[rpos] == ch:
+            while lpos <= rpos and s.chars[rpos] == ch:
                 rpos -= 1
         if rpos < lpos:
             return s.empty()
@@ -459,7 +459,7 @@
             while lpos <= rpos and s.chars[lpos].isspace():
                 lpos += 1
         if right:
-            while lpos < rpos + 1 and s.chars[rpos].isspace():
+            while lpos <= rpos and s.chars[rpos].isspace():
                 rpos -= 1
         if rpos < lpos:
             return s.empty()
@@ -480,7 +480,7 @@
             while lpos <= rpos and LLHelpers.ll_contains(s2, s.chars[lpos]):
                 lpos += 1
         if right:
-            while lpos < rpos + 1 and LLHelpers.ll_contains(s2, s.chars[rpos]):
+            while lpos <= rpos and LLHelpers.ll_contains(s2, s.chars[rpos]):
                 rpos -= 1
         if rpos < lpos:
             return s.empty()

From pypy.commits at gmail.com  Tue Mar 28 07:26:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 04:26:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: branch doc
Message-ID: <58da4882.93012e0a.d4816.3e6d@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90838:eb94f113091f
Date: 2017-03-28 13:26 +0200
http://bitbucket.org/pypy/pypy/changeset/eb94f113091f/

Log:	branch doc

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,4 +16,5 @@
 Fix missing tp_new on w_object called through multiple inheritance
 (will be part of release 5.7.1)
 
+.. branch: lstrip_to_empty_string
 

From pypy.commits at gmail.com  Tue Mar 28 09:14:52 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Mar 2017 06:14:52 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Stop abusing SubBuffer to represent
 memoryview slices
Message-ID: <58da61cc.d3d4190a.59f2.40a9@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90840:55b0a70b52e1
Date: 2017-03-28 14:14 +0100
http://bitbucket.org/pypy/pypy/changeset/55b0a70b52e1/

Log:	Stop abusing SubBuffer to represent memoryview slices

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -302,24 +302,14 @@
             else:
                 raise oefmt(space.w_NotImplementedError, "multi-dimensional sub-views are not implemented")
         elif is_slice:
-            mv = W_MemoryView.copy(self)
-            mv.init_slice(start, stop, step, slicelength, 0)
-            mv.init_len()
-            mv._init_flags()
-            return mv
+            return self.new_slice(start, stop, step, slicelength, 0)
         # multi index is handled at the top of this function
         else:
             raise TypeError("memoryview: invalid slice key")
 
-    def init_slice(self, start, stop, step, slicelength, dim):
-        # modifies the buffer, shape and stride to allow step to be > 1
-        self.strides = strides = self.getstrides()[:]
-        self.shape = shape = self.getshape()[:]
-        bytesize = self.getitemsize() * slicelength
-        self.buf = SubBuffer(self.buf, strides[dim] * start, bytesize)
-        shape[dim] = slicelength
-        strides[dim] = strides[dim] * step
-        # TODO subbuffer
+    def new_slice(self, start, stop, step, slicelength, dim):
+        sliced = BufferSlice(self.buf, start, step, slicelength)
+        return W_MemoryView(sliced, sliced.getformat(), sliced.getitemsize())
 
     def init_len(self):
         self.length = self.bytecount_from_shape()
@@ -678,7 +668,7 @@
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
-        return _array_to_hexstring(space, self.buf, 0, 1, self.getlength())
+        return _array_to_hexstring(space, self.buf.as_binary(), 0, 1, self.getlength())
 
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'
@@ -765,3 +755,65 @@
                 _IsFortranContiguous(ndim, shape, strides, itemsize))
     return 0
 
+class BufferSlice(Buffer):
+    def __init__(self, buf, start, step, length):
+        self.buf = buf
+        self.readonly = self.buf.readonly
+        self.strides = buf.getstrides()[:]
+        itemsize = buf.getitemsize()
+        self.offset = start * itemsize
+        self.step = step
+        self.strides[0] *= step
+        self.shape = buf.getshape()[:]
+        self.shape[0] = length
+
+    def getlength(self):
+        return self.shape[0] * self.getitemsize()
+
+    def as_str(self):
+        return self.getslice(0, self.getlength(), 1, self.getlength())
+
+    def as_str_and_offset_maybe(self):
+        string, offset = self.buf.as_str_and_offset_maybe()
+        if string is not None:
+            return string, offset + self.offset
+        return None, 0
+
+    def getitem(self, index):
+        return self.buf.getitem(self.offset + index)
+
+    def setitem(self, index, char):
+        self.buf.setitem(self.offset + index, char)
+
+    def getslice(self, start, stop, step, size):
+        if start == stop:
+            return ''     # otherwise, adding self.offset might make them
+                          # out of bounds
+        return self.buf.getslice(self.offset + start, self.offset + stop,
+                                    step, size)
+
+    def setslice(self, start, string):
+        if len(string) == 0:
+            return        # otherwise, adding self.offset might make 'start'
+                          # out of bounds
+        self.buf.setslice(self.offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        ptr = self.buf.get_raw_address()
+        return rffi.ptradd(ptr, self.offset)
+
+    def getformat(self):
+        return self.buf.getformat()
+
+    def getitemsize(self):
+        return self.buf.getitemsize()
+
+    def getndim(self):
+        return self.buf.getndim()
+
+    def getshape(self):
+        return self.shape
+
+    def getstrides(self):
+        return self.strides

From pypy.commits at gmail.com  Tue Mar 28 09:14:50 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Mar 2017 06:14:50 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Don't mutate the memoryview in
 _cast_to_1D()
Message-ID: <58da61ca.51a0190a.5733e.3f71@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90839:cd13570ce008
Date: 2017-03-28 02:25 +0100
http://bitbucket.org/pypy/pypy/changeset/cd13570ce008/

Log:	Don't mutate the memoryview in _cast_to_1D()

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -123,6 +123,50 @@
     def getstrides(self):
         return [1]
 
+class BufferView1D(Buffer):
+    def __init__(self, data, format, itemsize):
+        self.data = data
+        self.readonly = data.readonly
+        self.format = format
+        self.itemsize = itemsize
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def as_str(self):
+        return self.data.as_str()
+
+    def as_str_and_offset_maybe(self):
+        return self.data.as_str_and_offset_maybe()
+
+    def getitem(self, index):
+        return self.data.getitem(index)
+
+    def setitem(self, index, value):
+        return self.data.setitem(index, value)
+
+    def get_raw_address(self):
+        return self.data.get_raw_address()
+
+    def as_binary(self):
+        return self.data
+
+    def getformat(self):
+        return self.format
+
+    def getitemsize(self):
+        return self.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength() // self.itemsize]
+
+    def getstrides(self):
+        return [self.itemsize]
+
+
 class BinaryBuffer(Buffer):
     """Base class for buffers of bytes"""
     _attrs_ = ['readonly']
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -6,7 +6,7 @@
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib.rstruct.error import StructError
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.buffer import Buffer, SubBuffer, StringBuffer
+from pypy.interpreter.buffer import Buffer, SubBuffer, StringBuffer, BufferView1D
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
@@ -543,7 +543,6 @@
                         "memoryview: cannot casts view with"
                         " zeros in shape or strides")
 
-        itemsize = self.get_native_fmtchar(fmt)
         if w_shape:
             if not (space.isinstance_w(w_shape, space.w_list) or space.isinstance_w(w_shape, space.w_tuple)):
                 raise oefmt(space.w_TypeError, "expected list or tuple got %T", w_shape)
@@ -557,7 +556,7 @@
                             "memoryview: cast must be 1D -> ND or ND -> 1D")
 
         origfmt = self.getformat()
-        mv = self._cast_to_1D(space, buf, origfmt, fmt, itemsize)
+        mv = self._cast_to_1D(space, buf, origfmt, fmt)
         if w_shape:
             fview = space.fixedview(w_shape)
             shape = [space.int_w(w_obj) for w_obj in fview]
@@ -594,7 +593,8 @@
 
         self.flags = flags
 
-    def _cast_to_1D(self, space, buf, origfmt, fmt, itemsize):
+    def _cast_to_1D(self, space, buf, origfmt, fmt):
+        itemsize = self.get_native_fmtchar(fmt)
         if itemsize < 0:
             raise oefmt(space.w_ValueError, "memoryview: destination" \
                     " format must be a native single character format prefixed" \
@@ -614,12 +614,8 @@
         if not newfmt:
             raise oefmt(space.w_RuntimeError,
                     "memoryview: internal error")
-        mv = W_MemoryView(buf, newfmt, itemsize)
-        mv.ndim = 1
-        mv.shape = [buf.getlength() // itemsize]
-        mv.strides = [itemsize]
-        # XX suboffsets
-
+        newbuf = BufferView1D(buf, newfmt, itemsize)
+        mv = W_MemoryView(newbuf, newbuf.getformat(), newbuf.getitemsize())
         mv._init_flags()
         return mv
 
@@ -682,7 +678,7 @@
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring
         self._check_released(space)
-        return _array_to_hexstring(space, self.buf.as_binary(), 0, 1, self.getlength())
+        return _array_to_hexstring(space, self.buf, 0, 1, self.getlength())
 
 def is_byte_format(char):
     return char == 'b' or char == 'B' or char == 'c'

From pypy.commits at gmail.com  Tue Mar 28 09:40:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 06:40:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix inconsistencies in the
 xml.etree.ElementTree.Element class, which on
Message-ID: <58da67ba.12582e0a.15ce1.4454@mx.google.com>

Author: Armin Rigo 
Branch: py3.5
Changeset: r90841:0939e3a8a08d
Date: 2017-03-28 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/0939e3a8a08d/

Log:	Fix inconsistencies in the xml.etree.ElementTree.Element class,
	which on CPython is hidden by the C version from '_elementree'. Add
	test.

diff --git a/lib-python/3/test/test_xml_etree.py b/lib-python/3/test/test_xml_etree.py
--- a/lib-python/3/test/test_xml_etree.py
+++ b/lib-python/3/test/test_xml_etree.py
@@ -1878,6 +1878,17 @@
             with self.assertRaises(RuntimeError):
                 repr(e)  # Should not crash
 
+    def test_bad_find_returns_none(self):
+        # This behavior is the one we get historically when the C
+        # extension module is enabled.  With the Python version, it
+        # raised a TypeError instead.  There are projects out there
+        # that depend on the non-raising behavior, of course.
+        e = ET.Element('foo')
+        assert e.find('') is None
+        assert e.findall('') == []
+        assert e.findtext('') is None
+        assert e.findtext('', default="default.") == "default."
+
 class MutatingElementPath(str):
     def __new__(cls, elem, *args):
         self = str.__new__(cls, *args)
diff --git a/lib-python/3/xml/etree/ElementTree.py b/lib-python/3/xml/etree/ElementTree.py
--- a/lib-python/3/xml/etree/ElementTree.py
+++ b/lib-python/3/xml/etree/ElementTree.py
@@ -294,7 +294,14 @@
         Return the first matching element, or None if no element was found.
 
         """
-        return ElementPath.find(self, path, namespaces)
+        # Used to be:  return ElementPath.find(self, path, namespaces)
+        # but there are projects out there that rely on it getting None
+        # instead of an internal TypeError.  This is what the C version
+        # of this class does.
+        result = ElementPath.iterfind(self, path, namespaces)
+        if result is None:
+            return None
+        return next(result, None)
 
     def findtext(self, path, default=None, namespaces=None):
         """Find text for first matching element by tag name or path.
@@ -308,7 +315,17 @@
         content, the empty string is returned.
 
         """
-        return ElementPath.findtext(self, path, default, namespaces)
+        # Used to be:
+        #     return ElementPath.findtext(self, path, default, namespaces)
+        # See find().
+        result = ElementPath.iterfind(self, path, namespaces)
+        if result is None:
+            return default
+        try:
+            elem = next(result)
+            return elem.text or ""
+        except StopIteration:
+            return default
 
     def findall(self, path, namespaces=None):
         """Find all matching subelements by tag name or path.
@@ -319,7 +336,12 @@
         Returns list containing all matching elements in document order.
 
         """
-        return ElementPath.findall(self, path, namespaces)
+        # Used to be:  return ElementPath.findall(self, path, namespaces)
+        # See find().
+        result = ElementPath.iterfind(self, path, namespaces)
+        if result is None:
+            return []
+        return list(result)
 
     def iterfind(self, path, namespaces=None):
         """Find all matching subelements by tag name or path.

From pypy.commits at gmail.com  Tue Mar 28 09:43:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 06:43:41 -0700 (PDT)
Subject: [pypy-commit] cffi default: Pull request #79 by xwang
Message-ID: <58da688d.0fd2190a.d52e2.45ad@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r2923:9fe2a9e06094
Date: 2017-03-28 14:36 +0200
http://bitbucket.org/cffi/cffi/changeset/9fe2a9e06094/

Log:	Pull request #79 by xwang

	We can actually call PyThreadState_Delete(), which works without the
	GIL and seems to pass the same tests.

diff --git a/c/misc_thread_common.h b/c/misc_thread_common.h
--- a/c/misc_thread_common.h
+++ b/c/misc_thread_common.h
@@ -29,12 +29,7 @@
     struct cffi_tls_s *tls = (struct cffi_tls_s *)p;
 
     if (tls->local_thread_state != NULL) {
-        /* We need to re-acquire the GIL temporarily to free the
-           thread state.  I hope it is not a problem to do it in
-           a thread-local destructor.
-        */
-        PyEval_RestoreThread(tls->local_thread_state);
-        PyThreadState_DeleteCurrent();
+        PyThreadState_Delete(tls->local_thread_state);
     }
     free(tls);
 }

From pypy.commits at gmail.com  Tue Mar 28 09:54:05 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Mar 2017 06:54:05 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Simplify _cast_to_ND()
Message-ID: <58da6afd.5e532e0a.a35d5.424c@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90842:912d9dbe8d1f
Date: 2017-03-28 14:53 +0100
http://bitbucket.org/pypy/pypy/changeset/912d9dbe8d1f/

Log:	Simplify _cast_to_ND()

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -546,7 +546,8 @@
                             "memoryview: cast must be 1D -> ND or ND -> 1D")
 
         origfmt = self.getformat()
-        mv = self._cast_to_1D(space, buf, origfmt, fmt)
+        newbuf = self._cast_to_1D(space, buf, origfmt, fmt)
+        mv = W_MemoryView(newbuf, newbuf.getformat(), newbuf.getitemsize())
         if w_shape:
             fview = space.fixedview(w_shape)
             shape = [space.int_w(w_obj) for w_obj in fview]
@@ -604,10 +605,7 @@
         if not newfmt:
             raise oefmt(space.w_RuntimeError,
                     "memoryview: internal error")
-        newbuf = BufferView1D(buf, newfmt, itemsize)
-        mv = W_MemoryView(newbuf, newbuf.getformat(), newbuf.getitemsize())
-        mv._init_flags()
-        return mv
+        return BufferView1D(buf, newfmt, itemsize)
 
     def get_native_fmtstr(self, fmt):
         lenfmt = len(fmt)
@@ -637,33 +635,31 @@
         return None
 
     def _cast_to_ND(self, space, shape, ndim):
-        self.ndim = ndim
-        length = self.itemsize
-        if ndim == 0:
-            self.shape = []
-            self.strides = []
-        else:
-            self.shape = shape
-            for i in range(ndim):
-                length *= shape[i]
-            self._init_strides_from_shape()
-
+        buf = self.buf
+        length = itemsize = buf.getitemsize()
+        for i in range(ndim):
+            length *= shape[i]
         if length != self.buf.getlength():
             raise oefmt(space.w_TypeError,
                         "memoryview: product(shape) * itemsize != buffer size")
 
+        self.ndim = ndim
+        self.shape = shape
+        self.strides = self._strides_from_shape(shape, itemsize)
         self._init_flags()
 
-    def _init_strides_from_shape(self):
-        shape = self.getshape()
-        s = [0] * len(shape)
-        self.strides = s
-        ndim = self.getndim()
-        s[ndim-1] = self.itemsize
-        i = ndim-2
+    @staticmethod
+    def _strides_from_shape(shape, itemsize):
+        ndim = len(shape)
+        if ndim == 0:
+            return []
+        s = [0] * ndim
+        s[ndim - 1] = itemsize
+        i = ndim - 2
         while i >= 0:
             s[i] = s[i+1] * shape[i+1]
             i -= 1
+        return s
 
     def descr_hex(self, space):
         from pypy.objspace.std.bytearrayobject import _array_to_hexstring

From pypy.commits at gmail.com  Tue Mar 28 10:55:01 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Mar 2017 07:55:01 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Create BufferViewND to avoid mutating
 the memoryview in _cast_to_ND()
Message-ID: <58da7945.4a572e0a.c628d.4597@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90843:dbc12a5f29c5
Date: 2017-03-28 15:54 +0100
http://bitbucket.org/pypy/pypy/changeset/dbc12a5f29c5/

Log:	Create BufferViewND to avoid mutating the memoryview in
	_cast_to_ND()

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -547,11 +547,11 @@
 
         origfmt = self.getformat()
         newbuf = self._cast_to_1D(space, buf, origfmt, fmt)
-        mv = W_MemoryView(newbuf, newbuf.getformat(), newbuf.getitemsize())
         if w_shape:
             fview = space.fixedview(w_shape)
             shape = [space.int_w(w_obj) for w_obj in fview]
-            mv._cast_to_ND(space, shape, ndim)
+            newbuf = self._cast_to_ND(space, newbuf, shape, ndim)
+        mv = W_MemoryView(newbuf, newbuf.getformat(), newbuf.getitemsize())
         return mv
 
     def _init_flags(self):
@@ -634,19 +634,16 @@
 
         return None
 
-    def _cast_to_ND(self, space, shape, ndim):
-        buf = self.buf
+    def _cast_to_ND(self, space, buf, shape, ndim):
         length = itemsize = buf.getitemsize()
         for i in range(ndim):
             length *= shape[i]
-        if length != self.buf.getlength():
+        if length != buf.getlength():
             raise oefmt(space.w_TypeError,
                         "memoryview: product(shape) * itemsize != buffer size")
 
-        self.ndim = ndim
-        self.shape = shape
-        self.strides = self._strides_from_shape(shape, itemsize)
-        self._init_flags()
+        strides = self._strides_from_shape(shape, itemsize)
+        return BufferViewND(buf, ndim, shape, strides)
 
     @staticmethod
     def _strides_from_shape(shape, itemsize):
@@ -813,3 +810,55 @@
 
     def getstrides(self):
         return self.strides
+
+class BufferViewND(Buffer):
+    def __init__(self, parent, ndim, shape, strides):
+        assert parent.getndim() == 1
+        assert len(shape) == len(strides) == ndim
+        self.parent = parent
+        self.readonly = parent.readonly
+        self.ndim = ndim
+        self.shape = shape
+        self.strides = strides
+
+    def getlength(self):
+        return self.parent.getlength()
+
+    def as_str(self):
+        return self.parent.as_str()
+
+    def as_str_and_offset_maybe(self):
+        return self.parent.as_str_and_offset_maybe()
+
+    def as_binary(self):
+        return self.parent.as_binary()
+
+    def getitem(self, index):
+        return self.parent.getitem(index)
+
+    def setitem(self, index, char):
+        self.parent.setitem(index, char)
+
+    def getslice(self, start, stop, step, size):
+        return self.parent.getslice(start, stop, step, size)
+
+    def setslice(self, start, string):
+        self.parent.setslice(start, string)
+
+    def get_raw_address(self):
+        return self.parent.get_raw_address()
+
+    def getformat(self):
+        return self.parent.getformat()
+
+    def getitemsize(self):
+        return self.parent.getitemsize()
+
+    def getndim(self):
+        return self.ndim
+
+    def getshape(self):
+        return self.shape
+
+    def getstrides(self):
+        return self.strides

From pypy.commits at gmail.com  Tue Mar 28 11:21:31 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Mar 2017 08:21:31 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Kill copies of buffer attributes on
 W_MemoryObject
Message-ID: <58da7f7b.82ce190a.321cd.478f@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90844:74d97901f74e
Date: 2017-03-28 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/74d97901f74e/

Log:	Kill copies of buffer attributes on W_MemoryObject

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -84,7 +84,7 @@
     # Allow subclassing W_MemeoryView
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
     w_obj = space.allocate_instance(W_MemoryView, w_type)
-    w_obj.__init__(buf, itemsize=buf.itemsize)
+    w_obj.__init__(buf)
     track_reference(space, obj, w_obj)
     return w_obj
 
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -28,51 +28,27 @@
     an interp-level buffer.
     """
 
-    def __init__(self, buf, format=None, itemsize=1, ndim=-1,
-                 shape=None, strides=None, suboffsets=None):
+    def __init__(self, buf):
         assert isinstance(buf, Buffer)
         self.buf = buf
         self._hash = -1
-        # private copies of format, shape, itemsize, ... on this class
-        self.format = format
-        self.itemsize = itemsize
-        self.shape = shape
-        self.strides = strides
-        self.suboffsets = suboffsets
-        self.ndim = ndim
         self.flags = 0
-        self.length = -1
         self._init_flags()
 
-    # several fields are "overwritten" by the memory view (shape, strides, ...)
-    # thus use only those getter fields instead of directly accessing the fields
     def getndim(self):
-        if self.ndim == -1:
-            return self.buf.getndim()
-        return self.ndim
+        return self.buf.getndim()
 
     def getshape(self):
-        if self.shape is None:
-            return self.buf.getshape()
-        return self.shape
+        return self.buf.getshape()
 
     def getstrides(self):
-        if self.strides is None:
-            return self.buf.getstrides()
-        return self.strides
+        return self.buf.getstrides()
 
     def getitemsize(self):
-        return self.itemsize
+        return self.buf.getitemsize()
 
-    # memoryview needs to modify the field 'format', to prevent the modification
-    # of the buffer, we save the new format here!
     def getformat(self):
-        if self.format is None:
-            return self.buf.getformat()
-        return self.format
-
-    def setformat(self, value):
-        self.format = value
+        return self.buf.getformat()
 
     def buffer_w(self, space, flags):
         self._check_released(space)
@@ -85,7 +61,7 @@
             w_object._check_released(space)
             return W_MemoryView.copy(w_object)
         buf = space.buffer_w(w_object, space.BUF_FULL_RO)
-        return W_MemoryView(buf, buf.getformat(), buf.getitemsize())
+        return W_MemoryView(buf)
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
@@ -154,8 +130,6 @@
                 break
 
     def getlength(self):
-        if self.length != -1:
-            return self.length
         return self.buf.getlength()
 
     def descr_tobytes(self, space):
@@ -297,7 +271,7 @@
                     buf = SubBuffer(self.buf, idx, itemsize)
                     fmtiter = UnpackFormatIterator(space, buf)
                     fmtiter.length = buf.getlength()
-                    fmtiter.interpret(self.format)
+                    fmtiter.interpret(self.buf.getformat())
                     return fmtiter.result_w[0]
             else:
                 raise oefmt(space.w_NotImplementedError, "multi-dimensional sub-views are not implemented")
@@ -309,7 +283,7 @@
 
     def new_slice(self, start, stop, step, slicelength, dim):
         sliced = BufferSlice(self.buf, start, step, slicelength)
-        return W_MemoryView(sliced, sliced.getformat(), sliced.getitemsize())
+        return W_MemoryView(sliced)
 
     def init_len(self):
         self.length = self.bytecount_from_shape()
@@ -326,8 +300,7 @@
     def copy(view):
         # TODO suboffsets
         buf = view.buf
-        return W_MemoryView(buf, view.getformat(), view.getitemsize(),
-                            view.getndim(), view.getshape()[:], view.getstrides()[:])
+        return W_MemoryView(buf)
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
@@ -350,11 +323,11 @@
                 # TODO: this probably isn't very fast
                 fmtiter = PackFormatIterator(space, [w_obj], itemsize)
                 try:
-                    fmtiter.interpret(self.format)
+                    fmtiter.interpret(self.getformat())
                 except StructError as e:
                     raise oefmt(space.w_TypeError,
                                 "memoryview: invalid type for format '%s'",
-                                self.format)
+                                self.getformat())
                 self.buf.setslice(idx, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
@@ -551,7 +524,7 @@
             fview = space.fixedview(w_shape)
             shape = [space.int_w(w_obj) for w_obj in fview]
             newbuf = self._cast_to_ND(space, newbuf, shape, ndim)
-        mv = W_MemoryView(newbuf, newbuf.getformat(), newbuf.getitemsize())
+        mv = W_MemoryView(newbuf)
         return mv
 
     def _init_flags(self):
@@ -577,10 +550,9 @@
                                       itemsize, 'F'):
                 flags |= MEMORYVIEW_FORTRAN
 
-        if self.suboffsets:
+        if False:  # TODO missing suboffsets
             flags |= MEMORYVIEW_PIL
             flags &= ~(MEMORYVIEW_C|MEMORYVIEW_FORTRAN)
-        # TODO missing suboffsets
 
         self.flags = flags
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -361,7 +361,7 @@
         return W_SeqIterObject(w_obj)
 
     def newbuffer(self, w_obj, itemsize=1):
-        return W_MemoryView(w_obj, itemsize=itemsize)
+        return W_MemoryView(w_obj)
 
     def newbytes(self, s):
         assert isinstance(s, str)

From pypy.commits at gmail.com  Tue Mar 28 11:42:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 08:42:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix docs (thanks 'and2' on irc)
Message-ID: <58da8483.05452e0a.f1891.48a1@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90845:0e2af4d11b0c
Date: 2017-03-28 17:42 +0200
http://bitbucket.org/pypy/pypy/changeset/0e2af4d11b0c/

Log:	Fix docs (thanks 'and2' on irc)

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -185,7 +185,7 @@
 ::
 
     cd pypy/tool/release
-    ./package.py pypy-VER-PLATFORM
+    ./package.py --archive-name=pypy-VER-PLATFORM
 
 This creates a clean and prepared hierarchy, as well as a ``.tar.bz2``
 with the same content; both are found by default in

From pypy.commits at gmail.com  Tue Mar 28 11:56:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 08:56:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: oops, this is also RPython code
Message-ID: <58da87c8.1099190a.4b9a5.492e@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90846:f564555ae003
Date: 2017-03-28 17:56 +0200
http://bitbucket.org/pypy/pypy/changeset/f564555ae003/

Log:	oops, this is also RPython code

diff --git a/rpython/jit/backend/x86/detect_feature.py b/rpython/jit/backend/x86/detect_feature.py
--- a/rpython/jit/backend/x86/detect_feature.py
+++ b/rpython/jit/backend/x86/detect_feature.py
@@ -21,7 +21,11 @@
     return bool(code & (1<<25)) and bool(code & (1<<26))
 
 def cpu_id(eax = 1, ret_edx = True, ret_ecx = False):
-    asm = ["\xB8", struct.pack("> 8) & 0xff),
+                chr((eax >> 16) & 0xff),
+                chr((eax >> 24) & 0xff),
            "\x53",                     # PUSH EBX
            "\x0F\xA2",                 # CPUID
            "\x5B",                     # POP EBX

From pypy.commits at gmail.com  Tue Mar 28 12:24:19 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Mar 2017 09:24:19 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Simplify _cast_to_1D()
Message-ID: <58da8e33.0c2b190a.9dd7b.48fc@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90847:7aa0eac946fe
Date: 2017-03-28 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/7aa0eac946fe/

Log:	Simplify _cast_to_1D()

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -518,8 +518,7 @@
                 raise oefmt(space.w_TypeError,
                             "memoryview: cast must be 1D -> ND or ND -> 1D")
 
-        origfmt = self.getformat()
-        newbuf = self._cast_to_1D(space, buf, origfmt, fmt)
+        newbuf = self._cast_to_1D(space, buf, fmt)
         if w_shape:
             fview = space.fixedview(w_shape)
             shape = [space.int_w(w_obj) for w_obj in fview]
@@ -556,13 +555,14 @@
 
         self.flags = flags
 
-    def _cast_to_1D(self, space, buf, origfmt, fmt):
+    def _cast_to_1D(self, space, buf, fmt):
         itemsize = self.get_native_fmtchar(fmt)
         if itemsize < 0:
             raise oefmt(space.w_ValueError, "memoryview: destination" \
                     " format must be a native single character format prefixed" \
                     " with an optional '@'")
 
+        origfmt = buf.getformat()
         if self.get_native_fmtchar(origfmt) < 0 or \
            (not is_byte_format(fmt) and not is_byte_format(origfmt)):
             raise oefmt(space.w_TypeError,

From pypy.commits at gmail.com  Tue Mar 28 12:24:22 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Mar 2017 09:24:22 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Fix translation and give BufferView1D
 and BufferViewND a common base class
Message-ID: <58da8e36.93012e0a.d4816.4d05@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90848:afa6dad368e3
Date: 2017-03-28 17:23 +0100
http://bitbucket.org/pypy/pypy/changeset/afa6dad368e3/

Log:	Fix translation and give BufferView1D and BufferViewND a common base
	class

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -123,49 +123,6 @@
     def getstrides(self):
         return [1]
 
-class BufferView1D(Buffer):
-    def __init__(self, data, format, itemsize):
-        self.data = data
-        self.readonly = data.readonly
-        self.format = format
-        self.itemsize = itemsize
-
-    def getlength(self):
-        return self.data.getlength()
-
-    def as_str(self):
-        return self.data.as_str()
-
-    def as_str_and_offset_maybe(self):
-        return self.data.as_str_and_offset_maybe()
-
-    def getitem(self, index):
-        return self.data.getitem(index)
-
-    def setitem(self, index, value):
-        return self.data.setitem(index, value)
-
-    def get_raw_address(self):
-        return self.data.get_raw_address()
-
-    def as_binary(self):
-        return self.data
-
-    def getformat(self):
-        return self.format
-
-    def getitemsize(self):
-        return self.itemsize
-
-    def getndim(self):
-        return 1
-
-    def getshape(self):
-        return [self.getlength() // self.itemsize]
-
-    def getstrides(self):
-        return [self.itemsize]
-
 
 class BinaryBuffer(Buffer):
     """Base class for buffers of bytes"""
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -6,7 +6,7 @@
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib.rstruct.error import StructError
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.buffer import Buffer, SubBuffer, StringBuffer, BufferView1D
+from pypy.interpreter.buffer import Buffer, SubBuffer
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
@@ -721,6 +721,8 @@
     return 0
 
 class BufferSlice(Buffer):
+    _immutable_ = True
+    _attrs_ = ['buf', 'readonly', 'shape', 'strides', 'offset', 'step']
     def __init__(self, buf, start, step, length):
         self.buf = buf
         self.readonly = self.buf.readonly
@@ -783,15 +785,10 @@
     def getstrides(self):
         return self.strides
 
-class BufferViewND(Buffer):
-    def __init__(self, parent, ndim, shape, strides):
-        assert parent.getndim() == 1
-        assert len(shape) == len(strides) == ndim
-        self.parent = parent
-        self.readonly = parent.readonly
-        self.ndim = ndim
-        self.shape = shape
-        self.strides = strides
+
+class BufferViewBase(Buffer):
+    _immutable_ = True
+    _attrs_ = ['readonly', 'parent']
 
     def getlength(self):
         return self.parent.getlength()
@@ -802,14 +799,11 @@
     def as_str_and_offset_maybe(self):
         return self.parent.as_str_and_offset_maybe()
 
-    def as_binary(self):
-        return self.parent.as_binary()
-
     def getitem(self, index):
         return self.parent.getitem(index)
 
-    def setitem(self, index, char):
-        self.parent.setitem(index, char)
+    def setitem(self, index, value):
+        return self.parent.setitem(index, value)
 
     def getslice(self, start, stop, step, size):
         return self.parent.getslice(start, stop, step, size)
@@ -820,6 +814,47 @@
     def get_raw_address(self):
         return self.parent.get_raw_address()
 
+    def as_binary(self):
+        return self.parent.as_binary()
+
+class BufferView1D(BufferViewBase):
+    _immutable_ = True
+    _attrs_ = ['readonly', 'parent', 'format', 'itemsize']
+
+    def __init__(self, parent, format, itemsize):
+        self.parent = parent
+        self.readonly = parent.readonly
+        self.format = format
+        self.itemsize = itemsize
+
+    def getformat(self):
+        return self.format
+
+    def getitemsize(self):
+        return self.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength() // self.itemsize]
+
+    def getstrides(self):
+        return [self.itemsize]
+
+class BufferViewND(BufferViewBase):
+    _immutable_ = True
+    _attrs_ = ['readonly', 'parent', 'ndim', 'shape', 'strides']
+
+    def __init__(self, parent, ndim, shape, strides):
+        assert parent.getndim() == 1
+        assert len(shape) == len(strides) == ndim
+        self.parent = parent
+        self.readonly = parent.readonly
+        self.ndim = ndim
+        self.shape = shape
+        self.strides = strides
+
     def getformat(self):
         return self.parent.getformat()
 

From pypy.commits at gmail.com  Tue Mar 28 12:30:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 09:30:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2523
Message-ID: <58da8f99.690f190a.bf8eb.4f48@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r90849:f77701c9f570
Date: 2017-03-28 18:14 +0200
http://bitbucket.org/pypy/pypy/changeset/f77701c9f570/

Log:	Issue #2523

	This should fix it. The slotdefs entry was removed in 1b0451031b2e
	but we didn't have any test for that.

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -1032,6 +1032,7 @@
         TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
                wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"),
         TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""),
+        TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
         TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
                "x.__setattr__('name', value) <==> x.name = value"),
         TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -670,6 +670,34 @@
     return PyInt_FromLong(tf);
 }
 
+static PyTypeObject GetType1 = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "foo.GetType1",          /*tp_name*/
+    sizeof(PyObject),        /*tp_size*/
+};
+static PyTypeObject GetType2 = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "foo.GetType2",          /*tp_name*/
+    sizeof(PyObject),        /*tp_size*/
+};
+static PyObject *gettype1, *gettype2;
+
+static PyObject *gettype1_getattr(PyObject *self, char *name)
+{
+    char buf[200];
+    strcpy(buf, "getattr:");
+    strcat(buf, name);
+    return PyString_FromString(buf);
+}
+static PyObject *gettype2_getattro(PyObject *self, PyObject *name)
+{
+    char buf[200];
+    strcpy(buf, "getattro:");
+    strcat(buf, PyString_AS_STRING(name));
+    return PyString_FromString(buf);
+}
+
+
 /* List of functions exported by this module */
 
 static PyMethodDef foo_functions[] = {
@@ -765,6 +793,18 @@
     if (PyType_Ready(&TupleLike) < 0)
         INITERROR;
 
+    GetType1.tp_flags = Py_TPFLAGS_DEFAULT;
+    GetType1.tp_getattr = &gettype1_getattr;
+    if (PyType_Ready(&GetType1) < 0)
+        INITERROR;
+    gettype1 = PyObject_New(PyObject, &GetType1);
+
+    GetType2.tp_flags = Py_TPFLAGS_DEFAULT;
+    GetType2.tp_getattro = &gettype2_getattro;
+    if (PyType_Ready(&GetType2) < 0)
+        INITERROR;
+    gettype2 = PyObject_New(PyObject, &GetType2);
+
 
     d = PyModule_GetDict(module);
     if (d == NULL)
@@ -787,6 +827,10 @@
         INITERROR;
     if (PyDict_SetItemString(d, "TupleLike", (PyObject *) &TupleLike) < 0)
         INITERROR;
+    if (PyDict_SetItemString(d, "gettype1", gettype1) < 0)
+        INITERROR;
+    if (PyDict_SetItemString(d, "gettype2", gettype2) < 0)
+        INITERROR;
 #if PY_MAJOR_VERSION >=3
     return module;
 #endif
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1245,3 +1245,14 @@
                 pass
             bases = module.foo(C)
             assert bases == (A, B)
+
+    def test_getattr_getattro(self):
+        module = self.import_module(name='foo')
+        assert module.gettype2.dcba == 'getattro:dcba'
+        assert (type(module.gettype2).__getattribute__(module.gettype2, 'dcBA')
+            == 'getattro:dcBA')
+        assert module.gettype1.abcd == 'getattr:abcd'
+        # GetType1 objects have a __getattribute__ method, but this
+        # doesn't call tp_getattr at all, also on CPython
+        raises(AttributeError, type(module.gettype1).__getattribute__,
+                               module.gettype1, 'dcBA')

From pypy.commits at gmail.com  Tue Mar 28 13:12:48 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 28 Mar 2017 10:12:48 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Fix BufferSlice.as_str()
Message-ID: <58da9990.5125190a.c3ff7.4a65@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90850:6395c8c41478
Date: 2017-03-28 18:04 +0100
http://bitbucket.org/pypy/pypy/changeset/6395c8c41478/

Log:	Fix BufferSlice.as_str()

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -738,7 +738,8 @@
         return self.shape[0] * self.getitemsize()
 
     def as_str(self):
-        return self.getslice(0, self.getlength(), 1, self.getlength())
+        slicelen = self.shape[0]
+        return self.getslice(0, slicelen * self.step, self.step, slicelen)
 
     def as_str_and_offset_maybe(self):
         string, offset = self.buf.as_str_and_offset_maybe()

From pypy.commits at gmail.com  Tue Mar 28 15:23:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 28 Mar 2017 12:23:40 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Issue #2523
Message-ID: <58dab83c.5125190a.c3ff7.4f23@mx.google.com>

Author: Armin Rigo 
Branch: release-pypy2.7-5.x
Changeset: r90851:32cd8a5d7221
Date: 2017-03-28 22:17 +0300
http://bitbucket.org/pypy/pypy/changeset/32cd8a5d7221/

Log:	Issue #2523

	This should fix it. The slotdefs entry was removed in 1b0451031b2e
	but we didn't have any test for that. (grafted from
	f77701c9f5706a297821c43779219df4a0aa70b0)

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -1032,6 +1032,7 @@
         TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
                wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"),
         TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""),
+        TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
         TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
                "x.__setattr__('name', value) <==> x.name = value"),
         TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -670,6 +670,34 @@
     return PyInt_FromLong(tf);
 }
 
+static PyTypeObject GetType1 = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "foo.GetType1",          /*tp_name*/
+    sizeof(PyObject),        /*tp_size*/
+};
+static PyTypeObject GetType2 = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "foo.GetType2",          /*tp_name*/
+    sizeof(PyObject),        /*tp_size*/
+};
+static PyObject *gettype1, *gettype2;
+
+static PyObject *gettype1_getattr(PyObject *self, char *name)
+{
+    char buf[200];
+    strcpy(buf, "getattr:");
+    strcat(buf, name);
+    return PyString_FromString(buf);
+}
+static PyObject *gettype2_getattro(PyObject *self, PyObject *name)
+{
+    char buf[200];
+    strcpy(buf, "getattro:");
+    strcat(buf, PyString_AS_STRING(name));
+    return PyString_FromString(buf);
+}
+
+
 /* List of functions exported by this module */
 
 static PyMethodDef foo_functions[] = {
@@ -769,6 +797,18 @@
     if (PyType_Ready(&TupleLike) < 0)
         INITERROR;
 
+    GetType1.tp_flags = Py_TPFLAGS_DEFAULT;
+    GetType1.tp_getattr = &gettype1_getattr;
+    if (PyType_Ready(&GetType1) < 0)
+        INITERROR;
+    gettype1 = PyObject_New(PyObject, &GetType1);
+
+    GetType2.tp_flags = Py_TPFLAGS_DEFAULT;
+    GetType2.tp_getattro = &gettype2_getattro;
+    if (PyType_Ready(&GetType2) < 0)
+        INITERROR;
+    gettype2 = PyObject_New(PyObject, &GetType2);
+
 
     d = PyModule_GetDict(module);
     if (d == NULL)
@@ -791,6 +831,10 @@
         INITERROR;
     if (PyDict_SetItemString(d, "TupleLike", (PyObject *) &TupleLike) < 0)
         INITERROR;
+    if (PyDict_SetItemString(d, "gettype1", gettype1) < 0)
+        INITERROR;
+    if (PyDict_SetItemString(d, "gettype2", gettype2) < 0)
+        INITERROR;
 #if PY_MAJOR_VERSION >=3
     return module;
 #endif
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1245,3 +1245,14 @@
                 pass
             bases = module.foo(C)
             assert bases == (A, B)
+
+    def test_getattr_getattro(self):
+        module = self.import_module(name='foo')
+        assert module.gettype2.dcba == 'getattro:dcba'
+        assert (type(module.gettype2).__getattribute__(module.gettype2, 'dcBA')
+            == 'getattro:dcBA')
+        assert module.gettype1.abcd == 'getattr:abcd'
+        # GetType1 objects have a __getattribute__ method, but this
+        # doesn't call tp_getattr at all, also on CPython
+        raises(AttributeError, type(module.gettype1).__getattribute__,
+                               module.gettype1, 'dcBA')

From pypy.commits at gmail.com  Tue Mar 28 15:23:43 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 28 Mar 2017 12:23:43 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: increment release version
 to 5.7.1
Message-ID: <58dab83f.1b582e0a.32fd3.50f1@mx.google.com>

Author: Matti Picus 
Branch: release-pypy2.7-5.x
Changeset: r90852:7d660be66914
Date: 2017-03-28 22:20 +0300
http://bitbucket.org/pypy/pypy/changeset/7d660be66914/

Log:	increment release version to 5.7.1

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.13"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.7.0"
-#define PYPY_VERSION_NUM  0x05070000
+#define PYPY_VERSION "5.7.1"
+#define PYPY_VERSION_NUM  0x05070100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 7, 0, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 7, 1, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Tue Mar 28 15:23:47 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 28 Mar 2017 12:23:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: update repackage script,
 deprecate md5 and sha1
Message-ID: <58dab843.1a142e0a.f41a5.51df@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90854:b3ea13f4e6a1
Date: 2017-03-28 22:22 +0300
http://bitbucket.org/pypy/pypy/changeset/b3ea13f4e6a1/

Log:	update repackage script, deprecate md5 and sha1 (grafted from
	dcda4090780ddfcf2d27c45d88e355a16666c4b2)

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
 min=7
-rev=0
+rev=1
 branchname=release-pypy2.7-5.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
@@ -50,8 +50,8 @@
 rm -rf $rel-src
 
 # Print out the md5, sha1, sha256
-md5sum *.bz2 *.zip
-sha1sum *.bz2 *.zip
+#md5sum *.bz2 *.zip
+#sha1sum *.bz2 *.zip
 sha256sum *.bz2 *.zip
 
 # Now upload all the bz2 and zip

From pypy.commits at gmail.com  Tue Mar 28 15:23:45 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 28 Mar 2017 12:23:45 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: update repackage script,
 deprecate md5 and sha1
Message-ID: <58dab841.0a212e0a.4be63.534b@mx.google.com>

Author: Matti Picus 
Branch: release-pypy2.7-5.x
Changeset: r90853:dcda4090780d
Date: 2017-03-28 22:21 +0300
http://bitbucket.org/pypy/pypy/changeset/dcda4090780d/

Log:	update repackage script, deprecate md5 and sha1

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=6
-rev=0
+min=7
+rev=1
 branchname=release-pypy2.7-5.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
@@ -50,8 +50,8 @@
 rm -rf $rel-src
 
 # Print out the md5, sha1, sha256
-md5sum *.bz2 *.zip
-sha1sum *.bz2 *.zip
+#md5sum *.bz2 *.zip
+#sha1sum *.bz2 *.zip
 sha256sum *.bz2 *.zip
 
 # Now upload all the bz2 and zip

From pypy.commits at gmail.com  Tue Mar 28 16:36:11 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 28 Mar 2017 13:36:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: issue #2002: opening an sqlite3
 connection should add memory pressure
Message-ID: <58dac93b.51a0190a.5733e.50a8@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r90855:e866b37a3b0c
Date: 2017-03-28 22:34 +0200
http://bitbucket.org/pypy/pypy/changeset/e866b37a3b0c/

Log:	issue #2002: opening an sqlite3 connection should add memory
	pressure

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -31,10 +31,11 @@
 import weakref
 from threading import _get_ident as _thread_get_ident
 try:
-    from __pypy__ import newlist_hint
+    from __pypy__ import newlist_hint, add_memory_pressure
 except ImportError:
     assert '__pypy__' not in sys.builtin_module_names
     newlist_hint = lambda sizehint: []
+    add_memory_pressure = lambda size: None
 
 if sys.version_info[0] >= 3:
     StandardError = Exception
@@ -150,6 +151,9 @@
 def connect(database, timeout=5.0, detect_types=0, isolation_level="",
                  check_same_thread=True, factory=None, cached_statements=100):
     factory = Connection if not factory else factory
+    # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if
+    # backed by :memory: or a file)
+    add_memory_pressure(100 * 1024)
     return factory(database, timeout, detect_types, isolation_level,
                     check_same_thread, factory, cached_statements)
 

From pypy.commits at gmail.com  Tue Mar 28 17:29:39 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 28 Mar 2017 14:29:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: make our file's universal newline
 .readline implementation less ridiculous
Message-ID: <58dad5c3.1459190a.b5657.550c@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r90856:aa29d9052da9
Date: 2017-03-28 23:28 +0200
http://bitbucket.org/pypy/pypy/changeset/aa29d9052da9/

Log:	make our file's universal newline .readline implementation less
	ridiculous (still room for optimization left, but this fixes the
	worst issue)

diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -1032,11 +1032,13 @@
             # we can safely read without reading past an end-of-line
             startindex, peeked = self.base.peek()
             assert 0 <= startindex <= len(peeked)
-            pn = peeked.find("\n", startindex)
-            pr = peeked.find("\r", startindex)
-            if pn < 0: pn = len(peeked)
-            if pr < 0: pr = len(peeked)
-            c = self.read(min(pn, pr) - startindex + 1)
+            cl_or_lf_pos = len(peeked)
+            for i in range(startindex, len(peeked)):
+                ch = peeked[i]
+                if ch == '\n' or ch == '\r':
+                    cl_or_lf_pos = i
+                    break
+            c = self.read(cl_or_lf_pos - startindex + 1)
             if not c:
                 break
             result.append(c)

From pypy.commits at gmail.com  Wed Mar 29 04:10:52 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 29 Mar 2017 01:10:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: Remove stray word in whatsnew.
Message-ID: <58db6c0c.d357190a.13852.65ee@mx.google.com>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r90857:0b53e4713936
Date: 2017-03-29 10:09 +0200
http://bitbucket.org/pypy/pypy/changeset/0b53e4713936/

Log:	Remove stray word in whatsnew.

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,8 +7,8 @@
 
 Add cpyext interfaces for ``PyModule_New``
 
-Correctly handle `dict.pop`` where the ``pop``ping 
-key is not the same type as the ``dict``'s and ``pop`` 
+Correctly handle `dict.pop`` where the ``pop``
+key is not the same type as the ``dict``'s and ``pop``
 is called with a default (will be part of release 5.7.1)
 
 .. branch: issue2522

From pypy.commits at gmail.com  Wed Mar 29 06:36:53 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 29 Mar 2017 03:36:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: issue #2186: implement the missing
 string concatenation optimization in
Message-ID: <58db8e45.12582e0a.15ce1.6de2@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r90858:d52a4942fee0
Date: 2017-03-29 12:34 +0200
http://bitbucket.org/pypy/pypy/changeset/d52a4942fee0/

Log:	issue #2186: implement the missing string concatenation optimization
	in cpython_differences

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -357,6 +357,24 @@
 
 .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of
 
+Performance Differences
+-------------------------
+
+CPython has an optimization that can make repeated string concatenation not
+quadratic. For example, this kind of code runs in O(n) time::
+
+    s = ''
+    for string in mylist:
+        s += string
+
+In PyPy, this code will always have quadratic complexity. Note also, that the
+CPython optimization is brittle and can break by having slight variations in
+your code anyway. So you should anyway replace the code with::
+
+    parts = []
+    for string in mylist:
+        parts.append(string)
+    s = "".join(parts)
 
 Miscellaneous
 -------------

From pypy.commits at gmail.com  Wed Mar 29 07:38:31 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 29 Mar 2017 04:38:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: add docstrings to some of the __pypy__
 function
Message-ID: <58db9cb7.0a502e0a.f8cf9.6d46@mx.google.com>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r90859:762a7454ce1c
Date: 2017-03-29 13:26 +0200
http://bitbucket.org/pypy/pypy/changeset/762a7454ce1c/

Log:	add docstrings to some of the __pypy__ function

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -6,6 +6,8 @@
 
 
 class BuildersModule(MixedModule):
+    """ Module containing string and unicode builders """
+
     appleveldefs = {}
 
     interpleveldefs = {
@@ -34,6 +36,8 @@
 
 
 class IntOpModule(MixedModule):
+    """ Module for integer operations that have two-complement overflow
+    behaviour instead of overflowing to longs """
     appleveldefs = {}
     interpleveldefs = {
         'int_add':         'interp_intop.int_add',
@@ -55,6 +59,8 @@
 
 
 class Module(MixedModule):
+    """ PyPy specific "magic" functions. A lot of them are experimental and
+    subject to change, many are internal. """
     appleveldefs = {
     }
 
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -123,13 +123,15 @@
         ])
 
 @unwrap_spec(sizehint=int)
-def resizelist_hint(space, w_iterable, sizehint):
-    if not isinstance(w_iterable, W_ListObject):
+def resizelist_hint(space, w_list, sizehint):
+    """ Reallocate the underlying storage of the argument list to sizehint """
+    if not isinstance(w_list, W_ListObject):
         raise oefmt(space.w_TypeError, "arg 1 must be a 'list'")
-    w_iterable._resize_hint(sizehint)
+    w_list._resize_hint(sizehint)
 
 @unwrap_spec(sizehint=int)
 def newlist_hint(space, sizehint):
+    """ Create a new empty list that has an underlying storage of length sizehint """
     return space.newlist_hint(sizehint)
 
 @unwrap_spec(debug=bool)
@@ -141,6 +143,9 @@
 
 @unwrap_spec(estimate=int)
 def add_memory_pressure(estimate):
+    """ Add memory pressure of estimate bytes. Useful when calling a C function
+    that internally allocates a big chunk of memory. This instructs the GC to
+    garbage collect sooner than it would otherwise."""
     rgc.add_memory_pressure(estimate)
 
 @unwrap_spec(w_frame=PyFrame)

From pypy.commits at gmail.com  Wed Mar 29 11:11:12 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 29 Mar 2017 08:11:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge vmprof-native,
 I thought I have already done that before
Message-ID: <58dbce90.12582e0a.15ce1.7a04@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90860:d1a9d22b323c
Date: 2017-03-29 11:10 -0400
http://bitbucket.org/pypy/pypy/changeset/d1a9d22b323c/

Log:	merge vmprof-native, I thought I have already done that before

diff too long, truncating to 2000 out of 48309 lines

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -79,6 +79,9 @@
 _ssl
     libssl
 
+_vmprof
+    libunwind (optional, loaded dynamically at runtime)
+
 Make sure to have these libraries (with development headers) installed
 before building PyPy, otherwise the resulting binary will not contain
 these modules.  Furthermore, the following libraries should be present
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -50,8 +50,8 @@
     return OperationError(w_VMProfError, space.newtext(e.msg))
 
 
- at unwrap_spec(fileno=int, period=float)
-def enable(space, fileno, period):
+ at unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int)
+def enable(space, fileno, period, memory, lines, native):
     """Enable vmprof.  Writes go to the given 'fileno', a file descriptor
     opened for writing.  *The file descriptor must remain open at least
     until disable() is called.*
@@ -65,7 +65,7 @@
     #                             "with vmprof will crash"),
     #               space.w_RuntimeWarning)
     try:
-        rvmprof.enable(fileno, period)
+        rvmprof.enable(fileno, period, memory, native)
     except rvmprof.VMProfError as e:
         raise VMProfError(space, e)
 
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -24,10 +24,11 @@
             i += 5 * WORD # header
             assert s[i    ] == '\x05'    # MARKER_HEADER
             assert s[i + 1] == '\x00'    # 0
-            assert s[i + 2] == '\x02'    # VERSION_THREAD_ID
-            assert s[i + 3] == chr(4)    # len('pypy')
-            assert s[i + 4: i + 8] == 'pypy'
-            i += 8
+            assert s[i + 2] == '\x06'    # VERSION_TIMESTAMP
+            assert s[i + 3] == '\x08'    # PROFILE_RPYTHON
+            assert s[i + 4] == chr(4)    # len('pypy')
+            assert s[i + 5: i + 9] == 'pypy'
+            i += 9
             while i < len(s):
                 if s[i] == '\x03':
                     break
@@ -41,6 +42,17 @@
                     _, size = struct.unpack("ll", s[i:i + 2 * WORD])
                     count += 1
                     i += 2 * WORD + size
+                elif s[i] == '\x06':
+                    print(s[i:i+24])
+                    i += 1+8+8+8
+                elif s[i] == '\x07':
+                    i += 1
+                    # skip string
+                    size, = struct.unpack("l", s[i:i + WORD])
+                    i += WORD+size
+                    # skip string
+                    size, = struct.unpack("l", s[i:i + WORD])
+                    i += WORD+size
                 else:
                     raise AssertionError(ord(s[i]))
             return count
@@ -48,7 +60,7 @@
         import _vmprof
         gc.collect()  # try to make the weakref list deterministic
         gc.collect()  # by freeing all dead code objects
-        _vmprof.enable(tmpfileno, 0.01)
+        _vmprof.enable(tmpfileno, 0.01, 0, 0, 0)
         _vmprof.disable()
         s = open(self.tmpfilename, 'rb').read()
         no_of_codes = count(s)
@@ -61,7 +73,7 @@
 
         gc.collect()
         gc.collect()
-        _vmprof.enable(tmpfileno2, 0.01)
+        _vmprof.enable(tmpfileno2, 0.01, 0, 0, 0)
 
         exec """def foo2():
             pass
@@ -76,9 +88,9 @@
 
     def test_enable_ovf(self):
         import _vmprof
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 0)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, -2.5)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 0, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, -2.5, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300, 0, 0, 0)
         NaN = (1e300*1e300) / (1e300*1e300)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN, 0, 0, 0)
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -43,7 +43,7 @@
 }
 
 
-""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read(), include_dirs=[str(srcdir)])
+""" + open(str(srcdir.join("shared/vmprof_get_custom_offset.h"))).read(), include_dirs=[str(srcdir)])
 
 class TestDirect(object):
     def test_infrastructure(self):
diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -5,10 +5,14 @@
 
 
 cwd = py.path.local(__file__).dirpath()
+rvmp = cwd.join('../../..')
+rvmp = rvmp.join('rpython/rlib/rvmprof/src')
+
 eci = ExternalCompilationInfo(
     includes=[cwd.join('faulthandler.h')],
-    include_dirs=[str(cwd), cdir],
-    separate_module_files=[cwd.join('faulthandler.c')])
+    include_dirs=[str(cwd), cdir, rvmp],
+    separate_module_files=[cwd.join('faulthandler.c')],
+    compile_extra=['-DRPYTHON_VMPROF=1'])
 
 eci_later = eci.merge(ExternalCompilationInfo(
     pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n']))
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -244,6 +244,10 @@
     return inner
 
 def oopspec(spec):
+    """ The JIT compiler won't look inside this decorated function,
+        but instead during translation, rewrites it according to the handler in
+        rpython/jit/codewriter/jtransform.py.
+    """
     def decorator(func):
         func.oopspec = spec
         return func
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -32,8 +32,8 @@
         return code._vmprof_unique_id
     return 0
 
-def enable(fileno, interval):
-    _get_vmprof().enable(fileno, interval)
+def enable(fileno, interval, memory=0, native=0):
+    _get_vmprof().enable(fileno, interval, memory, native)
 
 def disable():
     _get_vmprof().disable()
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -14,32 +14,64 @@
 
 ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof')
 SRC = ROOT.join('src')
+SHARED = SRC.join('shared')
+BACKTRACE = SHARED.join('libbacktrace')
 
+compile_extra = ['-DRPYTHON_VMPROF', '-O3']
 if sys.platform.startswith('linux'):
+    separate_module_files = [
+       BACKTRACE.join('backtrace.c'),
+       BACKTRACE.join('state.c'),
+       BACKTRACE.join('elf.c'),
+       BACKTRACE.join('dwarf.c'),
+       BACKTRACE.join('fileline.c'),
+       BACKTRACE.join('mmap.c'),
+       BACKTRACE.join('mmapio.c'),
+       BACKTRACE.join('posix.c'),
+       BACKTRACE.join('sort.c'),
+    ]
     _libs = ['dl']
+    compile_extra += ['-DVMPROF_UNIX']
+    compile_extra += ['-DVMPROF_LINUX']
+elif sys.platform == 'darwin':
+    compile_extra += ['-DVMPROF_UNIX']
+    compile_extra += ['-DVMPROF_MAC']
+    separate_module_files = []
+    _libs = []
 else:
+    # windows
+    separate_module_files = []
     _libs = []
+
 eci_kwds = dict(
-    include_dirs = [SRC],
-    includes = ['rvmprof.h', 'vmprof_stack.h'],
+    include_dirs = [SRC, SHARED, BACKTRACE],
+    includes = ['rvmprof.h','vmprof_stack.h'],
     libraries = _libs,
-    separate_module_files = [SRC.join('rvmprof.c')],
-    post_include_bits=['#define RPYTHON_VMPROF\n'],
+    separate_module_files = [
+        SRC.join('rvmprof.c'),
+        SHARED.join('compat.c'),
+        SHARED.join('machine.c'),
+        SHARED.join('symboltable.c'),
+        SHARED.join('vmp_stack.c'),
+    ] + separate_module_files,
+    post_include_bits=[],
+    compile_extra=compile_extra
     )
 global_eci = ExternalCompilationInfo(**eci_kwds)
 
 
 def setup():
-    compile_extra = ['-DRPYTHON_LL2CTYPES']
+    eci_kwds['compile_extra'].append('-DRPYTHON_LL2CTYPES')
     platform.verify_eci(ExternalCompilationInfo(
-        compile_extra=compile_extra,
-        **eci_kwds))
+                        **eci_kwds))
 
     eci = global_eci
     vmprof_init = rffi.llexternal("vmprof_init",
-                                  [rffi.INT, rffi.DOUBLE, rffi.CCHARP],
+                                  [rffi.INT, rffi.DOUBLE, rffi.INT, rffi.INT,
+                                   rffi.CCHARP, rffi.INT],
                                   rffi.CCHARP, compilation_info=eci)
-    vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT,
+    vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.INT, rffi.INT],
+                                    rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
     vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
@@ -62,6 +94,7 @@
     return CInterface(locals())
 
 
+
 class CInterface(object):
     def __init__(self, namespace):
         for k, v in namespace.iteritems():
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -10,6 +10,8 @@
 
 MAX_FUNC_NAME = 1023
 
+PLAT_WINDOWS = sys.platform == 'win32'
+
 # ____________________________________________________________
 
 # keep in sync with vmprof_stack.h
@@ -122,7 +124,7 @@
         self._gather_all_code_objs = gather_all_code_objs
 
     @jit.dont_look_inside
-    def enable(self, fileno, interval):
+    def enable(self, fileno, interval, memory=0, native=0):
         """Enable vmprof.  Writes go to the given 'fileno'.
         The sampling interval is given by 'interval' as a number of
         seconds, as a float which must be smaller than 1.0.
@@ -132,12 +134,16 @@
         if self.is_enabled:
             raise VMProfError("vmprof is already enabled")
 
-        p_error = self.cintf.vmprof_init(fileno, interval, "pypy")
+        if PLAT_WINDOWS:
+            native = 0 # force disabled on Windows
+        lines = 0 # not supported on PyPy currently
+
+        p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native)
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
         self._gather_all_code_objs()
-        res = self.cintf.vmprof_enable()
+        res = self.cintf.vmprof_enable(memory, native)
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
@@ -154,6 +160,7 @@
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
 
+
     def _write_code_registration(self, uid, name):
         assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, (
             "the name must be 'class:func_name:func_line:filename' "
@@ -171,6 +178,23 @@
     arguments given to the decorated function.
 
     'result_class' is ignored (backward compatibility).
+
+    ====================================
+    TRANSLATION NOTE CALL THIS ONLY ONCE
+    ====================================
+
+    This function can only be called once during translation.
+    It generates a C function called __vmprof_eval_vmprof which is used by
+    the vmprof C source code and is bound as an extern function.
+    This is necessary while walking the native stack.
+    If you see __vmprof_eval_vmprof defined twice during
+    translation, read on:
+
+    To remove this restriction do the following:
+
+    *) Extend the macro IS_VMPROF_EVAL in the vmprof source repo to check several
+       sybmols.
+    *) Give each function provided to this decorator a unique symbol name in C
     """
     if _hack_update_stack_untranslated:
         from rpython.rtyper.annlowlevel import llhelper
@@ -198,16 +222,27 @@
             unique_id = get_code_fn(*args)._vmprof_unique_id
             unique_id = rffi.cast(lltype.Signed, unique_id)
             # ^^^ removes the "known non-negative" hint for annotation
+            #
+            # Signals can occur at the two places (1) and (2), that will
+            # have added a stack entry, but the function __vmprof_eval_vmprof
+            # is not entered. This behaviour will swallow one Python stack frame
+            #
+            # Current fix: vmprof will discard this sample. (happens
+            # very infrequently)
+            #
             if not jit.we_are_jitted():
                 x = enter_code(unique_id)
+                # (1) signal here
                 try:
                     return func(*args)
                 finally:
+                    # (2) signal here
                     leave_code(x)
             else:
                 return decorated_jitted_function(unique_id, *args)
 
         decorated_function.__name__ = func.__name__ + '_rvmprof'
+        decorated_function.c_name = '__vmprof_eval_vmprof'
         return decorated_function
 
     return decorate
@@ -216,7 +251,6 @@
 def _was_registered(CodeClass):
     return hasattr(CodeClass, '_vmprof_unique_id')
 
-
 _vmprof_instance = None
 
 @specialize.memo()
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -2,25 +2,30 @@
 
 #ifdef RPYTHON_LL2CTYPES
    /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
-#ifndef RPY_EXTERN
-#define RPY_EXTERN RPY_EXPORTED
-#endif
-#ifdef _WIN32
-#define RPY_EXPORTED __declspec(dllexport)
-#else
-#define RPY_EXPORTED  extern __attribute__((visibility("default")))
-#endif
 
 #else
 #  include "common_header.h"
 #  include "structdef.h"
 #  include "src/threadlocal.h"
 #  include "rvmprof.h"
+#  include "forwarddecl.h"
 #endif
 
 
-#if defined(__unix__) || defined(__APPLE__)
-#include "vmprof_main.h"
+
+#include "shared/vmprof_get_custom_offset.h"
+#ifdef VMPROF_UNIX
+#include "shared/vmprof_main.h"
 #else
-#include "vmprof_main_win32.h"
+#include "shared/vmprof_main_win32.h"
 #endif
+
+
+#ifdef RPYTHON_LL2CTYPES
+int IS_VMPROF_EVAL(void * ptr) { return 0; }
+#else
+int IS_VMPROF_EVAL(void * ptr)
+{
+    return ptr == __vmprof_eval_vmprof;
+}
+#endif
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,12 +1,30 @@
-#ifdef _WIN32
-typedef long intptr_t;
+#pragma once
+
+#include "shared/vmprof.h"
+
+#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int))
+
+#ifdef VMPROF_WINDOWS
+#include "msiinttypes/inttypes.h"
+#include "msiinttypes/stdint.h"
 #else
-# include 
+#include 
+#include 
 #endif
 
-RPY_EXTERN char *vmprof_init(int, double, char *);
+#ifndef RPY_EXTERN
+#define RPY_EXTERN RPY_EXPORTED
+#endif
+#ifdef _WIN32
+#define RPY_EXPORTED __declspec(dllexport)
+#else
+#define RPY_EXPORTED  extern __attribute__((visibility("default")))
+#endif
+
+RPY_EXTERN char *vmprof_init(int fd, double interval, int memory,
+                     int lines, const char *interp_name, int native);
 RPY_EXTERN void vmprof_ignore_signals(int);
-RPY_EXTERN int vmprof_enable(void);
+RPY_EXTERN int vmprof_enable(int memory, int native);
 RPY_EXTERN int vmprof_disable(void);
 RPY_EXTERN int vmprof_register_virtual_function(char *, long, int);
 RPY_EXTERN void* vmprof_stack_new(void);
@@ -15,4 +33,7 @@
 RPY_EXTERN void vmprof_stack_free(void*);
 RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t);
 
+long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
+                                      intptr_t addr, int max_depth);
+
 #define RVMPROF_TRACEBACK_ESTIMATE_N(num_entries)  (2 * (num_entries) + 4)
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -0,0 +1,371 @@
+/*[clinic input]
+module _vmprof
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b443489e38f2be7d]*/
+
+#define _GNU_SOURCE 1
+
+#include 
+#include 
+#include 
+
+#include "_vmprof.h"
+
+static volatile int is_enabled = 0;
+static destructor Original_code_dealloc = 0;
+static PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
+void dump_native_symbols(int fileno);
+
+#if VMPROF_UNIX
+#include "trampoline.h"
+#include "machine.h"
+#include "symboltable.h"
+#include "vmprof_main.h"
+#else
+#include "vmprof_main_win32.h"
+#endif
+#include "vmp_stack.h"
+
+#ifdef VMPROF_UNIX
+#ifdef __clang__
+__attribute__((optnone))
+#elif defined(__GNUC__)
+__attribute__((optimize("O1")))
+#endif
+PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag)
+{
+#ifdef X86_64
+    register PY_STACK_FRAME_T * callee_saved asm("rbx");
+#elif defined(X86_32)
+    register PY_STACK_FRAME_T * callee_saved asm("edi");
+#else
+#    error "platform not supported"
+#endif
+
+    asm volatile(
+#ifdef X86_64
+        "movq %1, %0\t\n"
+#elif defined(X86_32)
+        "mov %1, %0\t\n"
+#else
+#    error "platform not supported"
+#endif
+        : "=r" (callee_saved)
+        : "r" (f) );
+    return _default_eval_loop(f, throwflag);
+}
+#endif
+
+static int emit_code_object(PyCodeObject *co)
+{
+    char buf[MAX_FUNC_NAME + 1];
+    char *co_name, *co_filename;
+    int co_firstlineno;
+    int sz;
+#if PY_MAJOR_VERSION >= 3
+    co_name = PyUnicode_AsUTF8(co->co_name);
+    if (co_name == NULL)
+        return -1;
+    co_filename = PyUnicode_AsUTF8(co->co_filename);
+    if (co_filename == NULL)
+        return -1;
+#else
+    co_name = PyString_AS_STRING(co->co_name);
+    co_filename = PyString_AS_STRING(co->co_filename);
+#endif
+    co_firstlineno = co->co_firstlineno;
+
+    sz = snprintf(buf, MAX_FUNC_NAME / 2, "py:%s", co_name);
+    if (sz < 0) sz = 0;
+    if (sz > MAX_FUNC_NAME / 2) sz = MAX_FUNC_NAME / 2;
+    snprintf(buf + sz, MAX_FUNC_NAME / 2, ":%d:%s", co_firstlineno,
+             co_filename);
+    return vmprof_register_virtual_function(buf, CODE_ADDR_TO_UID(co), 500000);
+}
+
+static int _look_for_code_object(PyObject *o, void *all_codes)
+{
+    if (PyCode_Check(o) && !PySet_Contains((PyObject *)all_codes, o)) {
+        Py_ssize_t i;
+        PyCodeObject *co = (PyCodeObject *)o;
+        if (emit_code_object(co) < 0)
+            return -1;
+        if (PySet_Add((PyObject *)all_codes, o) < 0)
+            return -1;
+
+        /* as a special case, recursively look for and add code
+           objects found in the co_consts.  The problem is that code
+           objects are not created as GC-aware in CPython, so we need
+           to hack like this to hope to find most of them. 
+        */
+        i = PyTuple_Size(co->co_consts);
+        while (i > 0) {
+            --i;
+            if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i),
+                                      all_codes) < 0)
+                return -1;
+        }
+    }
+    return 0;
+}
+
+static void emit_all_code_objects(void)
+{
+    PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL;
+    Py_ssize_t i, size;
+
+    gc_module = PyImport_ImportModuleNoBlock("gc");
+    if (gc_module == NULL)
+        goto error;
+
+    lst = PyObject_CallMethod(gc_module, "get_objects", "");
+    if (lst == NULL || !PyList_Check(lst))
+        goto error;
+
+    all_codes = PySet_New(NULL);
+    if (all_codes == NULL)
+        goto error;
+
+    size = PyList_GET_SIZE(lst);
+    for (i = 0; i < size; i++) {
+        PyObject *o = PyList_GET_ITEM(lst, i);
+        if (o->ob_type->tp_traverse &&
+            o->ob_type->tp_traverse(o, _look_for_code_object, (void *)all_codes)
+                < 0)
+            goto error;
+    }
+
+ error:
+    Py_XDECREF(all_codes);
+    Py_XDECREF(lst);
+    Py_XDECREF(gc_module);
+}
+
+static void cpyprof_code_dealloc(PyObject *co)
+{
+    if (is_enabled) {
+        emit_code_object((PyCodeObject *)co);
+        /* xxx error return values are ignored */
+    }
+    Original_code_dealloc(co);
+}
+
+
+void dump_native_symbols(int fileno)
+{
+    PyObject * mod = NULL;
+
+    mod = PyImport_ImportModuleNoBlock("vmprof");
+    if (mod == NULL)
+        goto error;
+
+    PyObject_CallMethod(mod, "dump_native_symbols", "(l)", fileno);
+
+error:
+    Py_XDECREF(mod);
+}
+
+
+
+static PyObject *enable_vmprof(PyObject* self, PyObject *args)
+{
+    int fd;
+    int memory = 0;
+    int lines = 0;
+    int native = 0;
+    double interval;
+    char *p_error;
+
+    if (!PyArg_ParseTuple(args, "id|iii", &fd, &interval, &memory, &lines, &native)) {
+        return NULL;
+    }
+    assert(fd >= 0 && "file descripter provided to vmprof must not" \
+                      " be less then zero.");
+
+    if (is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is already enabled");
+        return NULL;
+    }
+
+    vmp_profile_lines(lines);
+
+    if (!Original_code_dealloc) {
+        Original_code_dealloc = PyCode_Type.tp_dealloc;
+        PyCode_Type.tp_dealloc = &cpyprof_code_dealloc;
+    }
+
+    p_error = vmprof_init(fd, interval, memory, lines, "cpython", native);
+    if (p_error) {
+        PyErr_SetString(PyExc_ValueError, p_error);
+        return NULL;
+    }
+
+    if (vmprof_enable(memory, native) < 0) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    is_enabled = 1;
+
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static PyObject *
+disable_vmprof(PyObject *module, PyObject *noarg)
+{
+    if (!is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
+        return NULL;
+    }
+    is_enabled = 0;
+    vmprof_ignore_signals(1);
+    emit_all_code_objects();
+
+    if (vmprof_disable() < 0) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+    if (PyErr_Occurred())
+        return NULL;
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static PyObject *
+write_all_code_objects(PyObject *module, PyObject *noargs)
+{
+    if (!is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
+        return NULL;
+    }
+    emit_all_code_objects();
+    if (PyErr_Occurred())
+        return NULL;
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+
+static PyObject *
+sample_stack_now(PyObject *module, PyObject * args)
+{
+    PyThreadState * tstate = NULL;
+    PyObject * list = NULL;
+    int i;
+    int entry_count;
+    void ** m;
+    void * routine_ip;
+    long skip = 0;
+
+    // stop any signal to occur
+    vmprof_ignore_signals(1);
+
+    list = PyList_New(0);
+    if (list == NULL) {
+        goto error;
+    }
+
+    if (!PyArg_ParseTuple(args, "l", &skip)) {
+        goto error;
+    }
+
+    tstate = PyGILState_GetThisThreadState();
+    m = (void**)malloc(SINGLE_BUF_SIZE);
+    if (m == NULL) {
+        PyErr_SetString(PyExc_MemoryError, "could not allocate buffer for stack trace");
+        vmprof_ignore_signals(0);
+        return NULL;
+    }
+    entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, skip, 0);
+
+    for (i = 0; i < entry_count; i++) {
+        routine_ip = m[i];
+        PyList_Append(list, PyLong_NEW((ssize_t)routine_ip));
+    }
+
+    free(m);
+
+    Py_INCREF(list);
+
+    vmprof_ignore_signals(0);
+    return list;
+error:
+    Py_DECREF(list);
+    Py_INCREF(Py_None);
+
+    vmprof_ignore_signals(0);
+    return Py_None;
+}
+
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+static PyObject *
+resolve_addr(PyObject *module, PyObject *args) {
+    long long addr;
+    PyObject * o_name = NULL;
+    PyObject * o_lineno = NULL;
+    PyObject * o_srcfile = NULL;
+    char name[128];
+    int lineno = 0;
+    char srcfile[256];
+
+    if (!PyArg_ParseTuple(args, "L", &addr)) {
+        return NULL;
+    }
+    name[0] = '\x00';
+    srcfile[0] = '-';
+    srcfile[1] = '\x00';
+    if (vmp_resolve_addr((void*)addr, name, 128, &lineno, srcfile, 256) != 0) {
+        goto error;
+    }
+
+    o_name = PyStr_NEW(name);
+    if (o_name == NULL) goto error;
+    o_lineno = PyLong_NEW(lineno);
+    if (o_lineno == NULL) goto error;
+    o_srcfile = PyStr_NEW(srcfile);
+    if (o_srcfile == NULL) goto error;
+    //
+    return PyTuple_Pack(3, o_name, o_lineno, o_srcfile);
+error:
+    Py_XDECREF(o_name);
+    Py_XDECREF(o_lineno);
+    Py_XDECREF(o_srcfile);
+
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+#endif
+
+static PyMethodDef VMProfMethods[] = {
+    {"enable",  enable_vmprof, METH_VARARGS, "Enable profiling."},
+    {"disable", disable_vmprof, METH_NOARGS, "Disable profiling."},
+    {"write_all_code_objects", write_all_code_objects, METH_NOARGS,
+     "Write eagerly all the IDs of code objects"},
+    {"sample_stack_now", sample_stack_now, METH_VARARGS, "Sample the stack now"},
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
+    {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"},
+#endif
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef VmprofModule = {
+    PyModuleDef_HEAD_INIT,
+    "_vmprof",
+    "",  // doc
+    -1,  // size
+    VMProfMethods
+};
+
+PyMODINIT_FUNC PyInit__vmprof(void)
+{
+    return PyModule_Create(&VmprofModule);
+}
+#else
+PyMODINIT_FUNC init_vmprof(void)
+{
+    Py_InitModule("_vmprof", VMProfMethods);
+}
+#endif
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.h b/rpython/rlib/rvmprof/src/shared/_vmprof.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.h
@@ -0,0 +1,46 @@
+#pragma once
+
+#include "vmprof.h"
+
+#ifdef VMPROF_WINDOWS
+#include "msiinttypes/inttypes.h"
+#include "msiinttypes/stdint.h"
+#else
+#include 
+#include 
+#include 
+#endif
+
+/**
+ * This whole setup is very strange. There was just one C file called
+ * _vmprof.c which included all *.h files to copy code. Unsure what
+ * the goal was with this design, but I assume it just 'GREW'
+ *
+ * Thus I'm (plan_rich) slowly trying to separate this. *.h files
+ * should not have complex implementations (all of them currently have them)
+ */
+
+
+#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int))
+
+#define ROUTINE_IS_PYTHON(RIP) ((unsigned long long)RIP & 0x1) == 0
+#define ROUTINE_IS_C(RIP) ((unsigned long long)RIP & 0x1) == 1
+
+/* This returns the address of the code object
+   as the identifier.  The mapping from identifiers to string
+   representations of the code object is done elsewhere, namely:
+
+   * If the code object dies while vmprof is enabled,
+     PyCode_Type.tp_dealloc will emit it.  (We don't handle nicely
+     for now the case where several code objects are created and die
+     at the same memory address.)
+
+   * When _vmprof.disable() is called, then we look around the
+     process for code objects and emit all the ones that we can
+     find (which we hope is very close to 100% of them).
+*/
+#define CODE_ADDR_TO_UID(co)  (((intptr_t)(co)))
+
+#define CPYTHON_HAS_FRAME_EVALUATION PY_VERSION_HEX >= 0x30600B0
+
+int vmp_write_all(const char *buf, size_t bufsize);
diff --git a/rpython/rlib/rvmprof/src/shared/compat.c b/rpython/rlib/rvmprof/src/shared/compat.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/compat.c
@@ -0,0 +1,140 @@
+#include "compat.h"
+
+#include 
+#include 
+#if VMPROF_WINDOWS
+#define WIN32_LEAN_AND_MEAN
+#include 
+#else
+#include 
+#include 
+#endif
+
+static int _vmp_profile_fileno = -1;
+
+int vmp_profile_fileno(void) {
+    return _vmp_profile_fileno;
+}
+void vmp_set_profile_fileno(int fileno) {
+    _vmp_profile_fileno = fileno;
+}
+
+#ifndef VMPROF_WINDOWS
+int vmp_write_all(const char *buf, size_t bufsize)
+{
+    ssize_t count;
+    if (_vmp_profile_fileno == -1) {
+        return -1;
+    }
+    while (bufsize > 0) {
+        count = write(_vmp_profile_fileno, buf, bufsize);
+        if (count <= 0)
+            return -1;   /* failed */
+        buf += count;
+        bufsize -= count;
+    }
+    return 0;
+}
+#endif
+
+int vmp_write_meta(const char * key, const char * value)
+{
+    char marker = MARKER_META;
+    long x = (long)strlen(key);
+    vmp_write_all(&marker, 1);
+    vmp_write_all((char*)&x, sizeof(long));
+    vmp_write_all(key, x);
+    x = (long)strlen(value);
+    vmp_write_all((char*)&x, sizeof(long));
+    vmp_write_all(value, x);
+    return 0;
+}
+
+/**
+ * Write the time and zone now.
+ */
+
+struct timezone_buf {
+    int64_t tv_sec;
+    int64_t tv_usec;
+};
+#define __SIZE (1+sizeof(struct timezone_buf)+8)
+
+#ifdef VMPROF_UNIX
+int vmp_write_time_now(int marker) {
+    char buffer[__SIZE];
+    struct timezone_buf buf;
+
+    (void)memset(&buffer, 0, __SIZE);
+
+    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
+           "marker must be either a trailer or time_n_zone!");
+
+    struct timeval tv;
+    time_t now;
+    struct tm tm;
+
+
+    /* copy over to the struct */
+    if (gettimeofday(&tv, NULL) != 0) {
+        return -1;
+    }
+    if (time(&now) == (time_t)-1) {
+        return -1;
+    }
+    if (localtime_r(&now, &tm) == NULL) {
+        return -1;
+    }
+    buf.tv_sec = tv.tv_sec;
+    buf.tv_usec = tv.tv_usec;
+    strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
+
+    buffer[0] = marker;
+    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
+    vmp_write_all(buffer, __SIZE);
+    return 0;
+}
+#endif
+
+#ifdef VMPROF_WINDOWS
+int vmp_write_time_now(int marker) {
+    char buffer[__SIZE];
+    struct timezone_buf buf;
+
+    /**
+     * http://stackoverflow.com/questions/10905892/equivalent-of-gettimeday-for-windows
+     */
+
+    // Note: some broken versions only have 8 trailing zero's, the correct
+    // epoch has 9 trailing zero's
+    static const uint64_t EPOCH = ((uint64_t) 116444736000000000ULL);
+
+    SYSTEMTIME  system_time;
+    FILETIME    file_time;
+    uint64_t    time;
+
+    (void)memset(&buffer, 0, __SIZE);
+
+    assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \
+           "marker must be either a trailer or time_n_zone!");
+
+
+    GetSystemTime( &system_time );
+    SystemTimeToFileTime( &system_time, &file_time );
+    time =  ((uint64_t)file_time.dwLowDateTime )      ;
+    time += ((uint64_t)file_time.dwHighDateTime) << 32;
+
+    buf.tv_sec = ((time - EPOCH) / 10000000L);
+    buf.tv_usec = (system_time.wMilliseconds * 1000);
+
+    // time zone not implemented on windows
+    memset(((char*)buffer)+__SIZE-8, 0, 8);
+    (void)memcpy(((char*)buffer)+__SIZE-8, "UTC", 3);
+
+    buffer[0] = marker;
+    (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
+    vmp_write_all(buffer, __SIZE);
+    return 0;
+}
+#endif
+#undef __SIZE
diff --git a/rpython/rlib/rvmprof/src/shared/compat.h b/rpython/rlib/rvmprof/src/shared/compat.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/compat.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include "vmprof.h"
+
+#ifndef RPYTHON_VMPROF
+#  if PY_MAJOR_VERSION >= 3
+      #define PyStr_AS_STRING PyBytes_AS_STRING
+      #define PyStr_GET_SIZE PyBytes_GET_SIZE
+      #define PyStr_NEW      PyUnicode_FromString
+      #define PyLong_NEW     PyLong_FromSsize_t
+#  else
+      #define PyStr_AS_STRING PyString_AS_STRING
+      #define PyStr_GET_SIZE PyString_GET_SIZE
+      #define PyStr_NEW      PyString_FromString
+      #define PyLong_NEW     PyInt_FromSsize_t
+      #define PyLong_AsLong  PyInt_AsLong
+#  endif
+#endif
+
+int vmp_write_all(const char *buf, size_t bufsize);
+int vmp_write_time_now(int marker);
+int vmp_write_meta(const char * key, const char * value);
+
+int vmp_profile_fileno(void);
+void vmp_set_profile_fileno(int fileno);
diff --git a/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog b/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog
@@ -0,0 +1,602 @@
+2017-01-01  Jakub Jelinek  
+
+	Update copyright years.
+
+2016-11-15  Matthias Klose  
+
+	* configure: Regenerate.
+
+2016-09-11  Carlos Liam  
+
+	* all: Remove meaningless trailing whitespace.
+
+2016-05-18  Uros Bizjak  
+
+	PR target/71161
+	* elf.c (phdr_callback) [__i386__]: Add
+	__attribute__((__force_align_arg_pointer__)).
+
+2016-03-02  Maxim Ostapenko  
+
+	* elf.c (backtrace_initialize): Properly initialize elf_fileline_fn to
+	avoid possible crash.
+	(elf_add): Don't set *fileline_fn to elf_nodebug value in case of
+	missing debug info anymore.
+
+2016-02-06  John David Anglin  
+
+	* mmap.c (MAP_FAILED): Define if not defined.
+
+2016-01-04  Jakub Jelinek  
+
+	Update copyright years.
+
+2015-12-18  Andris Pavenis  
+
+	* configure.ac: Specify that DJGPP do not have mmap
+	even when sys/mman.h exists.
+	* configure: Regenerate
+
+2015-12-09  John David Anglin  
+
+	PR libgfortran/68115
+	* configure.ac: Set libbacktrace_cv_sys_sync to no on hppa*-*-hpux*.
+	* configure: Regenerate.
+	* elf.c (backtrace_initialize): Cast __sync_bool_compare_and_swap call
+	to void.
+
+2015-09-17  Ian Lance Taylor  
+
+	* posix.c (backtrace_open): Cast second argument of open() to int.
+
+2015-09-11  Ian Lance Taylor  
+
+	* Makefile.am (backtrace.lo): Depend on internal.h.
+	(sort.lo, stest.lo): Add explicit dependencies.
+	* Makefile.in: Rebuild.
+
+2015-09-09  Hans-Peter Nilsson  
+
+	* backtrace.c: #include .
+
+2015-09-08  Ian Lance Taylor  
+
+	PR other/67457
+	* backtrace.c: #include "internal.h".
+	(struct backtrace_data): Add can_alloc field.
+	(unwind): If can_alloc is false, don't try to get file/line
+	information.
+	(backtrace_full): Set can_alloc field in bdata.
+	* alloc.c (backtrace_alloc): Don't call error_callback if it is
+	NULL.
+	* mmap.c (backtrace_alloc): Likewise.
+	* internal.h: Update comments for backtrace_alloc and
+	backtrace_free.
+
+2015-09-08  Ian Lance Taylor  
+
+	PR other/67457
+	* mmap.c (backtrace_alloc): Correct test for mmap failure.
+
+2015-08-31  Ulrich Weigand  
+
+	* configure.ac: For spu-*-* targets, set have_fcntl to no.
+	* configure: Regenerate.
+
+2015-08-27  Ulrich Weigand  
+
+	* configure.ac: Remove [disable-shared] argument to LT_INIT.
+	Remove setting PIC_FLAG when building as target library.
+	* configure: Regenerate.
+
+2015-08-26  Hans-Peter Nilsson  
+
+	* configure.ac: Only compile with -fPIC if the target
+	supports it.
+	* configure: Regenerate.
+
+2015-08-24  Ulrich Weigand  
+
+	* configure.ac: Set have_mmap to no on spu-*-* targets.
+	* configure: Regenerate.
+
+2015-08-13  Ian Lance Taylor  
+
+	* dwarf.c (read_function_entry): Add vec_inlined parameter.
+	Change all callers.
+
+2015-06-11  Martin Sebor  
+
+	PR sanitizer/65479
+	* dwarf.c (struct line): Add new field idx.
+	(line_compare): Use it.
+	(add_line): Set it.
+	(read_line_info): Reset it.
+
+2015-05-29  Tristan Gingold  
+
+	* pecoff.c: New file.
+	* Makefile.am (FORMAT_FILES): Add pecoff.c and dependencies.
+	* Makefile.in: Regenerate.
+	* filetype.awk: Detect pecoff.
+	* configure.ac: Define BACKTRACE_SUPPORTS_DATA on elf platforms.
+	Add pecoff.
+	* btest.c (test5): Test enabled only if BACKTRACE_SUPPORTS_DATA is
+	true.
+	* backtrace-supported.h.in (BACKTRACE_SUPPORTS_DATA): Define.
+	* configure: Regenerate.
+	* pecoff.c: New file.
+
+2015-05-13  Michael Haubenwallner  
+
+	* Makefile.in: Regenerated with automake-1.11.6.
+	* aclocal.m4: Likewise.
+	* configure: Likewise.
+
+2015-01-24  Matthias Klose  
+
+	* configure.ac: Move AM_ENABLE_MULTILIB before AC_PROG_CC.
+	* configure: Regenerate.
+
+2015-01-05  Jakub Jelinek  
+
+	Update copyright years.
+
+2014-11-21  H.J. Lu  
+
+	PR bootstrap/63784
+	* configure: Regenerated.
+
+2014-11-11  David Malcolm  
+
+	* ChangeLog.jit: New.
+
+2014-11-11  Francois-Xavier Coudert  
+
+	PR target/63610
+	* configure: Regenerate.
+
+2014-10-23  Ian Lance Taylor  
+
+	* internal.h (backtrace_atomic_load_pointer) [no atomic or sync]:
+	Fix to return void *.
+
+2014-05-08  Ian Lance Taylor  
+
+	* mmap.c (backtrace_free): If freeing a large aligned block of
+	memory, call munmap rather than holding onto it.
+	(backtrace_vector_grow): When growing a vector, double the number
+	of pages requested.  When releasing the old version of a grown
+	vector, pass the correct size to backtrace_free.
+
+2014-03-07  Ian Lance Taylor  
+
+	* sort.c (backtrace_qsort): Use middle element as pivot.
+
+2014-03-06  Ian Lance Taylor  
+
+	* sort.c: New file.
+	* stest.c: New file.
+	* internal.h (backtrace_qsort): Declare.
+	* dwarf.c (read_abbrevs): Call backtrace_qsort instead of qsort.
+	(read_line_info, read_function_entry): Likewise.
+	(read_function_info, build_dwarf_data): Likewise.
+	* elf.c (elf_initialize_syminfo): Likewise.
+	* Makefile.am (libbacktrace_la_SOURCES): Add sort.c.
+	(stest_SOURCES, stest_LDADD): Define.
+	(check_PROGRAMS): Add stest.
+
+2014-02-07  Misty De Meo  
+
+	PR target/58710
+	* configure.ac: Use AC_LINK_IFELSE in check for
+	_Unwind_GetIPInfo.
+	* configure: Regenerate.
+
+2014-01-02  Richard Sandiford  
+
+	Update copyright years
+
+2013-12-06  Jakub Jelinek  
+
+	* elf.c (ET_DYN): Undefine and define again.
+	(elf_add): Add exe argument, if true and ehdr.e_type is ET_DYN,
+	return early -1 without closing the descriptor.
+	(struct phdr_data): Add exe_descriptor.
+	(phdr_callback): If pd->exe_descriptor is not -1, for very first
+	call if dlpi_name is NULL just call elf_add with the exe_descriptor,
+	otherwise backtrace_close the exe_descriptor if not -1.  Adjust
+	call to elf_add.
+	(backtrace_initialize): Adjust call to elf_add.  If it returns
+	-1, set pd.exe_descriptor to descriptor, otherwise set it to -1.
+
+2013-12-05  Ian Lance Taylor  
+
+	* alloc.c (backtrace_vector_finish): Add error_callback and data
+	parameters.  Call backtrace_vector_release.  Return address base.
+	* mmap.c (backtrace_vector_finish): Add error_callback and data
+	parameters.  Return address base.
+	* dwarf.c (read_function_info): Get new address base from
+	backtrace_vector_finish.
+	* internal.h (backtrace_vector_finish): Update declaration.
+
+2013-11-27  Ian Lance Taylor  
+
+	* dwarf.c (find_address_ranges): New static function, broken out
+	of build_address_map.
+	(build_address_map): Call it.
+	* btest.c (check): Check for missing filename or function, rather
+	than crashing.
+	(f3): Check that enough frames were returned.
+
+2013-11-19  Jakub Jelinek  
+
+	* backtrace.h (backtrace_syminfo_callback): Add symsize argument.
+	* elf.c (elf_syminfo): Pass 0 or sym->size to the callback as
+	last argument.
+	* btest.c (struct symdata): Add size field.
+	(callback_three): Add symsize argument.  Copy it to the data->size
+	field.
+	(f23): Set symdata.size to 0.
+	(test5): Likewise.  If sizeof (int) > 1, lookup address of
+	((uintptr_t) &global) + 1.  Verify symdata.val and symdata.size
+	values.
+
+	* atomic.c: Include sys/types.h.
+
+2013-11-18  Ian Lance Taylor  
+
+	* configure.ac: Check for support of __atomic extensions.
+	* internal.h: Declare or #define atomic functions for use in
+	backtrace code.
+	* atomic.c: New file.
+	* dwarf.c (dwarf_lookup_pc): Use atomic functions.
+	(dwarf_fileline, backtrace_dwarf_add): Likewise.
+	* elf.c (elf_add_syminfo_data, elf_syminfo): Likewise.
+	(backtrace_initialize): Likewise.
+	* fileline.c (fileline_initialize): Likewise.
+	* Makefile.am (libbacktrace_la_SOURCES): Add atomic.c.
+	* configure, config.h.in, Makefile.in: Rebuild.
+
+2013-11-18  Jakub Jelinek  
+
+	* elf.c (SHN_UNDEF): Define.
+	(elf_initialize_syminfo): Add base_address argument.  Ignore symbols
+	with st_shndx == SHN_UNDEF.  Add base_address to address fields.
+	(elf_add): Adjust caller.
+
+	* elf.c (phdr_callback): Process info->dlpi_addr == 0 normally.
+
+2013-11-16  Ian Lance Taylor  
+
+	* backtrace.h (backtrace_create_state): Correct comment about
+	threading.
+
+2013-11-15  Ian Lance Taylor  
+
+	* backtrace.h (backtrace_syminfo): Update comment and parameter
+	name to take any address, not just a PC value.
+	* elf.c (STT_OBJECT): Define.
+	(elf_nosyms): Rename parameter pc to addr.
+	(elf_symbol_search): Rename local variable pc to addr.
+	(elf_initialize_syminfo): Add STT_OBJECT symbols to elf_symbols.
+	(elf_syminfo): Rename parameter pc to addr.
+	* btest.c (global): New global variable.
+	(test5): New test.
+	(main): Call test5.
+
+2013-10-17  Ian Lance Taylor  
+
+	* elf.c (elf_add): Don't get the wrong offsets if a debug section
+	is missing.
+
+2013-10-15  David Malcolm  
+
+	* configure.ac: Add --enable-host-shared, setting up
+	pre-existing PIC_FLAG variable within Makefile.am et al.
+	* configure: Regenerate.
+
+2013-09-20  Alan Modra  
+
+	* configure: Regenerate.
+
+2013-07-23  Alexander Monakov  
+
+	* elf.c (elf_syminfo): Loop over the elf_syminfo_data chain.
+
+2013-07-23  Alexander Monakov  
+
+	* elf.c (backtrace_initialize): Pass elf_fileline_fn to
+	dl_iterate_phdr callbacks.
+
+2013-03-25  Ian Lance Taylor  
+
+	* alloc.c: #include .
+	* mmap.c: Likewise.
+
+2013-01-31  Ian Lance Taylor  
+
+	* dwarf.c (read_function_info): Permit fvec parameter to be NULL.
+	(dwarf_lookup_pc): Don't use ddata->fvec if threaded.
+
+2013-01-25  Jakub Jelinek  
+
+	PR other/56076
+	* dwarf.c (read_line_header): Don't crash if DW_AT_comp_dir
+	attribute was not seen.
+
+2013-01-16  Ian Lance Taylor  
+
+	* dwarf.c (struct unit): Add filename and abs_filename fields.
+	(build_address_map): Set new fields when reading unit.
+	(dwarf_lookup_pc): If we don't find an entry in the line table,
+	just return the main file name.
+
+2013-01-14  Richard Sandiford  
+
+	Update copyright years.
+
+2013-01-01  Ian Lance Taylor  
+
+	PR bootstrap/54834
+	* Makefile.am (AM_CPPFLAGS): Remove -I ../gcc/include and -I
+	$(MULTIBUILDTOP)/../../gcc/include.
+	* Makefile.in: Rebuild.
+
+2013-01-01  Ian Lance Taylor  
+
+	PR other/55536
+	* mmap.c (backtrace_alloc): Don't call sync functions if not
+	threaded.
+	(backtrace_free): Likewise.
+
+2012-12-12  John David Anglin  
+
+	* mmapio.c: Define MAP_FAILED if not defined.
+
+2012-12-11  Jakub Jelinek  
+
+	PR bootstrap/54926
+	* Makefile.am (AM_CFLAGS): Remove -frandom-seed=$@.
+	* configure.ac: If --with-target-subdir, add -frandom-seed=$@
+	to EXTRA_FLAGS unconditionally, otherwise check whether the compiler
+	accepts it.
+	* Makefile.in: Regenerated.
+	* configure: Regenerated.
+
+2012-12-07  Jakub Jelinek  
+
+	PR bootstrap/54926
+	* Makefile.am (AM_CFLAGS): Add -frandom-seed=$@.
+	* Makefile.in: Regenerated.
+
+2012-11-20  Ian Lance Taylor  
+
+	* dwarf.c (read_attribute): Always clear val.
+
+2012-11-13  Ian Lance Taylor  
+
+	PR other/55312
+	* configure.ac: Only add -Werror if building a target library.
+	* configure: Rebuild.
+
+2012-11-12  Ian Lance Taylor  
+	    Rainer Orth  
+	    Gerald Pfeifer  
+
+	* configure.ac: Check for getexecname.
+	* fileline.c: #include .  Define getexecname if not
+	available.
+	(fileline_initialize): Try to find the executable in a few
+	different ways.
+	* print.c (error_callback): Only print the filename if it came
+	from the backtrace state.
+	* configure, config.h.in: Rebuild.
+
+2012-10-29  Ian Lance Taylor  
+
+	* mmap.c (backtrace_vector_release): Correct last patch: add
+	aligned, not size.
+
+2012-10-29  Ian Lance Taylor  
+
+	* mmap.c (backtrace_vector_release): Make sure freed block is
+	aligned on 8-byte boundary.
+
+2012-10-26  Ian Lance Taylor  
+
+	PR other/55087
+	* posix.c (backtrace_open): Add does_not_exist parameter.
+	* elf.c (phdr_callback): Do not warn if shared library could not
+	be opened.
+	* fileline.c (fileline_initialize): Update calls to
+	backtrace_open.
+	* internal.h (backtrace_open): Update declaration.
+
+2012-10-26  Jack Howarth  
+
+	PR target/55061
+	* configure.ac: Check for _Unwind_GetIPInfo function declaration.
+	* configure: Regenerate.
+
+2012-10-24  Ian Lance Taylor  
+
+	PR target/55061
+	* configure.ac: Check whether -funwind-tables option works.
+	* configure: Rebuild.
+
+2012-10-11  Ian Lance Taylor  
+
+	* configure.ac: Do not use dl_iterate_phdr on Solaris 10.
+	* configure: Rebuild.
+
+2012-10-10  Ian Lance Taylor  
+
+	* elf.c: Rename all Elf typedefs to start with b_elf, and be all
+	lower case.
+
+2012-10-10  Hans-Peter Nilsson  
+
+	* elf.c (elf_add_syminfo_data): Add casts to avoid warning.
+
+2012-10-09  Ian Lance Taylor  
+
+	* dwarf.c (dwarf_fileline): Add cast to avoid warning.
+	(backtrace_dwarf_add): Likewise.
+
+2012-10-09  Ian Lance Taylor  
+
+	Add support for tracing through shared libraries.
+	* configure.ac: Check for link.h and dl_iterate_phdr.
+	* elf.c: #include  if system has dl_iterate_phdr.  #undef
+	ELF macros before #defining them.
+	(dl_phdr_info, dl_iterate_phdr): Define if system does not have
+	dl_iterate_phdr.
+	(struct elf_syminfo_data): Add next field.
+	(elf_initialize_syminfo): Initialize next field.
+	(elf_add_syminfo_data): New static function.
+	(elf_add): New static function, broken out of
+	backtrace_initialize.  Call backtrace_dwarf_add instead of
+	backtrace_dwarf_initialize.
+	(struct phdr_data): Define.
+	(phdr_callback): New static function.
+	(backtrace_initialize): Call elf_add.
+	* dwarf.c (struct dwarf_data): Add next and base_address fields.
+	(add_unit_addr): Add base_address parameter.  Change all callers.
+	(add_unit_ranges, build_address_map): Likewise.
+	(add_line): Add ddata parameter.  Change all callers.
+	(read_line_program, add_function_range): Likewise.
+	(dwarf_lookup_pc): New static function, broken out of
+	dwarf_fileline.
+	(dwarf_fileline): Call dwarf_lookup_pc.
+	(build_dwarf_data): New static function.
+	(backtrace_dwarf_add): New function.
+	(backtrace_dwarf_initialize): Remove.
+	* internal.h (backtrace_dwarf_initialize): Don't declare.
+	(backtrace_dwarf_add): Declare.
+	* configure, config.h.in: Rebuild.
+
+2012-10-04  Gerald Pfeifer  
+
+	* btest.c (f23): Avoid uninitialized variable warning.
+
+2012-10-04  Ian Lance Taylor  
+
+	* dwarf.c: If the system header files do not declare strnlen,
+	provide our own version.
+
+2012-10-03  Ian Lance Taylor  
+
+	* dwarf.c (read_uleb128): Fix overflow test.
+	(read_sleb128): Likewise.
+	(build_address_map): Don't change unit_buf.start.
+
+2012-10-02  Uros Bizjak  
+
+	PR other/54761
+	* configure.ac (EXTRA_FLAGS): New.
+	* Makefile.am (AM_FLAGS): Add $(EXTRA_FLAGS).
+	* configure, Makefile.in: Regenerate.
+
+2012-09-29  Ian Lance Taylor  
+
+	PR other/54749
+	* fileline.c (fileline_initialize): Pass errnum as -1 when
+	reporting that we could not read executable information after a
+	previous failure.
+
+2012-09-27  Ian Lance Taylor  
+
+	PR bootstrap/54732
+	* configure.ac: Add no-dependencies to AM_INIT_AUTOMAKE.
+	* Makefile.am: Add dependencies for all objects.
+	* configure, aclocal.m4, Makefile.in: Rebuild.
+
+2012-09-27  Ian Lance Taylor  
+
+	PR other/54726
+	* elf.c (backtrace_initialize): Set *fileln_fn, not
+	state->fileln_fn.
+
+2012-09-19  Ian Lance Taylor  
+
+	* configure.ac: Only use GCC_CHECK_UNWIND_GETIPINFO when compiled
+	as a target library.
+	* configure: Rebuild.
+
+2012-09-19  Rainer Orth  
+	    Ian Lance Taylor  
+
+        * configure.ac (GCC_HEADER_STDINT): Invoke.
+        * backtrace.h: If we can't find , use "gstdint.h".
+        * btest.c: Don't include .
+        * dwarf.c: Likewise.
+        * configure, aclocal.m4, Makefile.in, config.h.in: Rebuild.
+
+2012-09-18  Ian Lance Taylor  
+
+	PR bootstrap/54623
+	* Makefile.am (AM_CPPFLAGS): Define.
+	(AM_CFLAGS): Remove -I options.
+	* Makefile.in: Rebuild.
+
+2012-09-18  Ian Lance Taylor  
+
+	* posix.c (O_BINARY): Define if not defined.
+	(backtrace_open): Pass O_BINARY to open.  Only call fcntl if
+	HAVE_FCNTL is defined.
+	* configure.ac: Test for the fcntl function.
+	* configure, config.h.in: Rebuild.
+
+2012-09-18  Ian Lance Taylor  
+
+	* btest.c (test1, test2, test3, test4): Add the unused attribute.
+
+2012-09-18  Ian Lance Taylor  
+
+	* dwarf.c: Correct test of HAVE_DECL_STRNLEN.
+
+2012-09-18  Ian Lance Taylor  
+
+	* configure.ac: Add AC_USE_SYSTEM_EXTENSIONS.
+	* mmapio.c: Don't define _GNU_SOURCE.
+	* configure, config.h.in: Rebuild.
+
+2012-09-18  Ian Lance Taylor  
+
+	* configure.ac: Check whether strnlen is declared.
+	* dwarf.c: Declare strnlen if not declared.
+	* configure, config.h.in: Rebuild.
+
+2012-09-18  Rainer Orth  
+
+	* fileline.c: Include .
+	* mmap.c: Likewise.
+
+2012-09-17  Ian Lance Taylor  
+
+	PR bootstrap/54611
+	* nounwind.c (backtrace_full): Rename from backtrace.  Add state
+	parameter.
+
+2012-09-17  Gerald Pfeifer  
+
+	PR bootstrap/54611
+	* nounwind.c (backtrace_simple): Add state parameter.
+
+2012-09-17  Ian Lance Taylor  
+
+	PR bootstrap/54609
+	* unknown.c (unknown_fileline): Add state parameter, remove
+	fileline_data parameter, name error_callback parameter.
+	(backtrace_initialize): Add state parameter.
+
+2012-09-17  Ian Lance Taylor  
+
+	* Initial implementation.
+

+Copyright (C) 2012-2017 Free Software Foundation, Inc.
+
+Copying and distribution of this file, with or without modification,
+are permitted in any medium without royalty provided the copyright
+notice and this notice are preserved.
diff --git a/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog.jit b/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog.jit
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/libbacktrace/ChangeLog.jit
@@ -0,0 +1,14 @@
+2014-09-24  David Malcolm  
+
+	* ChangeLog.jit: Add copyright footer.
+
+2013-10-03  David Malcolm  
+
+	* configure.ac: Add --enable-host-shared.
+	* configure: Regenerate.
+

+Copyright (C) 2013-2014 Free Software Foundation, Inc.
+
+Copying and distribution of this file, with or without modification,
+are permitted in any medium without royalty provided the copyright
+notice and this notice are preserved.
diff --git a/rpython/rlib/rvmprof/src/shared/libbacktrace/Makefile b/rpython/rlib/rvmprof/src/shared/libbacktrace/Makefile
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/shared/libbacktrace/Makefile
@@ -0,0 +1,770 @@
+# Makefile.in generated by automake 1.11.6 from Makefile.am.
+# Makefile.  Generated from Makefile.in by configure.
+
+# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+# Foundation, Inc.
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+
+
+# Makefile.am -- Backtrace Makefile.
+# Copyright (C) 2012-2016 Free Software Foundation, Inc.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+#     (1) Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+
+#     (2) Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+
+#     (3) The name of the author may not be used to
+#     endorse or promote products derived from this software without
+#     specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+
+am__make_dryrun = \
+  { \
+    am__dry=no; \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        echo 'am--echo: ; @echo "AM"  OK' | $(MAKE) -f - 2>/dev/null \
+          | grep '^AM OK$$' >/dev/null || am__dry=yes;; \
+      *) \
+        for am__flg in $$MAKEFLAGS; do \
+          case $$am__flg in \
+            *=*|--*) ;; \
+            *n*) am__dry=yes; break;; \
+          esac; \
+        done;; \
+    esac; \
+    test $$am__dry = yes; \
+  }
+pkgdatadir = $(datadir)/libbacktrace
+pkgincludedir = $(includedir)/libbacktrace
+pkglibdir = $(libdir)/libbacktrace
+pkglibexecdir = $(libexecdir)/libbacktrace
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = x86_64-pc-linux-gnu
+host_triplet = x86_64-pc-linux-gnu
+target_triplet = x86_64-pc-linux-gnu
+check_PROGRAMS = $(am__EXEEXT_1)
+am__append_1 = btest stest
+subdir = .
+DIST_COMMON = README ChangeLog $(srcdir)/Makefile.in \
+	$(srcdir)/Makefile.am $(top_srcdir)/configure \
+	$(am__configure_deps) $(srcdir)/config.h.in \
+	$(srcdir)/../mkinstalldirs $(srcdir)/backtrace-supported.h.in
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/../config/lead-dot.m4 \
+	$(top_srcdir)/../config/multi.m4 \
+	$(top_srcdir)/../config/override.m4 \
+	$(top_srcdir)/../config/stdint.m4 \
+	$(top_srcdir)/../config/unwind_ipinfo.m4 \
+	$(top_srcdir)/../config/warnings.m4 \
+	$(top_srcdir)/../libtool.m4 $(top_srcdir)/../ltoptions.m4 \
+	$(top_srcdir)/../ltsugar.m4 $(top_srcdir)/../ltversion.m4 \
+	$(top_srcdir)/../lt~obsolete.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
+ configure.lineno config.status.lineno
+mkinstalldirs = $(SHELL) $(top_srcdir)/../mkinstalldirs
+CONFIG_HEADER = config.h
+CONFIG_CLEAN_FILES = backtrace-supported.h
+CONFIG_CLEAN_VPATH_FILES =
+LTLIBRARIES = $(noinst_LTLIBRARIES)
+am__DEPENDENCIES_1 =
+am_libbacktrace_la_OBJECTS = atomic.lo dwarf.lo fileline.lo posix.lo \
+	print.lo sort.lo state.lo
+libbacktrace_la_OBJECTS = $(am_libbacktrace_la_OBJECTS)
+am__EXEEXT_1 = btest$(EXEEXT) stest$(EXEEXT)
+am_btest_OBJECTS = btest-btest.$(OBJEXT)
+btest_OBJECTS = $(am_btest_OBJECTS)
+btest_DEPENDENCIES = libbacktrace.la
+btest_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=link $(CCLD) $(btest_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
+	$(LDFLAGS) -o $@
+am_stest_OBJECTS = stest.$(OBJEXT)
+stest_OBJECTS = $(am_stest_OBJECTS)
+stest_DEPENDENCIES = libbacktrace.la
+DEFAULT_INCLUDES = -I.
+depcomp =
+am__depfiles_maybe =
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+CCLD = $(CC)
+LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+	--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
+	$(LDFLAGS) -o $@
+SOURCES = $(libbacktrace_la_SOURCES) $(EXTRA_libbacktrace_la_SOURCES) \
+	$(btest_SOURCES) $(stest_SOURCES)
+MULTISRCTOP = 
+MULTIBUILDTOP = 
+MULTIDIRS = 
+MULTISUBDIR = 
+MULTIDO = true
+MULTICLEAN = true
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+ETAGS = etags
+CTAGS = ctags
+am__tty_colors = \
+red=; grn=; lgn=; blu=; std=
+ACLOCAL = ${SHELL} /home/rich/src/gcc/missing --run aclocal-1.11
+ALLOC_FILE = mmap.lo
+AMTAR = $${TAR-tar}
+AR = ar
+AUTOCONF = ${SHELL} /home/rich/src/gcc/missing --run autoconf
+AUTOHEADER = ${SHELL} /home/rich/src/gcc/missing --run autoheader
+AUTOMAKE = ${SHELL} /home/rich/src/gcc/missing --run automake-1.11
+AWK = gawk
+BACKTRACE_FILE = backtrace.lo simple.lo
+BACKTRACE_SUPPORTED = 1
+BACKTRACE_SUPPORTS_DATA = 1
+BACKTRACE_SUPPORTS_THREADS = 1
+BACKTRACE_USES_MALLOC = 0
+CC = gcc
+CFLAGS = -g -O2
+CPP = gcc -E
+CPPFLAGS = 
+CYGPATH_W = echo
+DEFS = -DHAVE_CONFIG_H
+DSYMUTIL = 
+DUMPBIN = 
+ECHO_C = 
+ECHO_N = -n
+ECHO_T = 
+EGREP = /usr/bin/grep -E
+EXEEXT = 
+EXTRA_FLAGS = -funwind-tables -frandom-seed=$@
+FGREP = /usr/bin/grep -F
+FORMAT_FILE = elf.lo
+GREP = /usr/bin/grep
+INSTALL = /usr/bin/install -c
+INSTALL_DATA = ${INSTALL} -m 644
+INSTALL_PROGRAM = ${INSTALL}
+INSTALL_SCRIPT = ${INSTALL}
+INSTALL_STRIP_PROGRAM = $(install_sh) -c -s
+LD = /usr/bin/ld -m elf_x86_64
+LDFLAGS = 
+LIBOBJS = 
+LIBS = 
+LIBTOOL = $(SHELL) $(top_builddir)/libtool
+LIPO = 
+LN_S = ln -s
+LTLIBOBJS = 
+MAINT = #
+MAKEINFO = ${SHELL} /home/rich/src/gcc/missing --run makeinfo
+MKDIR_P = /usr/bin/mkdir -p
+NM = /usr/bin/nm -B
+NMEDIT = 
+OBJDUMP = objdump
+OBJEXT = o
+OTOOL = 
+OTOOL64 = 
+PACKAGE = libbacktrace
+PACKAGE_BUGREPORT = 
+PACKAGE_NAME = package-unused
+PACKAGE_STRING = package-unused version-unused
+PACKAGE_TARNAME = libbacktrace
+PACKAGE_URL = 
+PACKAGE_VERSION = version-unused
+PATH_SEPARATOR = :
+PIC_FLAG = 
+RANLIB = ranlib
+SED = /usr/bin/sed
+SET_MAKE = 
+SHELL = /bin/sh
+STRIP = strip
+VERSION = version-unused
+VIEW_FILE = mmapio.lo
+WARN_FLAGS = -W -Wall -Wwrite-strings -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition -Wmissing-format-attribute -Wcast-qual
+abs_builddir = /home/rich/src/gcc/libbacktrace
+abs_srcdir = /home/rich/src/gcc/libbacktrace
+abs_top_builddir = /home/rich/src/gcc/libbacktrace
+abs_top_srcdir = /home/rich/src/gcc/libbacktrace
+ac_ct_CC = gcc
+ac_ct_DUMPBIN = 
+am__leading_dot = .
+am__tar = $${TAR-tar} chof - "$$tardir"
+am__untar = $${TAR-tar} xf -
+bindir = ${exec_prefix}/bin
+build = x86_64-pc-linux-gnu
+build_alias = 
+build_cpu = x86_64
+build_os = linux-gnu
+build_vendor = pc
+builddir = .
+datadir = ${datarootdir}
+datarootdir = ${prefix}/share
+docdir = ${datarootdir}/doc/${PACKAGE_TARNAME}
+dvidir = ${docdir}
+exec_prefix = ${prefix}
+host = x86_64-pc-linux-gnu
+host_alias = 
+host_cpu = x86_64
+host_os = linux-gnu
+host_vendor = pc
+htmldir = ${docdir}
+includedir = ${prefix}/include
+infodir = ${datarootdir}/info
+install_sh = ${SHELL} /home/rich/src/gcc/install-sh
+libdir = ${exec_prefix}/lib
+libexecdir = ${exec_prefix}/libexec
+libtool_VERSION = 1:0:0
+localedir = ${datarootdir}/locale
+localstatedir = ${prefix}/var
+mandir = ${datarootdir}/man
+mkdir_p = /usr/bin/mkdir -p
+multi_basedir = 
+oldincludedir = /usr/include
+pdfdir = ${docdir}
+prefix = /usr/local
+program_transform_name = s,x,x,
+psdir = ${docdir}
+sbindir = ${exec_prefix}/sbin
+sharedstatedir = ${prefix}/com
+srcdir = .
+sysconfdir = ${prefix}/etc
+target = x86_64-pc-linux-gnu
+target_alias = 
+target_cpu = x86_64
+target_os = linux-gnu
+target_vendor = pc
+top_build_prefix = 
+top_builddir = .
+top_srcdir = .
+ACLOCAL_AMFLAGS = -I .. -I ../config
+AM_CPPFLAGS = -I $(top_srcdir)/../include -I $(top_srcdir)/../libgcc \
+	-I ../libgcc
+
+AM_CFLAGS = $(EXTRA_FLAGS) $(WARN_FLAGS) $(PIC_FLAG)
+noinst_LTLIBRARIES = libbacktrace.la
+libbacktrace_la_SOURCES = \
+	backtrace.h \
+	atomic.c \
+	dwarf.c \
+	fileline.c \
+	internal.h \
+	posix.c \
+	print.c \
+	sort.c \
+	state.c
+
+BACKTRACE_FILES = \
+	backtrace.c \
+	simple.c \
+	nounwind.c
+
+FORMAT_FILES = \
+	elf.c \
+	pecoff.c \
+	unknown.c
+
+VIEW_FILES = \
+	read.c \
+	mmapio.c
+
+ALLOC_FILES = \
+	alloc.c \
+	mmap.c
+
+EXTRA_libbacktrace_la_SOURCES = \

From pypy.commits at gmail.com  Wed Mar 29 13:11:12 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 29 Mar 2017 10:11:12 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Make sure that SubBuffer only receives
 BinaryBuffers
Message-ID: <58dbeab0.97212e0a.ef36c.7fd9@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90861:7d94af72aafd
Date: 2017-03-29 18:10 +0100
http://bitbucket.org/pypy/pypy/changeset/7d94af72aafd/

Log:	Make sure that SubBuffer only receives BinaryBuffers

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1540,7 +1540,7 @@
                 self._getarg_error("bytes or read-only buffer", w_obj)
         elif code == 'w*':
             try:
-                return w_obj.buffer_w(self, self.BUF_WRITABLE)
+                return w_obj.buffer_w(self, self.BUF_WRITABLE).as_binary()
             except OperationError:
                 pass
             except BufferInterfaceNotFound:
@@ -1548,7 +1548,7 @@
             self._getarg_error("read-write buffer", w_obj)
         elif code == 'y*':
             try:
-                return w_obj.buffer_w(self, self.BUF_SIMPLE)
+                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_binary()
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes-like object", w_obj)
         elif code == 'y#':
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -219,7 +219,7 @@
     _attrs_ = ['buffer', 'offset', 'size', 'readonly']
     _immutable_ = True
 
-    #@signature(types.any(), types.instance(BinaryBuffer), types.int(), types.int(), returns=types.none())
+    @signature(types.any(), types.instance(BinaryBuffer), types.int(), types.int(), returns=types.none())
     def __init__(self, buffer, offset, size):
         self.readonly = buffer.readonly
         if isinstance(buffer, SubBuffer):     # don't nest them
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -577,6 +577,7 @@
         return space.newbytes(builder.build())
 
     def _raw_read(self, space, buffer, start, length):
+        assert buffer is not None
         length = intmask(length)
         start = intmask(start)
         w_buf = space.newbuffer(SimpleBuffer(SubBuffer(buffer, start, length)))
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -113,7 +113,7 @@
 
 def do_unpack_from(space, format, w_buffer, offset=0):
     size = _calcsize(space, format)
-    buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
+    buf = space.buffer_w(w_buffer, space.BUF_SIMPLE).as_binary()
     if offset < 0:
         offset += buf.getlength()
     if offset < 0 or (buf.getlength() - offset) < size:
@@ -126,7 +126,7 @@
 
 class W_UnpackIter(W_Root):
     def __init__(self, space, w_struct, w_buffer):
-        buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
+        buf = space.buffer_w(w_buffer, space.BUF_SIMPLE).as_binary()
         if w_struct.size <= 0:
             raise oefmt(get_error(space),
                 "cannot iteratively unpack with a struct of length %d",
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -149,7 +149,7 @@
             return self._tolist(space, buf, self.getlength(), itemsize, fmt,
                                 self.getstrides())
         else:
-            return self._tolist_rec(space, buf, 0, 0, fmt)
+            return self._tolist_rec(space, buf.as_binary(), 0, 0, fmt)
 
     def _tolist(self, space, buf, bytecount, itemsize, fmt, strides=None):
         # TODO: this probably isn't very fast
@@ -179,7 +179,7 @@
 
         orig_buf = buf
         for i in range(dimshape):
-            buf = SubBuffer(orig_buf, start, stride)
+            buf = SubBuffer(orig_buf.as_binary(), start, stride)
             item = self._tolist_rec(space, buf, start, idim+1, fmt)
             items[i] = item
             start += stride
@@ -233,7 +233,7 @@
 
         start = self._start_from_tuple(space, w_index)
 
-        buf = SubBuffer(self.buf, start, view.getitemsize())
+        buf = SubBuffer(self.buf.as_binary(), start, view.getitemsize())
         fmtiter = UnpackFormatIterator(space, buf)
         fmtiter.interpret(fmt)
         return fmtiter.result_w[0]
@@ -268,7 +268,7 @@
                     return space.newint(ord(ch))
                 else:
                     # TODO: this probably isn't very fast
-                    buf = SubBuffer(self.buf, idx, itemsize)
+                    buf = SubBuffer(self.buf.as_binary(), idx, itemsize)
                     fmtiter = UnpackFormatIterator(space, buf)
                     fmtiter.length = buf.getlength()
                     fmtiter.interpret(self.buf.getformat())
@@ -347,14 +347,12 @@
             src = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             dst_strides = self.getstrides()
             dim = 0
-            dst = SubBuffer(self.buf, start * itemsize, slicelength * itemsize)
+            dst = SubBuffer(self.buf.as_binary(), start * itemsize, slicelength * itemsize)
             src_stride0 = dst_strides[dim]
 
             off = 0
             src_shape0 = slicelength
             src_stride0 = src.getstrides()[0]
-            if isinstance(w_obj, W_MemoryView):
-                src_stride0 = w_obj.getstrides()[0]
             for i in range(src_shape0):
                 data.append(src.getslice(off,off+itemsize,1,itemsize))
                 off += src_stride0
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -323,6 +323,9 @@
                 self.data.append(c)
         self.data = ''.join(self.data)
 
+    def as_str(self):
+        return self.data
+
     def getformat(self):
         return self.format
 

From pypy.commits at gmail.com  Wed Mar 29 15:28:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 29 Mar 2017 12:28:25 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Stop inheriting Buffer from
 BinaryBuffer
Message-ID: <58dc0ad9.5125190a.c3ff7.80b2@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90864:710267dd4284
Date: 2017-03-29 20:27 +0100
http://bitbucket.org/pypy/pypy/changeset/710267dd4284/

Log:	Stop inheriting Buffer from BinaryBuffer

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -20,14 +20,6 @@
         "Returns an interp-level string with the whole content of the buffer."
         raise NotImplementedError
 
-    def as_str_and_offset_maybe(self):
-        """
-        If the buffer is backed by a string, return a pair (string, offset),
-        where offset is the offset inside the string where the buffer start.
-        Else, return (None, 0).
-        """
-        return None, 0
-
     def getitem(self, index):
         "Returns the index'th character in the buffer."
         raise NotImplementedError   # Must be overriden.  No bounds checks.
@@ -39,16 +31,10 @@
         # May be overridden.  No bounds checks.
         return ''.join([self.getitem(i) for i in range(start, stop, step)])
 
-    def __getslice__(self, start, stop):
-        return self.getslice(start, stop, 1, stop - start)
-
     def setitem(self, index, char):
         "Write a character into the buffer."
         raise NotImplementedError   # Must be overriden.  No bounds checks.
 
-    def __setitem__(self, i, char):
-        return self.setitem(i, char)
-
     def setslice(self, start, string):
         # May be overridden.  No bounds checks.
         for i in range(len(string)):
@@ -93,9 +79,6 @@
     def as_str(self):
         return self.data.as_str()
 
-    def as_str_and_offset_maybe(self):
-        return self.data.as_str_and_offset_maybe()
-
     def getitem(self, index):
         return self.data.getitem(index)
 
@@ -124,43 +107,59 @@
         return [1]
 
 
-class BinaryBuffer(Buffer):
+class BinaryBuffer(object):
     """Base class for buffers of bytes"""
     _attrs_ = ['readonly']
     _immutable_ = True
 
+    def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
+        raise NotImplementedError
+
+    def __len__(self):
+        res = self.getlength()
+        assert res >= 0
+        return res
+
     def as_str(self):
         "Returns an interp-level string with the whole content of the buffer."
         # May be overridden.
         return self.getslice(0, self.getlength(), 1, self.getlength())
 
+    def as_str_and_offset_maybe(self):
+        """
+        If the buffer is backed by a string, return a pair (string, offset),
+        where offset is the offset inside the string where the buffer start.
+        Else, return (None, 0).
+        """
+        return None, 0
+
+    def getitem(self, index):
+        "Returns the index'th character in the buffer."
+        raise NotImplementedError   # Must be overriden.  No bounds checks.
+
+    def __getitem__(self, i):
+        return self.getitem(i)
+
     def getslice(self, start, stop, step, size):
         # May be overridden.  No bounds checks.
         return ''.join([self.getitem(i) for i in range(start, stop, step)])
 
+    def __getslice__(self, start, stop):
+        return self.getslice(start, stop, 1, stop - start)
+
+    def setitem(self, index, char):
+        "Write a character into the buffer."
+        raise NotImplementedError   # Must be overriden.  No bounds checks.
+
+    def __setitem__(self, i, char):
+        return self.setitem(i, char)
 
     def setslice(self, start, string):
         # May be overridden.  No bounds checks.
         for i in range(len(string)):
             self.setitem(start + i, string[i])
 
-
-    def getformat(self):
-        return 'B'
-
-    def getitemsize(self):
-        return 1
-
-    def getndim(self):
-        return 1
-
-    def getshape(self):
-        return [self.getlength()]
-
-    def getstrides(self):
-        return [1]
-
-
 class ByteBuffer(BinaryBuffer):
     _immutable_ = True
 
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -403,8 +403,6 @@
         fromfile() method).
         """
         buf = space.getarg_w('y*', w_s)
-        if buf.getitemsize() != 1:
-            raise oefmt(space.w_TypeError, "a bytes-like object is required")
         s = buf.as_str()
         self._frombytes(space, s)
 
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.interpreter.buffer import BinaryBuffer
+from pypy.interpreter.buffer import BinaryBuffer, SimpleBuffer
 from rpython.rlib import rmmap, rarithmetic, objectmodel
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
@@ -24,7 +24,7 @@
         write_required = bool(flags & space.BUF_WRITABLE)
         if write_required and readonly:
             raise oefmt(space.w_BufferError, "Object is not writable.")
-        return MMapBuffer(self.space, self.mmap, readonly)
+        return SimpleBuffer(MMapBuffer(self.space, self.mmap, readonly))
 
     def writebuf_w(self, space):
         self.check_writeable()
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -11,7 +11,7 @@
 from pypy.tool.option import make_config
 from pypy.interpreter import argument, gateway
 from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache
-from pypy.interpreter.buffer import StringBuffer
+from pypy.interpreter.buffer import StringBuffer, SimpleBuffer
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.objspace.std.sliceobject import W_SliceObject
 
@@ -42,7 +42,7 @@
         is_root(w_subtype)
 
     def buffer_w(self, space, flags):
-        return StringBuffer("foobar")
+        return SimpleBuffer(StringBuffer("foobar"))
 
     def text_w(self, space):
         return NonConstant("foobar")
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -146,7 +146,7 @@
             raise NotImplementedError
         elif dim == 1:
             itemsize = self.getitemsize()
-            return self._tolist(space, buf, self.getlength(), itemsize, fmt,
+            return self._tolist(space, buf.as_binary(), self.getlength(), itemsize, fmt,
                                 self.getstrides())
         else:
             return self._tolist_rec(space, buf.as_binary(), 0, 0, fmt)
@@ -179,7 +179,7 @@
 
         orig_buf = buf
         for i in range(dimshape):
-            buf = SubBuffer(orig_buf.as_binary(), start, stride)
+            buf = SubBuffer(orig_buf, start, stride)
             item = self._tolist_rec(space, buf, start, idim+1, fmt)
             items[i] = item
             start += stride
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -808,4 +808,4 @@
     return [selfvalue[start + i*step] for i in range(sl)]
 
 def _get_buffer(space, w_obj):
-    return space.buffer_w(w_obj, space.BUF_SIMPLE)
+    return space.buffer_w(w_obj, space.BUF_SIMPLE).as_binary()

From pypy.commits at gmail.com  Wed Mar 29 15:34:08 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Mar 2017 12:34:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: add test for exception ordering on
 failed import (issue 2524)
Message-ID: <58dc0c30.02072e0a.f37be.8898@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90865:e7aebde27cfc
Date: 2017-03-29 22:25 +0300
http://bitbucket.org/pypy/pypy/changeset/e7aebde27cfc/

Log:	add test for exception ordering on failed import (issue 2524)

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -361,7 +361,12 @@
         assert sys == o
 
     def test_import_fromlist_must_not_contain_unicodes(self):
-        raises(TypeError, __import__, 'encodings', None, None, [u'xxx'])
+        exc = raises(TypeError, __import__, 'encodings', None, None, [u'xxx'])
+        assert 'not a string' in exc.value.message
+        exc = raises(TypeError, __import__, 'encodings', None, None, [123])
+        assert 'not a string' in exc.value.message
+        # issue 2524
+        raises(ImportError, __import__, 'xxxbadmodule', fromlist=[u'xx']) 
 
     def test_import_relative_back_to_absolute2(self):
         from pkg import abs_x_y

From pypy.commits at gmail.com  Wed Mar 29 15:34:10 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Mar 2017 12:34:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: delay checking fromlist values till
 later, allows ImportError to happen first
Message-ID: <58dc0c32.548d190a.c3457.8381@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90866:9df4aced5d6e
Date: 2017-03-29 22:31 +0300
http://bitbucket.org/pypy/pypy/changeset/9df4aced5d6e/

Log:	delay checking fromlist values till later, allows ImportError to
	happen first

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -362,11 +362,6 @@
         # to get a better trace. if it is unwrapped, the immutability of the
         # tuple is lost
         length = space.len_w(w_fromlist)
-        for i in range(length):
-            w_name = space.getitem(w_fromlist, space.newint(i))
-            if not space.isinstance_w(w_name, space.w_text):
-                raise oefmt(space.w_TypeError,
-                    "'fromlist' items must be str, not %T", w_name)
         if w_path is not None:
             if length == 1 and space.eq_w(
                     space.getitem(w_fromlist, space.newint(0)),
@@ -384,6 +379,9 @@
             if w_fromlist is not None:
                 for i in range(length):
                     w_name = space.getitem(w_fromlist, space.newint(i))
+                    if not space.isinstance_w(w_name, space.w_text):
+                        raise oefmt(space.w_TypeError,
+                            "'Item in ``fromlist'' not a string")
                     if try_getattr(space, w_mod, w_name) is None:
                         return None
         return w_mod
@@ -430,6 +428,9 @@
             if w_fromlist is not None:
                 for i in range(length):
                     w_name = space.getitem(w_fromlist, space.newint(i))
+                    if not space.isinstance_w(w_name, space.w_text):
+                        raise oefmt(space.w_TypeError,
+                            "'Item in ``fromlist'' not a string")
                     if try_getattr(space, w_mod, w_name) is None:
                         load_part(space, w_path, prefix, space.text0_w(w_name),
                                   w_mod, tentative=1)

From pypy.commits at gmail.com  Wed Mar 29 19:11:07 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 29 Mar 2017 16:11:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: define VMPROF_WINDOWS,
 add one c file to the windows compilation
Message-ID: <58dc3f0b.cf4d2e0a.768ff.036d@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90867:1930762b681f
Date: 2017-03-29 19:10 -0400
http://bitbucket.org/pypy/pypy/changeset/1930762b681f/

Log:	define VMPROF_WINDOWS, add one c file to the windows compilation

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -40,7 +40,8 @@
     _libs = []
 else:
     # windows
-    separate_module_files = []
+    compile_extra += ['-DVMPROF_WINDOWS']
+    separate_module_files = [SHARED.join('vmprof_main_win32.c')]
     _libs = []
 
 eci_kwds = dict(

From pypy.commits at gmail.com  Thu Mar 30 03:27:51 2017
From: pypy.commits at gmail.com (tobweber)
Date: Thu, 30 Mar 2017 00:27:51 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Fix overflow of
 nanoseconds when summing up measurements
Message-ID: <58dcb377.45cc190a.f48a.0f64@mx.google.com>

Author: Tobias Weber 
Branch: c8-overheads-instrumentation
Changeset: r2035:3a7fb0854cc6
Date: 2017-03-29 12:08 +0200
http://bitbucket.org/pypy/stmgc/changeset/3a7fb0854cc6/

Log:	Fix overflow of nanoseconds when summing up measurements

diff --git a/c8/stm/timing.h b/c8/stm/timing.h
--- a/c8/stm/timing.h
+++ b/c8/stm/timing.h
@@ -18,7 +18,11 @@
                            sec_diff -= 1;                                   \
                        }                                                    \
                        duration.tv_sec += sec_diff;                         \
-                       duration.tv_nsec += nanosec_diff;
+                       duration.tv_nsec += nanosec_diff;                    \
+                       if (duration.tv_nsec >= 1000000000) {                \
+                           duration.tv_sec += 1;                            \
+                           duration.tv_nsec -= 1000000000;                  \
+                       }
 
 #define pause_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &stop);            \
                       get_duration()

From pypy.commits at gmail.com  Thu Mar 30 06:47:13 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 Mar 2017 03:47:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: add passing test
Message-ID: <58dce231.c9d8190a.50cda.1a4a@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r90868:44ad938b6e59
Date: 2017-03-30 13:46 +0300
http://bitbucket.org/pypy/pypy/changeset/44ad938b6e59/

Log:	add passing test

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -366,7 +366,10 @@
         exc = raises(TypeError, __import__, 'encodings', None, None, [123])
         assert 'not a string' in exc.value.message
         # issue 2524
-        raises(ImportError, __import__, 'xxxbadmodule', fromlist=[u'xx']) 
+        raises(ImportError, __import__, 'xxxbadmodule', fromlist=[u'xx'])
+        mod = __import__('collections', fromlist=[u'defaultdict'])
+        assert mod is not None
+        
 
     def test_import_relative_back_to_absolute2(self):
         from pkg import abs_x_y

From pypy.commits at gmail.com  Thu Mar 30 09:17:43 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 06:17:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: document vmprof-native branch
Message-ID: <58dd0577.99522e0a.b026f.22f7@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90869:fc665b3b560f
Date: 2017-03-30 09:16 -0400
http://bitbucket.org/pypy/pypy/changeset/fc665b3b560f/

Log:	document vmprof-native branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -18,3 +18,6 @@
 
 .. branch: lstrip_to_empty_string
 
+.. branch: vmprof-native
+
+PyPy support to profile native frames in vmprof.

From pypy.commits at gmail.com  Thu Mar 30 09:22:17 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 06:22:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: add extra include dir to add msiinttypes
Message-ID: <58dd0689.86d0190a.1748f.1fad@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90870:ccb6b20e1452
Date: 2017-03-30 09:21 -0400
http://bitbucket.org/pypy/pypy/changeset/ccb6b20e1452/

Log:	add extra include dir to add msiinttypes

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -16,7 +16,9 @@
 SRC = ROOT.join('src')
 SHARED = SRC.join('shared')
 BACKTRACE = SHARED.join('libbacktrace')
+MSIINTTYPES = SHARED.join('msiinttypes')
 
+extra_include_dirs = []
 compile_extra = ['-DRPYTHON_VMPROF', '-O3']
 if sys.platform.startswith('linux'):
     separate_module_files = [
@@ -41,11 +43,13 @@
 else:
     # windows
     compile_extra += ['-DVMPROF_WINDOWS']
+    extra_include_dirs += [MSIINTTYPES]
     separate_module_files = [SHARED.join('vmprof_main_win32.c')]
     _libs = []
 
+
 eci_kwds = dict(
-    include_dirs = [SRC, SHARED, BACKTRACE],
+    include_dirs = [SRC, SHARED, BACKTRACE] + extra_include_dirs,
     includes = ['rvmprof.h','vmprof_stack.h'],
     libraries = _libs,
     separate_module_files = [

From pypy.commits at gmail.com  Thu Mar 30 10:47:00 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 07:47:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove header file that is in shared
 folder
Message-ID: <58dd1a64.86d0190a.1748f.234a@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90871:97070a1f4cc1
Date: 2017-03-30 09:57 -0400
http://bitbucket.org/pypy/pypy/changeset/97070a1f4cc1/

Log:	remove header file that is in shared folder

diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ /dev/null
@@ -1,499 +0,0 @@
-#pragma once
-
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in
- * gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2017
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *   Armin Rigo - arigo at tunes.org
- *   Richard Plangger - planrichi at gmail.com
- *
- */
-
-#define _GNU_SOURCE 1
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include "vmprof.h"
-
-#include "vmp_stack.h"
-#include "vmprof_getpc.h"
-#include "vmprof_mt.h"
-#include "vmprof_common.h"
-#include "compat.h"
-
-#if defined(__unix__)
-#include "rss_unix.h"
-#elif defined(__APPLE__)
-#include "rss_darwin.h"
-#endif
-
-
-/************************************************************/
-
-static void *(*mainloop_get_virtual_ip)(char *) = 0;
-static int opened_profile(const char *interp_name, int memory, int proflines, int native);
-static void flush_codes(void);
-
-/************************************************************/
-
-/* value: last bit is 1 if signals must be ignored; all other bits
-   are a counter for how many threads are currently in a signal handler */
-static long volatile signal_handler_value = 1;
-
-RPY_EXTERN
-void vmprof_ignore_signals(int ignored)
-{
-    if (!ignored) {
-        __sync_fetch_and_and(&signal_handler_value, ~1L);
-    }
-    else {
-        /* set the last bit, and wait until concurrently-running signal
-           handlers finish */
-        while (__sync_or_and_fetch(&signal_handler_value, 1L) != 1L) {
-            usleep(1);
-        }
-    }
-}
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-static char atfork_hook_installed = 0;
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc)
-{
-    PY_STACK_FRAME_T * frame;
-#ifdef RPYTHON_VMPROF
-    // do nothing here, 
-    frame = (PY_STACK_FRAME_T*)current;
-#else
-    if (!current) {
-        return 0;
-    }
-    frame = current->frame;
-#endif
-    return vmp_walk_and_record_stack(frame, result, max_depth, 1, pc);
-}
-
-/* *************************************************************
- * the signal handler
- * *************************************************************
- */
-
-#include 
-
-static volatile int spinlock;
-static jmp_buf restore_point;
-
-static void segfault_handler(int arg)
-{
-    longjmp(restore_point, SIGSEGV);
-}
-
-int _vmprof_sample_stack(struct profbuf_s *p, PY_THREAD_STATE_T * tstate, ucontext_t * uc)
-{
-    int depth;
-    struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;
-    st->marker = MARKER_STACKTRACE;
-    st->count = 1;
-#ifdef RPYTHON_VMPROF
-    depth = get_stack_trace(get_vmprof_stack(), st->stack, MAX_STACK_DEPTH-1, (intptr_t)GetPC(uc));
-#else
-    depth = get_stack_trace(tstate, st->stack, MAX_STACK_DEPTH-1, (intptr_t)NULL);
-#endif
-    if (depth == 0) {
-        return 0;
-    }
-    st->depth = depth;
-    st->stack[depth++] = tstate;
-    long rss = get_current_proc_rss();
-    if (rss >= 0)
-        st->stack[depth++] = (void*)rss;
-    p->data_offset = offsetof(struct prof_stacktrace_s, marker);
-    p->data_size = (depth * sizeof(void *) +
-                    sizeof(struct prof_stacktrace_s) -
-                    offsetof(struct prof_stacktrace_s, marker));
-    return 1;
-}
-
-#ifndef RPYTHON_VMPROF
-static PY_THREAD_STATE_T * _get_pystate_for_this_thread(void) {
-    // see issue 116 on github.com/vmprof/vmprof-python.
-    // PyGILState_GetThisThreadState(); can hang forever
-    //
-    PyInterpreterState * istate;
-    PyThreadState * state;
-    long mythread_id;
-
-    istate = PyInterpreterState_Head();
-    if (istate == NULL) {
-        return NULL;
-    }
-    mythread_id = PyThread_get_thread_ident();
-    // fish fish fish, it will NOT lock the keymutex in pythread
-    do {
-        state = PyInterpreterState_ThreadHead(istate);
-        do {
-            if (state->thread_id == mythread_id) {
-                return state;
-            }
-        } while ((state = PyThreadState_Next(state)) != NULL);
-    } while ((istate = PyInterpreterState_Next(istate)) != NULL);
-
-    // uh? not found?
-    return NULL;
-}
-#endif
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext)
-{
-    int commit;
-    PY_THREAD_STATE_T * tstate = NULL;
-    void (*prevhandler)(int);
-#ifndef RPYTHON_VMPROF
-    // TERRIBLE HACK AHEAD
-    // on OS X, the thread local storage is sometimes uninitialized
-    // when the signal handler runs - it means it's impossible to read errno
-    // or call any syscall or read PyThread_Current or pthread_self. Additionally,
-    // it seems impossible to read the register gs.
-    // here we register segfault handler (all guarded by a spinlock) and call
-    // longjmp in case segfault happens while reading a thread local
-    //
-    // We do the same error detection for linux to ensure that
-    // get_current_thread_state returns a sane result
-    while (__sync_lock_test_and_set(&spinlock, 1)) {
-    }
-    prevhandler = signal(SIGSEGV, &segfault_handler);
-    int fault_code = setjmp(restore_point);
-    if (fault_code == 0) {
-        pthread_self();
-        tstate = _get_pystate_for_this_thread();
-    } else {
-        signal(SIGSEGV, prevhandler);
-        __sync_lock_release(&spinlock);
-        return;
-    }
-    signal(SIGSEGV, prevhandler);
-    __sync_lock_release(&spinlock);
-#endif
-
-    long val = __sync_fetch_and_add(&signal_handler_value, 2L);
-
-    if ((val & 1) == 0) {
-        int saved_errno = errno;
-        int fd = vmp_profile_fileno();
-        assert(fd >= 0);
-
-        struct profbuf_s *p = reserve_buffer(fd);
-        if (p == NULL) {
-            /* ignore this signal: there are no free buffers right now */
-        } else {
-#ifdef RPYTHON_VMPORF
-            commit = _vmprof_sample_stack(p, NULL, (ucontext_t*)ucontext);
-#else
-            commit = _vmprof_sample_stack(p, tstate, (ucontext_t*)ucontext);
-#endif
-            if (commit) {
-                commit_buffer(fd, p);
-            } else {
-                cancel_buffer(p);
-            }
-        }
-
-        errno = saved_errno;
-    }
-
-    __sync_sub_and_fetch(&signal_handler_value, 2L);
-}
-
-
-
-/* *************************************************************
- * the setup and teardown functions
- * *************************************************************
- */
-
-static int install_sigprof_handler(void)
-{
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-        sigaction(SIGPROF, &sa, NULL) == -1)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_handler(void)
-{
-    if (signal(SIGPROF, SIG_DFL) == SIG_ERR)
-        return -1;
-    return 0;
-}
-
-static int install_sigprof_timer(void)
-{
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = profile_interval_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static void atfork_disable_timer(void) {
-    if (profile_interval_usec > 0) {
-        remove_sigprof_timer();
-#ifndef RPYTHON_VMPROF
-        is_enabled = 0;
-#endif
-    }
-}
-
-static void atfork_enable_timer(void) {
-    if (profile_interval_usec > 0) {
-        install_sigprof_timer();
-#ifndef RPYTHON_VMPROF
-        is_enabled = 1;
-#endif
-    }
-}
-
-static void atfork_close_profile_file(void) {
-    int fd = vmp_profile_fileno();
-    if (fd != -1)
-        close(fd);
-    vmp_set_profile_fileno(-1);
-}
-
-static int install_pthread_atfork_hooks(void) {
-    /* this is needed to prevent the problems described there:
-         - http://code.google.com/p/gperftools/issues/detail?id=278
-         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
-        TL;DR: if the RSS of the process is large enough, the clone() syscall
-        will be interrupted by the SIGPROF before it can complete, then
-        retried, interrupted again and so on, in an endless loop.  The
-        solution is to disable the timer around the fork, and re-enable it
-        only inside the parent.
-    */
-    if (atfork_hook_installed)
-        return 0;
-    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, atfork_close_profile_file);
-    if (ret != 0)
-        return -1;
-    atfork_hook_installed = 1;
-    return 0;
-}
-
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-void init_cpyprof(int native)
-{
-    // skip this if native should not be enabled
-    if (!native) {
-        vmp_native_disable();
-        return;
-    }
-#if CPYTHON_HAS_FRAME_EVALUATION
-    PyThreadState *tstate = PyThreadState_GET();
-    tstate->interp->eval_frame = vmprof_eval;
-    _default_eval_loop = _PyEval_EvalFrameDefault;
-#elif defined(RPYTHON_VMPROF)
-    // do nothing here, the stack is maintained by rpython
-    // no need for a trampoline
-#else
-    if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx,
-                vmprof_eval, (void*)&_default_eval_loop) == 0) {
-    } else {
-        fprintf(stderr, "FATAL: could not insert trampline, try with --no-native\n");
-        // TODO dump the first few bytes and tell them to create an issue!
-        exit(-1);
-    }
-#endif
-    vmp_native_enable();
-}
-
-static void disable_cpyprof(void)
-{
-    vmp_native_disable();
-#if CPYTHON_HAS_FRAME_EVALUATION
-    PyThreadState *tstate = PyThreadState_GET();
-    tstate->interp->eval_frame = _PyEval_EvalFrameDefault;
-#elif defined(RPYTHON_VMPROF)
-    // TODO nothing?
-#else
-    if (vmp_unpatch_callee_trampoline(PyEval_EvalFrameEx) > 0) {
-        fprintf(stderr, "FATAL: could not remove trampoline\n");
-        exit(-1);
-    }
-#endif
-    dump_native_symbols(vmp_profile_fileno());
-}
-#endif
-
-RPY_EXTERN
-int vmprof_enable(int memory, int native)
-{
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    init_cpyprof(native);
-#endif
-    assert(vmp_profile_fileno() >= 0);
-    assert(prepare_interval_usec > 0);
-    profile_interval_usec = prepare_interval_usec;
-    if (memory && setup_rss() == -1)
-        goto error;
-    if (install_pthread_atfork_hooks() == -1)
-        goto error;
-    if (install_sigprof_handler() == -1)
-        goto error;
-    if (install_sigprof_timer() == -1)
-        goto error;
-    vmprof_ignore_signals(0);
-    return 0;
-
- error:
-    vmp_set_profile_fileno(-1);
-    profile_interval_usec = 0;
-    return -1;
-}
-
-
-int close_profile(void)
-{
-    (void)vmp_write_time_now(MARKER_TRAILER);
-
-    teardown_rss();
-    /* don't close() the file descriptor from here */
-    vmp_set_profile_fileno(-1);
-    return 0;
-}
-
-RPY_EXTERN
-int vmprof_disable(void)
-{
-    vmprof_ignore_signals(1);
-    profile_interval_usec = 0;
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    disable_cpyprof();
-#endif
-
-    if (remove_sigprof_timer() == -1)
-        return -1;
-    if (remove_sigprof_handler() == -1)
-        return -1;
-    flush_codes();
-    if (shutdown_concurrent_bufs(vmp_profile_fileno()) < 0)
-        return -1;
-    return close_profile();
-}
-
-RPY_EXTERN
-int vmprof_register_virtual_function(char *code_name, intptr_t code_uid,
-                                     int auto_retry)
-{
-    long namelen = strnlen(code_name, 1023);
-    long blocklen = 1 + sizeof(intptr_t) + sizeof(long) + namelen;
-    struct profbuf_s *p;
-    char *t;
-
- retry:
-    p = current_codes;
-    if (p != NULL) {
-        if (__sync_bool_compare_and_swap(¤t_codes, p, NULL)) {
-            /* grabbed 'current_codes': we will append the current block
-               to it if it contains enough room */
-            size_t freesize = SINGLE_BUF_SIZE - p->data_size;
-            if (freesize < (size_t)blocklen) {
-                /* full: flush it */
-                commit_buffer(vmp_profile_fileno(), p);
-                p = NULL;
-            }
-        }
-        else {
-            /* compare-and-swap failed, don't try again */
-            p = NULL;
-        }
-    }
-
-    if (p == NULL) {
-        p = reserve_buffer(vmp_profile_fileno());
-        if (p == NULL) {
-            /* can't get a free block; should almost never be the
-               case.  Spin loop if allowed, or return a failure code
-               if not (e.g. we're in a signal handler) */
-            if (auto_retry > 0) {
-                auto_retry--;
-                usleep(1);
-                goto retry;
-            }
-            return -1;
-        }
-    }
-
-    t = p->data + p->data_size;
-    p->data_size += blocklen;
-    assert(p->data_size <= SINGLE_BUF_SIZE);
-    *t++ = MARKER_VIRTUAL_IP;
-    memcpy(t, &code_uid, sizeof(intptr_t)); t += sizeof(intptr_t);
-    memcpy(t, &namelen, sizeof(long)); t += sizeof(long);
-    memcpy(t, code_name, namelen);
-
-    /* try to reattach 'p' to 'current_codes' */
-    if (!__sync_bool_compare_and_swap(¤t_codes, NULL, p)) {
-        /* failed, flush it */
-        commit_buffer(vmp_profile_fileno(), p);
-    }
-    return 0;
-}
-
-static void flush_codes(void)
-{
-    struct profbuf_s *p = current_codes;
-    if (p != NULL) {
-        current_codes = NULL;
-        commit_buffer(vmp_profile_fileno(), p);
-    }
-}

From pypy.commits at gmail.com  Thu Mar 30 10:47:03 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 07:47:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: copy over the implementation for pypy
 which was overwritten by the cpython version by merge
Message-ID: <58dd1a67.020f2e0a.2774.25c6@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90872:87791d83ead0
Date: 2017-03-30 10:44 -0400
http://bitbucket.org/pypy/pypy/changeset/87791d83ead0/

Log:	copy over the implementation for pypy which was overwritten by the
	cpython version by merge

diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -150,23 +150,6 @@
     Original_code_dealloc(co);
 }
 
-
-void dump_native_symbols(int fileno)
-{
-    PyObject * mod = NULL;
-
-    mod = PyImport_ImportModuleNoBlock("vmprof");
-    if (mod == NULL)
-        goto error;
-
-    PyObject_CallMethod(mod, "dump_native_symbols", "(l)", fileno);
-
-error:
-    Py_XDECREF(mod);
-}
-
-
-
 static PyObject *enable_vmprof(PyObject* self, PyObject *args)
 {
     int fd;
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.h b/rpython/rlib/rvmprof/src/shared/_vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.h
@@ -3,8 +3,15 @@
 #include "vmprof.h"
 
 #ifdef VMPROF_WINDOWS
+
+#include 
+// CPython 3.6 defines all the inttypes for us, we do not need the msiinttypes
+// library for that version or any newer!
+#if (PY_VERSION_HEX < 0x3060000)
 #include "msiinttypes/inttypes.h"
 #include "msiinttypes/stdint.h"
+#endif
+
 #else
 #include 
 #include 
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -243,8 +243,6 @@
     return 0;
 }
 
-#ifdef RPYTHON_VMPROF
-
 #define WORD_SIZE sizeof(long)
 #define ADDR_SIZE sizeof(void*)
 #define MAXLEN 1024
@@ -254,10 +252,12 @@
     off_t pos_before;
     struct str {
         void * addr;
-        // NOTE windows 64, not supported yet
+        // NOTE windows, not supported yet would be a problem for 64 bit
+        // hint: alignment
         long size;
         char str[1024];
     } s;
+    fsync(fileno);
     pos_before = lseek(fileno, 0, SEEK_CUR);
     lseek(fileno, 0, SEEK_END);
 
@@ -286,7 +286,7 @@
 {
     long chars;
     int count = read(fileno, &chars, sizeof(long));
-    LOG("reading string of %d chars\n", chars);
+    //LOG("reading string of %d chars\n", chars);
     if (count <= 0) {
         return 1;
     }
@@ -309,14 +309,14 @@
 long _read_word(int fileno)
 {
     long w;
-    read(fileno, &w, WORD_SIZE);
+    (void)read(fileno, &w, WORD_SIZE);
     return w;
 }
 
 void * _read_addr(int fileno)
 {
     void * a;
-    read(fileno, &a, ADDR_SIZE);
+    (void)read(fileno, &a, ADDR_SIZE);
     return a;
 }
 
@@ -341,21 +341,19 @@
 
 void dump_native_symbols(int fileno)
 {
-    // only call this function
     off_t orig_pos, cur_pos;
     char marker;
     ssize_t count;
     int version;
     int flags;
     int memory = 0, lines = 0, native = 0;
+    fsync(fileno);
     orig_pos = lseek(fileno, 0, SEEK_CUR);
 
     lseek(fileno, 5*WORD_SIZE, SEEK_SET);
 
     while (1) {
-        LOG("pre read\n");
         count = read(fileno, &marker, 1);
-        LOG("post read\n");
         if (count <= 0) {
             break;
         }
@@ -383,7 +381,7 @@
                 break;
             } case MARKER_VIRTUAL_IP:
               case MARKER_NATIVE_SYMBOLS: {
-                LOG("virtip 0x%llx\n", cur_pos);
+                //LOG("virtip 0x%llx\n", cur_pos);
                 if (_skip_addr(fileno) != 0) { return; }
                 if (_skip_string(fileno) != 0) { return; }
                 break;
@@ -394,10 +392,16 @@
 
                 LOG("stack 0x%llx %d %d\n", cur_pos, trace_count, depth);
 
+#ifdef RPYTHON_VMPROF
                 for (i = depth/2-1; i >= 0; i--) {
                     long kind = (long)_read_addr(fileno);
                     void * addr = _read_addr(fileno);
                     if (kind == VMPROF_NATIVE_TAG) {
+#else
+                for (i = 0; i < depth; i++) {
+                    void * addr = _read_addr(fileno);
+                    if (((intptr_t)addr & 0x1) == 1) {
+#endif
                         LOG("found kind %p\n", addr);
                         char name[MAXLEN];
                         char srcfile[MAXLEN];
@@ -412,7 +416,9 @@
                 }
                 LOG("passed  memory %d \n", memory);
 
-                if (_skip_addr(fileno) != 0) { return; } // thread id
+                if (version >= VERSION_THREAD_ID) {
+                    if (_skip_addr(fileno) != 0) { return; } // thread id
+                }
                 if (memory) {
                     if (_skip_addr(fileno) != 0) { return; } // profile memory
                 }
@@ -420,6 +426,7 @@
                 break;
             } default: {
                 fprintf(stderr, "unknown marker 0x%x\n", marker);
+                lseek(fileno, 0, SEEK_END);
                 return;
             }
         }
@@ -432,4 +439,3 @@
 
     lseek(fileno, 0, SEEK_END);
 }
-#endif
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -126,7 +126,7 @@
 }
 
 int vmp_walk_and_record_python_stack_only(PY_STACK_FRAME_T *frame, void ** result,
-                                     int max_depth, int depth, intptr_t pc)
+                                          int max_depth, int depth, intptr_t pc)
 {
     while (depth < max_depth && frame) {
         frame = _write_python_stack_entry(frame, result, &depth, max_depth);
@@ -154,6 +154,25 @@
                               int max_depth, int signal, intptr_t pc) {
 
     // called in signal handler
+    //
+    // This function records the stack trace for a python program. It also
+    // tracks native function calls if libunwind can be found on the system.
+    //
+    // The idea is the following (in the native case):
+    //
+    // 1) Remove frames until the signal frame is found (skipping it as well)
+    // 2) if the current frame corresponds to PyEval_EvalFrameEx (or the equivalent
+    //    for each python version), the jump to 4)
+    // 3) jump to 2)
+    // 4) walk each python frame and record it
+    //
+    //
+    // There are several cases that need to be taken care of.
+    //
+    // CPython supports line profiling, PyPy does not. At the same time
+    // PyPy saves the information of an address in the same way as line information
+    // is saved in CPython. _write_python_stack_entry for details.
+    //
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
     intptr_t func_addr;
     unw_cursor_t cursor;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -257,8 +257,15 @@
 
 static int remove_sigprof_handler(void)
 {
-    if (signal(SIGPROF, SIG_DFL) == SIG_ERR)
+    struct sigaction ign_sigint, prev;
+    ign_sigint.sa_handler = SIG_IGN;
+    ign_sigint.sa_flags = 0;
+    sigemptyset(&ign_sigint.sa_mask);
+
+    if (sigaction(SIGPROF, &ign_sigint, NULL) < 0) {
+        fprintf(stderr, "Could not remove the signal handler (for profiling)\n");
         return -1;
+    }
     return 0;
 }
 
@@ -275,12 +282,12 @@
 
 static int remove_sigprof_timer(void) {
     static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
+    timerclear(&(timer.it_interval));
+    timerclear(&(timer.it_value));
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
+        fprintf(stderr, "Could not disable the signal handler (for profiling)\n");
         return -1;
+    }
     return 0;
 }
 
@@ -343,7 +350,6 @@
 static void disable_cpyprof(void)
 {
     vmp_native_disable();
-    dump_native_symbols(vmp_profile_fileno());
 }
 #endif
 
@@ -376,9 +382,15 @@
 
 int close_profile(void)
 {
+    int fileno = vmp_profile_fileno();
+    fsync(fileno);
+    dump_native_symbols(fileno);
+
     (void)vmp_write_time_now(MARKER_TRAILER);
 
     teardown_rss();
+
+
     /* don't close() the file descriptor from here */
     vmp_set_profile_fileno(-1);
     return 0;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
@@ -19,6 +19,13 @@
 
 int vmp_write_all(const char *buf, size_t bufsize);
 
+#ifdef VMPROF_RPYTHON
+typedef struct pypy_threadlocal_s PY_WIN_THREAD_STATE;
+#else
+typedef PyThreadState PY_WIN_THREAD_STATE;
+#endif
+
+
 RPY_EXTERN
 int vmprof_register_virtual_function(char *code_name, intptr_t code_uid,
                                      int auto_retry)
@@ -35,11 +42,19 @@
     return 0;
 }
 
-int vmprof_snapshot_thread(DWORD thread_id, PyThreadState *tstate, prof_stacktrace_s *stack)
+int vmprof_snapshot_thread(DWORD thread_id, PY_WIN_THREAD_STATE *tstate, prof_stacktrace_s *stack)
 {
     HRESULT result;
-    HANDLE hThread = OpenThread(THREAD_ALL_ACCESS, FALSE, thread_id);
+    HANDLE hThread;
     int depth;
+    CONTEXT ctx;
+#ifdef RPYTHON_LL2CTYPES
+    return 0; // not much we can do
+#else
+#ifndef RPY_TLOFS_thread_ident
+    return 0; // we can't freeze threads, unsafe
+#else
+    hThread = OpenThread(THREAD_ALL_ACCESS, FALSE, thread_id);
     if (!hThread) {
         return -1;
     }
@@ -47,6 +62,19 @@
     if(result == 0xffffffff)
         return -1; // possible, e.g. attached debugger or thread alread suspended
     // find the correct thread
+#ifdef RPYTHON_VMPROF
+    ctx.ContextFlags = CONTEXT_FULL;
+    if (!GetThreadContext(hThread, &ctx))
+        return -1;
+    depth = get_stack_trace(tstate->vmprof_tl_stack,
+                     stack->stack, MAX_STACK_DEPTH-2, ctx.Eip);
+    stack->depth = depth;
+    stack->stack[depth++] = thread_id;
+    stack->count = 1;
+    stack->marker = MARKER_STACKTRACE;
+    ResumeThread(hThread);
+    return depth;
+#else
     depth = vmp_walk_and_record_stack(tstate->frame, stack->stack,
                                       MAX_STACK_DEPTH, 0, 0);
     stack->depth = depth;
@@ -55,10 +83,15 @@
     stack->marker = MARKER_STACKTRACE;
     ResumeThread(hThread);
     return depth;
+#endif
+
+#endif
+#endif
 }
 
+#ifdef RPYTHON_VMPROF
 static
-PyThreadState * get_current_thread_state(void)
+PY_WIN_THREAD_STATE * get_current_thread_state(void)
 {
 #if PY_MAJOR_VERSION < 3
     return _PyThreadState_Current;
@@ -68,14 +101,21 @@
     return _PyThreadState_UncheckedGet();
 #endif
 }
+#endif
 
 long __stdcall vmprof_mainloop(void *arg)
-{   
+{
+#ifdef RPYTHON_LL2CTYPES
+    // for tests only
+    return 0;
+#else
+    // it is not a test case!
+    PY_WIN_THREAD_STATE *tstate;
+    HANDLE hThreadSnap = INVALID_HANDLE_VALUE; 
     prof_stacktrace_s *stack = (prof_stacktrace_s*)malloc(SINGLE_BUF_SIZE);
-    HANDLE hThreadSnap = INVALID_HANDLE_VALUE; 
     int depth;
-    PyThreadState *tstate;
-
+#ifndef RPYTHON_VMPROF
+    // cpython version
     while (1) {
         Sleep(profile_interval_usec * 1000);
         if (!enabled) {
@@ -90,6 +130,33 @@
                           SIZEOF_PROF_STACKTRACE + depth * sizeof(void*));
         }
     }
+#else
+    // pypy version
+    while (1) {
+        //Sleep(profile_interval_usec * 1000);
+        Sleep(10);
+        if (!enabled) {
+            continue;
+        }
+        _RPython_ThreadLocals_Acquire();
+        tstate = _RPython_ThreadLocals_Head(); // the first one is one behind head
+        tstate = _RPython_ThreadLocals_Enum(tstate);
+        while (tstate) {
+            if (p->ready == 42) {
+                depth = vmprof_snapshot_thread(tstate->thread_ident, tstate, stack);
+                if (depth > 0) {
+                    _write_all((char*)stack + offsetof(prof_stacktrace_s, marker),
+                         depth * sizeof(void *) +
+                         sizeof(struct prof_stacktrace_s) -
+                         offsetof(struct prof_stacktrace_s, marker));
+                }
+            }
+            tstate = _RPython_ThreadLocals_Enum(tstate);
+        }
+        _RPython_ThreadLocals_Release();
+    }
+#endif
+#endif
 }
 
 RPY_EXTERN

From pypy.commits at gmail.com  Thu Mar 30 11:22:24 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 30 Mar 2017 08:22:24 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Move as_str() implementation from
 W_MemoryViewObject to Buffer
Message-ID: <58dd22b0.c9d8190a.50cda.26ea@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90873:f0b49d861488
Date: 2017-03-30 16:15 +0100
http://bitbucket.org/pypy/pypy/changeset/f0b49d861488/

Log:	Move as_str() implementation from W_MemoryViewObject to Buffer

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -18,7 +18,7 @@
 
     def as_str(self):
         "Returns an interp-level string with the whole content of the buffer."
-        raise NotImplementedError
+        return ''.join(self._copy_buffer())
 
     def getitem(self, index):
         "Returns the index'th character in the buffer."
@@ -65,6 +65,44 @@
     def releasebuffer(self):
         pass
 
+    def _copy_buffer(self):
+        if self.getndim() == 0:
+            itemsize = self.getitemsize()
+            return [self.getslice(0, itemsize, 1, itemsize)]
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+        strides = self.getstrides()
+
+        if self.getndim() - 1 == idim:
+            self._copy_base(data, off)
+            return
+
+        for i in range(shape):
+            self._copy_rec(idim + 1, data, off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0]
+        strides = self.getstrides()
+        itemsize = self.getitemsize()
+        bytesize = self.getlength()
+        copiedbytes = 0
+        for i in range(step):
+            bytes = self.getslice(off, off+itemsize, 1, itemsize)
+            data.append(bytes)
+            copiedbytes += len(bytes)
+            off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if copiedbytes >= bytesize:
+                break
+
+
 class SimpleBuffer(Buffer):
     _attrs_ = ['readonly', 'data']
     _immutable_ = True
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -69,8 +69,8 @@
                 return space.newbool(getattr(operator, name)(self, w_other))
             if isinstance(w_other, W_MemoryView):
                 # xxx not the most efficient implementation
-                str1 = self.as_str()
-                str2 = w_other.as_str()
+                str1 = self.buf.as_str()
+                str2 = w_other.buf.as_str()
                 return space.newbool(getattr(operator, name)(str1, str2))
 
             try:
@@ -80,7 +80,7 @@
                     raise
                 return space.w_NotImplemented
             else:
-                str1 = self.as_str()
+                str1 = self.buf.as_str()
                 str2 = buf.as_str()
                 return space.newbool(getattr(operator, name)(str1, str2))
         descr__cmp.func_name = name
@@ -89,52 +89,12 @@
     descr_eq = _make_descr__cmp('eq')
     descr_ne = _make_descr__cmp('ne')
 
-    def as_str(self):
-        return ''.join(self.copy_buffer())
-
-    def copy_buffer(self):
-        if self.getndim() == 0:
-            itemsize = self.getitemsize()
-            return [self.buf.getslice(0, itemsize, 1, itemsize)]
-        data = []
-        self._copy_rec(0, data, 0)
-        return data
-
-    def _copy_rec(self, idim, data, off):
-        shapes = self.getshape()
-        shape = shapes[idim]
-        strides = self.getstrides()
-
-        if self.getndim() - 1 == idim:
-            self._copy_base(data, off)
-            return
-
-        for i in range(shape):
-            self._copy_rec(idim + 1, data, off)
-            off += strides[idim]
-
-    def _copy_base(self, data, off):
-        shapes = self.getshape()
-        step = shapes[0]
-        strides = self.getstrides()
-        itemsize = self.getitemsize()
-        bytesize = self.getlength()
-        copiedbytes = 0
-        for i in range(step):
-            bytes = self.buf.getslice(off, off+itemsize, 1, itemsize)
-            data.append(bytes)
-            copiedbytes += len(bytes)
-            off += strides[0]
-            # do notcopy data if the sub buffer is out of bounds
-            if copiedbytes >= bytesize:
-                break
-
     def getlength(self):
         return self.buf.getlength()
 
     def descr_tobytes(self, space):
         self._check_released(space)
-        return space.newbytes(self.as_str())
+        return space.newbytes(self.buf.as_str())
 
     def descr_tolist(self, space):
         self._check_released(space)
@@ -735,16 +695,6 @@
     def getlength(self):
         return self.shape[0] * self.getitemsize()
 
-    def as_str(self):
-        slicelen = self.shape[0]
-        return self.getslice(0, slicelen * self.step, self.step, slicelen)
-
-    def as_str_and_offset_maybe(self):
-        string, offset = self.buf.as_str_and_offset_maybe()
-        if string is not None:
-            return string, offset + self.offset
-        return None, 0
-
     def getitem(self, index):
         return self.buf.getitem(self.offset + index)
 

From pypy.commits at gmail.com  Thu Mar 30 12:42:49 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 30 Mar 2017 09:42:49 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer: Split ArrayBuffer class into a
 BinaryBuffer and a Py_buffer
Message-ID: <58dd3589.194c2e0a.25332.2997@mx.google.com>

Author: Ronan Lamy 
Branch: PyBuffer
Changeset: r90874:4e09a31a43fe
Date: 2017-03-30 17:42 +0100
http://bitbucket.org/pypy/pypy/changeset/4e09a31a43fe/

Log:	Split ArrayBuffer class into a BinaryBuffer and a Py_buffer

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -6,7 +6,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
-from pypy.interpreter.buffer import Buffer
+from pypy.interpreter.buffer import Buffer, BinaryBuffer
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import (
@@ -257,7 +257,7 @@
             lltype.free(oldbuffer, flavor='raw')
 
     def buffer_w(self, space, flags):
-        return ArrayBuffer(self, False)
+        return ArrayBuffer(ArrayData(self), self.typecode, self.itemsize, False)
 
     def descr_append(self, space, w_x):
         """ append(x)
@@ -846,60 +846,89 @@
     v.typecode = k
 unroll_typecodes = unrolling_iterable(types.keys())
 
+class ArrayData(BinaryBuffer):
+    _immutable_ = True
+    readonly = False
+    def __init__(self, w_array):
+        self.w_array = w_array
+
+    def getlength(self):
+        return self.w_array.len * self.w_array.itemsize
+
+    def getitem(self, index):
+        w_array = self.w_array
+        data = w_array._charbuf_start()
+        char = data[index]
+        w_array._charbuf_stop()
+        return char
+
+    def setitem(self, index, char):
+        w_array = self.w_array
+        data = w_array._charbuf_start()
+        data[index] = char
+        w_array._charbuf_stop()
+
+    def getslice(self, start, stop, step, size):
+        if size == 0:
+            return ''
+        assert step == 1
+        data = self.w_array._charbuf_start()
+        try:
+            return rffi.charpsize2str(rffi.ptradd(data, start), size)
+        finally:
+            self.w_array._charbuf_stop()
+
+    def get_raw_address(self):
+        return self.w_array._charbuf_start()
+
+
 class ArrayBuffer(Buffer):
     _immutable_ = True
 
-    def __init__(self, array, readonly):
-        self.array = array
+    def __init__(self, data, fmt, itemsize, readonly):
+        self.data = data
+        self.fmt = fmt
+        self.itemsize = itemsize
         self.readonly = readonly
 
     def getlength(self):
-        return self.array.len * self.array.itemsize
+        return self.data.getlength()
 
     def as_str(self):
-        return self.getslice(0, self.getlength(), 1, self.getlength())
+        return self.data.as_str()
+
+    def getitem(self, index):
+        return self.data.getitem(index)
+
+    def setitem(self, index, value):
+        return self.data.setitem(index, value)
+
+    def getslice(self, start, stop, step, size):
+        return self.data.getslice(start, stop, step, size)
+
+    def setslice(self, start, string):
+        self.data.setslice(start, string)
 
     def getformat(self):
-        return self.array.typecode
+        return self.fmt
 
     def getitemsize(self):
-        return self.array.itemsize
+        return self.itemsize
 
     def getndim(self):
         return 1
 
     def getshape(self):
-        return [self.array.len]
+        return [self.getlength() // self.itemsize]
 
     def getstrides(self):
         return [self.getitemsize()]
 
-    def getitem(self, index):
-        array = self.array
-        data = array._charbuf_start()
-        char = data[index]
-        array._charbuf_stop()
-        return char
+    def get_raw_address(self):
+        return self.data.get_raw_address()
 
-    def setitem(self, index, char):
-        array = self.array
-        data = array._charbuf_start()
-        data[index] = char
-        array._charbuf_stop()
-
-    def getslice(self, start, stop, step, size):
-        if size == 0:
-            return ''
-        if step == 1:
-            data = self.array._charbuf_start()
-            try:
-                return rffi.charpsize2str(rffi.ptradd(data, start), size)
-            finally:
-                self.array._charbuf_stop()
-        return Buffer.getslice(self, start, stop, step, size)
-
-    def get_raw_address(self):
-        return self.array._charbuf_start()
+    def as_binary(self):
+        return self.data
 
 
 unpack_driver = jit.JitDriver(name='unpack_array',

From pypy.commits at gmail.com  Thu Mar 30 14:55:42 2017
From: pypy.commits at gmail.com (planrich)
Date: Thu, 30 Mar 2017 11:55:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix windows translation bug at the
 beginning
Message-ID: <58dd54ae.c4052e0a.97a9d.2ef7@mx.google.com>

Author: planrich
Branch: 
Changeset: r90875:eba581c00543
Date: 2017-03-30 18:54 +0000
http://bitbucket.org/pypy/pypy/changeset/eba581c00543/

Log:	fix windows translation bug at the beginning

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -16,12 +16,13 @@
 SRC = ROOT.join('src')
 SHARED = SRC.join('shared')
 BACKTRACE = SHARED.join('libbacktrace')
-MSIINTTYPES = SHARED.join('msiinttypes')
 
-extra_include_dirs = []
 compile_extra = ['-DRPYTHON_VMPROF', '-O3']
+separate_module_files = [
+    SHARED.join('symboltable.c')
+]
 if sys.platform.startswith('linux'):
-    separate_module_files = [
+    separate_module_files += [
        BACKTRACE.join('backtrace.c'),
        BACKTRACE.join('state.c'),
        BACKTRACE.join('elf.c'),
@@ -38,26 +39,24 @@
 elif sys.platform == 'darwin':
     compile_extra += ['-DVMPROF_UNIX']
     compile_extra += ['-DVMPROF_MAC']
-    separate_module_files = []
     _libs = []
 else:
     # windows
     compile_extra += ['-DVMPROF_WINDOWS']
-    extra_include_dirs += [MSIINTTYPES]
     separate_module_files = [SHARED.join('vmprof_main_win32.c')]
     _libs = []
 
 
 eci_kwds = dict(
-    include_dirs = [SRC, SHARED, BACKTRACE] + extra_include_dirs,
+    include_dirs = [SRC, SHARED, BACKTRACE],
     includes = ['rvmprof.h','vmprof_stack.h'],
     libraries = _libs,
     separate_module_files = [
         SRC.join('rvmprof.c'),
         SHARED.join('compat.c'),
         SHARED.join('machine.c'),
-        SHARED.join('symboltable.c'),
         SHARED.join('vmp_stack.c'),
+        # symbol table already in separate_module_files
     ] + separate_module_files,
     post_include_bits=[],
     compile_extra=compile_extra
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -49,9 +49,11 @@
 PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag) { return NULL; }
 #endif
 
-static int vmp_native_traces_enabled = 0;
+#ifdef VMP_SUPPORTS_NATIVE_PROFILING
 static intptr_t *vmp_ranges = NULL;
 static ssize_t vmp_range_count = 0;
+static int vmp_native_traces_enabled = 0;
+#endif
 static int _vmp_profiles_lines = 0;
 
 void vmp_profile_lines(int lines) {
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -70,4 +70,3 @@
 
 #endif
 
-
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -75,6 +75,7 @@
         return "out of memory";
 #if VMPROF_UNIX
     current_codes = NULL;
+    assert(fd >= 0);
 #else
     if (memory) {
         return "memory tracking only supported on unix";
@@ -83,7 +84,6 @@
         return "native profiling only supported on unix";
     }
 #endif
-    assert(fd >= 0);
     vmp_set_profile_fileno(fd);
     if (opened_profile(interp_name, memory, proflines, native) < 0) {
         vmp_set_profile_fileno(0);
@@ -181,7 +181,8 @@
                               intptr_t *result_p, intptr_t result_length)
 {
     int n;
-#ifdef _WIN32
+    int enabled;
+#ifdef VMPROF_WINDOWS
     intptr_t pc = 0;   /* XXX implement me */
 #else
     intptr_t pc = ucontext ? (intptr_t)GetPC((ucontext_t *)ucontext) : 0;
@@ -189,7 +190,7 @@
     if (stack == NULL) {
         stack = get_vmprof_stack();
     }
-    int enabled = vmp_native_enabled();
+    enabled = vmp_native_enabled();
     vmp_native_disable();
     n = get_stack_trace(stack, result_p, result_length - 2, pc);
     if (enabled) {
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_get_custom_offset.h
@@ -1,6 +1,10 @@
 #pragma once
 
+#ifdef VMPROF_WINDOWS
+#include "msiinttypes/stdint.h"
+#else
 #include 
+#endif
 
 void *pypy_find_codemap_at_addr(long addr, long *start_addr);
 long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.c
@@ -18,6 +18,7 @@
 {
     int res;
     int fd;
+    int count;
 
     res = WaitForSingleObject(write_mutex, INFINITE);
     fd = vmp_profile_fileno();
@@ -27,7 +28,7 @@
         return -1;
     }
     while (bufsize > 0) {
-        ssize_t count = _write(fd, buf, (long)bufsize);
+        count = _write(fd, buf, (long)bufsize);
         if (count <= 0) {
             ReleaseMutex(write_mutex);
             return -1;   /* failed */
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
@@ -19,7 +19,7 @@
 
 int vmp_write_all(const char *buf, size_t bufsize);
 
-#ifdef VMPROF_RPYTHON
+#ifdef RPYTHON_VMPROF
 typedef struct pypy_threadlocal_s PY_WIN_THREAD_STATE;
 #else
 typedef PyThreadState PY_WIN_THREAD_STATE;
@@ -89,7 +89,7 @@
 #endif
 }
 
-#ifdef RPYTHON_VMPROF
+#ifndef RPYTHON_VMPROF
 static
 PY_WIN_THREAD_STATE * get_current_thread_state(void)
 {
@@ -188,3 +188,16 @@
 {
     enabled = !ignored;
 }
+
+int vmp_native_enable(void) {
+    return 0;
+}
+
+void vmp_native_disable(void) {
+}
+
+int get_stack_trace(PY_WIN_THREAD_STATE * current, void** result,
+		    int max_depth, intptr_t pc)
+{
+    return 0;
+}

From pypy.commits at gmail.com  Thu Mar 30 14:58:48 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 Mar 2017 11:58:48 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: add test for exception
 ordering on failed import (issue 2524)
Message-ID: <58dd5568.84452e0a.71967.2e5e@mx.google.com>

Author: Matti Picus 
Branch: release-pypy2.7-5.x
Changeset: r90876:4f78163b13c0
Date: 2017-03-30 21:57 +0300
http://bitbucket.org/pypy/pypy/changeset/4f78163b13c0/

Log:	add test for exception ordering on failed import (issue 2524)
	(grafted from e7aebde27cfc6859eb65e83e40cc17a47b91be1d)

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -361,7 +361,12 @@
         assert sys == o
 
     def test_import_fromlist_must_not_contain_unicodes(self):
-        raises(TypeError, __import__, 'encodings', None, None, [u'xxx'])
+        exc = raises(TypeError, __import__, 'encodings', None, None, [u'xxx'])
+        assert 'not a string' in exc.value.message
+        exc = raises(TypeError, __import__, 'encodings', None, None, [123])
+        assert 'not a string' in exc.value.message
+        # issue 2524
+        raises(ImportError, __import__, 'xxxbadmodule', fromlist=[u'xx']) 
 
     def test_import_relative_back_to_absolute2(self):
         from pkg import abs_x_y

From pypy.commits at gmail.com  Thu Mar 30 14:58:50 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 Mar 2017 11:58:50 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: delay checking fromlist
 values till later, allows ImportError to happen first
Message-ID: <58dd556a.0e472e0a.7b741.2d09@mx.google.com>

Author: Matti Picus 
Branch: release-pypy2.7-5.x
Changeset: r90877:1aa2d8e03cdf
Date: 2017-03-30 21:57 +0300
http://bitbucket.org/pypy/pypy/changeset/1aa2d8e03cdf/

Log:	delay checking fromlist values till later, allows ImportError to
	happen first (grafted from 9df4aced5d6eb8b642b0527fb0bd6b6b423c4214)

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -362,11 +362,6 @@
         # to get a better trace. if it is unwrapped, the immutability of the
         # tuple is lost
         length = space.len_w(w_fromlist)
-        for i in range(length):
-            w_name = space.getitem(w_fromlist, space.newint(i))
-            if not space.isinstance_w(w_name, space.w_text):
-                raise oefmt(space.w_TypeError,
-                    "'fromlist' items must be str, not %T", w_name)
         if w_path is not None:
             if length == 1 and space.eq_w(
                     space.getitem(w_fromlist, space.newint(0)),
@@ -384,6 +379,9 @@
             if w_fromlist is not None:
                 for i in range(length):
                     w_name = space.getitem(w_fromlist, space.newint(i))
+                    if not space.isinstance_w(w_name, space.w_text):
+                        raise oefmt(space.w_TypeError,
+                            "'Item in ``fromlist'' not a string")
                     if try_getattr(space, w_mod, w_name) is None:
                         return None
         return w_mod
@@ -430,6 +428,9 @@
             if w_fromlist is not None:
                 for i in range(length):
                     w_name = space.getitem(w_fromlist, space.newint(i))
+                    if not space.isinstance_w(w_name, space.w_text):
+                        raise oefmt(space.w_TypeError,
+                            "'Item in ``fromlist'' not a string")
                     if try_getattr(space, w_mod, w_name) is None:
                         load_part(space, w_path, prefix, space.text0_w(w_name),
                                   w_mod, tentative=1)

From pypy.commits at gmail.com  Thu Mar 30 15:33:43 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 12:33:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: copy over changes from vmprof-python
 repo 34ec7ccd4
Message-ID: <58dd5d97.1bd8190a.4896e.3106@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90878:d567917b86fe
Date: 2017-03-30 15:33 -0400
http://bitbucket.org/pypy/pypy/changeset/d567917b86fe/

Log:	copy over changes from vmprof-python repo 34ec7ccd4

diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
@@ -51,7 +51,7 @@
 #ifdef RPYTHON_LL2CTYPES
     return 0; // not much we can do
 #else
-#ifndef RPY_TLOFS_thread_ident
+#if !defined(RPY_TLOFS_thread_ident) && defined(RPYTHON_VMPROF)
     return 0; // we can't freeze threads, unsafe
 #else
     hThread = OpenThread(THREAD_ALL_ACCESS, FALSE, thread_id);

From pypy.commits at gmail.com  Thu Mar 30 16:59:50 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 13:59:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: variable p has been renamed to tstate
Message-ID: <58dd71c6.12212e0a.411ca.31e9@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90879:dc4691e34ae0
Date: 2017-03-30 16:59 -0400
http://bitbucket.org/pypy/pypy/changeset/dc4691e34ae0/

Log:	variable p has been renamed to tstate

diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
@@ -142,7 +142,7 @@
         tstate = _RPython_ThreadLocals_Head(); // the first one is one behind head
         tstate = _RPython_ThreadLocals_Enum(tstate);
         while (tstate) {
-            if (p->ready == 42) {
+            if (tstate->ready == 42) {
                 depth = vmprof_snapshot_thread(tstate->thread_ident, tstate, stack);
                 if (depth > 0) {
                     _write_all((char*)stack + offsetof(prof_stacktrace_s, marker),

From pypy.commits at gmail.com  Thu Mar 30 17:01:20 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 14:01:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove -O3 command line switch on
 windows (removes lots of compiler warnings)
Message-ID: <58dd7220.46012e0a.d0b83.3287@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90880:c572d115dfc7
Date: 2017-03-30 17:00 -0400
http://bitbucket.org/pypy/pypy/changeset/c572d115dfc7/

Log:	remove -O3 command line switch on windows (removes lots of compiler
	warnings)

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -42,7 +42,7 @@
     _libs = []
 else:
     # windows
-    compile_extra += ['-DVMPROF_WINDOWS']
+    compile_extra = ['-DRPYTHON_VMPROF', '-DVMPROF_WINDOWS']
     separate_module_files = [SHARED.join('vmprof_main_win32.c')]
     _libs = []
 

From pypy.commits at gmail.com  Thu Mar 30 19:19:04 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 16:19:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: _write_all got vmp_ prefix
Message-ID: <58dd9268.1a4c2e0a.3e5d0.360a@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90881:0fdfe97b8ea5
Date: 2017-03-30 19:16 -0400
http://bitbucket.org/pypy/pypy/changeset/0fdfe97b8ea5/

Log:	_write_all got vmp_ prefix

diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
@@ -145,7 +145,7 @@
             if (tstate->ready == 42) {
                 depth = vmprof_snapshot_thread(tstate->thread_ident, tstate, stack);
                 if (depth > 0) {
-                    _write_all((char*)stack + offsetof(prof_stacktrace_s, marker),
+                    vmp_write_all((char*)stack + offsetof(prof_stacktrace_s, marker),
                          depth * sizeof(void *) +
                          sizeof(struct prof_stacktrace_s) -
                          offsetof(struct prof_stacktrace_s, marker));

From pypy.commits at gmail.com  Thu Mar 30 19:19:06 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 30 Mar 2017 16:19:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: only define RPY_EXPORTED if it has not
 already been defined
Message-ID: <58dd926a.9814190a.3f6de.3647@mx.google.com>

Author: Richard Plangger 
Branch: 
Changeset: r90882:cc6d8f1ca9bc
Date: 2017-03-30 19:18 -0400
http://bitbucket.org/pypy/pypy/changeset/cc6d8f1ca9bc/

Log:	only define RPY_EXPORTED if it has not already been defined

diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -16,7 +16,9 @@
 #define RPY_EXTERN RPY_EXPORTED
 #endif
 #ifdef _WIN32
+#ifndef RPY_EXPORTED
 #define RPY_EXPORTED __declspec(dllexport)
+#endif
 #else
 #define RPY_EXPORTED  extern __attribute__((visibility("default")))
 #endif

From pypy.commits at gmail.com  Fri Mar 31 11:05:45 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 31 Mar 2017 08:05:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: make error message compatible with
 cpython 2.7.13
Message-ID: <58de7049.4920190a.62aeb.54be@mx.google.com>

Author: mattip 
Branch: 
Changeset: r90883:add48f6012a1
Date: 2017-03-31 18:04 +0300
http://bitbucket.org/pypy/pypy/changeset/add48f6012a1/

Log:	make error message compatible with cpython 2.7.13

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -381,7 +381,7 @@
                     w_name = space.getitem(w_fromlist, space.newint(i))
                     if not space.isinstance_w(w_name, space.w_text):
                         raise oefmt(space.w_TypeError,
-                            "'Item in ``fromlist'' not a string")
+                            "'Item in ``fromlist'' must be str, not %T", w_name)
                     if try_getattr(space, w_mod, w_name) is None:
                         return None
         return w_mod
@@ -430,7 +430,7 @@
                     w_name = space.getitem(w_fromlist, space.newint(i))
                     if not space.isinstance_w(w_name, space.w_text):
                         raise oefmt(space.w_TypeError,
-                            "'Item in ``fromlist'' not a string")
+                            "'Item in ``fromlist'' must be str, not %T", w_name)
                     if try_getattr(space, w_mod, w_name) is None:
                         load_part(space, w_path, prefix, space.text0_w(w_name),
                                   w_mod, tentative=1)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -361,10 +361,14 @@
         assert sys == o
 
     def test_import_fromlist_must_not_contain_unicodes(self):
+        import sys
+        ver = sys.version_info
         exc = raises(TypeError, __import__, 'encodings', None, None, [u'xxx'])
-        assert 'not a string' in exc.value.message
+        if ver > (2, 7, 12):
+            assert 'must be str' in exc.value.message
         exc = raises(TypeError, __import__, 'encodings', None, None, [123])
-        assert 'not a string' in exc.value.message
+        if ver > (2, 7, 12):
+            assert 'must be str' in exc.value.message
         # issue 2524
         raises(ImportError, __import__, 'xxxbadmodule', fromlist=[u'xx'])
         mod = __import__('collections', fromlist=[u'defaultdict'])

From pypy.commits at gmail.com  Fri Mar 31 11:05:48 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 31 Mar 2017 08:05:48 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: make error message
 compatible with cpython 2.7.13
Message-ID: <58de704c.82e9190a.d4a67.5656@mx.google.com>

Author: mattip 
Branch: release-pypy2.7-5.x
Changeset: r90884:14f400f0498e
Date: 2017-03-31 18:04 +0300
http://bitbucket.org/pypy/pypy/changeset/14f400f0498e/

Log:	make error message compatible with cpython 2.7.13 (grafted from
	add48f6012a1702abb34076e9fd8a93a11020457)

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -381,7 +381,7 @@
                     w_name = space.getitem(w_fromlist, space.newint(i))
                     if not space.isinstance_w(w_name, space.w_text):
                         raise oefmt(space.w_TypeError,
-                            "'Item in ``fromlist'' not a string")
+                            "'Item in ``fromlist'' must be str, not %T", w_name)
                     if try_getattr(space, w_mod, w_name) is None:
                         return None
         return w_mod
@@ -430,7 +430,7 @@
                     w_name = space.getitem(w_fromlist, space.newint(i))
                     if not space.isinstance_w(w_name, space.w_text):
                         raise oefmt(space.w_TypeError,
-                            "'Item in ``fromlist'' not a string")
+                            "'Item in ``fromlist'' must be str, not %T", w_name)
                     if try_getattr(space, w_mod, w_name) is None:
                         load_part(space, w_path, prefix, space.text0_w(w_name),
                                   w_mod, tentative=1)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -361,10 +361,14 @@
         assert sys == o
 
     def test_import_fromlist_must_not_contain_unicodes(self):
+        import sys
+        ver = sys.version_info
         exc = raises(TypeError, __import__, 'encodings', None, None, [u'xxx'])
-        assert 'not a string' in exc.value.message
+        if ver > (2, 7, 12):
+            assert 'must be str' in exc.value.message
         exc = raises(TypeError, __import__, 'encodings', None, None, [123])
-        assert 'not a string' in exc.value.message
+        if ver > (2, 7, 12):
+            assert 'must be str' in exc.value.message
         # issue 2524
         raises(ImportError, __import__, 'xxxbadmodule', fromlist=[u'xx']) 
 

From pypy.commits at gmail.com  Fri Mar 31 13:04:45 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 31 Mar 2017 10:04:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add failing test (fixed on branch
 PyBuffer)
Message-ID: <58de8c2d.12532e0a.60be3.5b38@mx.google.com>

Author: Ronan Lamy 
Branch: py3.5
Changeset: r90885:66ea6bdef550
Date: 2017-03-31 18:04 +0100
http://bitbucket.org/pypy/pypy/changeset/66ea6bdef550/

Log:	Add failing test (fixed on branch PyBuffer)

diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,8 +56,10 @@
         assert list(w) == [97]
         v[::2] = b'ABC'
         assert data == bytearray(eval("b'AbBeCg'"))
-        assert v[::2].tobytes() == b'ABC'
-        assert v[::-2].tobytes() == b'geb'
+        w = v[::2]
+        assert w.tobytes() == bytes(w) == b'ABC'
+        w = v[::-2]
+        assert w.tobytes() == bytes(w) == b'geb'
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)