From pypy.commits at gmail.com  Thu Sep  1 01:49:35 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 31 Aug 2016 22:49:35 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: this was wrong. if arr.base
 and dtype='O', simply assume the pointers are gc objects
Message-ID: <57c7c16f.a427c20a.76643.179b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86807:6ee997b509c7
Date: 2016-09-01 08:45 +0300
http://bitbucket.org/pypy/pypy/changeset/6ee997b509c7/

Log:	this was wrong. if arr.base and dtype='O', simply assume the
	pointers are gc objects

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -465,13 +465,6 @@
         self.storage = storage
         self.start = start
         self.gcstruct = V_OBJECTSTORE
-        if dtype.num == NPY.OBJECT:
-            self.gcstruct = _create_objectstore(storage, self.size,
-                                                dtype.elsize)
-
-    def __del__(self):
-        if self.gcstruct:
-            self.gcstruct.length = 0
 
     def fill(self, space, box):
         self.dtype.itemtype.fill(
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1851,7 +1851,7 @@
                     arr.gcstruct)
 
     def read(self, arr, i, offset, dtype):
-        if arr.gcstruct is V_OBJECTSTORE:
+        if arr.gcstruct is V_OBJECTSTORE and not arr.base():
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot read object from array with no gc hook")
         return self.box(self._read(arr.storage, i, offset))

From pypy.commits at gmail.com  Thu Sep  1 01:49:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 31 Aug 2016 22:49:37 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: modifying ctor.py,
 might as well add a test for pypy/numpy issues #52, #53
Message-ID: <57c7c171.2916c20a.8b371.1988@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86808:3b99b5a021ee
Date: 2016-09-01 08:48 +0300
http://bitbucket.org/pypy/pypy/changeset/3b99b5a021ee/

Log:	modifying ctor.py, might as well add a test for pypy/numpy issues
	#52, #53

diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -702,3 +702,32 @@
         ret = obj.sum()
         print type(ret)
         assert ret.info == 'spam'
+
+    def test_ndarray_subclass_assigns_base(self):
+        import numpy as np
+        init_called = []
+        class _DummyArray(object):
+            """ Dummy object that just exists to hang __array_interface__ dictionaries
+            and possibly keep alive a reference to a base array.
+            """
+            def __init__(self, interface, base=None):
+                self.__array_interface__ = interface
+                init_called.append(1)
+                self.base = base
+
+        x = np.zeros(10)
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 1
+        assert y.base is d
+
+        x = np.zeros((0,), dtype='float32')
+        intf = x.__array_interface__.copy()
+        intf["strides"] = x.strides
+        x.__array_interface__["strides"] = x.strides
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 2
+        assert y.base is d
+
+

From pypy.commits at gmail.com  Thu Sep  1 03:35:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 00:35:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a passing test that direct running
 and annotation both crash
Message-ID: <57c7da5b.0575c20a.329f0.f1f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86809:ce916c69cf00
Date: 2016-09-01 09:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ce916c69cf00/

Log:	Add a passing test that direct running and annotation both crash if
	we give a float to an llexternal function expecting an int

diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -38,6 +38,24 @@
         xf = self.compile(f, [])
         assert xf() == 8+3
 
+    def test_no_float_to_int_conversion(self):
+        c_source = py.code.Source("""
+        int someexternalfunction(int x)
+        {
+            return (x + 3);
+        }
+        """)
+
+        eci = ExternalCompilationInfo(separate_module_sources=[c_source])
+        z = llexternal('someexternalfunction', [Signed], Signed,
+                       compilation_info=eci)
+
+        def f():
+            return z(8.2)
+
+        py.test.raises(TypeError, f)
+        py.test.raises(TypeError, self.compile, f, [])
+
     def test_hashdefine(self):
         h_source = """
         #define X(i) (i+3)

From pypy.commits at gmail.com  Thu Sep  1 03:43:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 00:43:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Another passing test
Message-ID: <57c7dc3b.e16ec20a.59a7a.37ef@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86810:809a3f78b64f
Date: 2016-09-01 09:43 +0200
http://bitbucket.org/pypy/pypy/changeset/809a3f78b64f/

Log:	Another passing test

diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -29,6 +29,7 @@
     assert msg == fnname
     msg = read_message(f)
     assert msg == args
+    assert [type(x) for x in msg] == [type(x) for x in args]
     if isinstance(result, Exception):
         write_exception(g, result)
     else:

From pypy.commits at gmail.com  Thu Sep  1 03:50:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 00:50:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: pfff another hack on top of this pile
 of hacks
Message-ID: <57c7dddb.d4301c0a.97158.2b96@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86811:76e37b5f30ae
Date: 2016-09-01 09:50 +0200
http://bitbucket.org/pypy/pypy/changeset/76e37b5f30ae/

Log:	pfff another hack on top of this pile of hacks

diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -170,11 +170,15 @@
         #
         # haaaack for 'pypy py.test -A' if libm.so is a linker script
         # (see reason in _dlerror_on_dlopen_untranslated())
+        must_free = False
         if not we_are_translated() and platform.name == "linux":
             if name and rffi.charp2str(name) == 'libm.so':
-                name = rffi.str2charp('libm.so.6', track_allocation=False)
+                name = rffi.str2charp('libm.so.6')
+                must_free = True
         #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
+        if must_free:
+            rffi.free_charp(name)
         if not res:
             if not we_are_translated():
                 err = _dlerror_on_dlopen_untranslated(name)

From pypy.commits at gmail.com  Thu Sep  1 04:45:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 01:45:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2388: the problem is obscure
 interaction with a different call (I
Message-ID: <57c7eab3.12331c0a.ee8e2.8538@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86812:7d6c66b14770
Date: 2016-09-01 10:26 +0200
http://bitbucket.org/pypy/pypy/changeset/7d6c66b14770/

Log:	Issue #2388: the problem is obscure interaction with a different
	call (I don't know which one) with the signature (string, float),
	which was considered as more general than the signature (string,
	int) of os.access().

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -17,7 +17,7 @@
 from rpython.flowspace.model import Variable, Constant, const
 from rpython.flowspace.operation import op
 from rpython.rlib import rarithmetic
-from rpython.annotator.model import AnnotatorError
+from rpython.annotator.model import AnnotatorError, TLS
 
 BINARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values()
                         if oper.dispatch == 2])
@@ -436,6 +436,11 @@
 class __extend__(pairtype(SomeFloat, SomeFloat)):
 
     def union((flt1, flt2)):
+        if not TLS.allow_int_to_float:
+            # in this mode, if one of the two is actually the
+            # subclass SomeInteger, complain
+            if isinstance(flt1, SomeInteger) or isinstance(flt2, SomeInteger):
+                raise UnionError(flt1, flt2)
         return SomeFloat()
 
     add = sub = mul = union
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -44,6 +44,7 @@
     # A global attribute :-(  Patch it with 'True' to enable checking of
     # the no_nul attribute...
     check_str_without_nul = False
+    allow_int_to_float = True
 TLS = State()
 
 class SomeObject(object):
diff --git a/rpython/rlib/rmarshal.py b/rpython/rlib/rmarshal.py
--- a/rpython/rlib/rmarshal.py
+++ b/rpython/rlib/rmarshal.py
@@ -346,11 +346,15 @@
     # on s_bigger.  It relies on the fact that s_bigger was created with
     # an expression like 'annotation([s_item])' which returns a ListDef with
     # no bookkeeper, on which side-effects are not allowed.
+    saved = annmodel.TLS.allow_int_to_float
     try:
+        annmodel.TLS.allow_int_to_float = False
         s_union = annmodel.unionof(s_bigger, s_smaller)
         return s_bigger.contains(s_union)
     except (annmodel.UnionError, TooLateForChange):
         return False
+    finally:
+        annmodel.TLS.allow_int_to_float = saved
 
 
 class __extend__(pairtype(MTag, annmodel.SomeObject)):
diff --git a/rpython/rlib/test/test_rmarshal.py b/rpython/rlib/test/test_rmarshal.py
--- a/rpython/rlib/test/test_rmarshal.py
+++ b/rpython/rlib/test/test_rmarshal.py
@@ -128,10 +128,12 @@
 
 def test_llinterp_marshal():
     from rpython.rtyper.test.test_llinterp import interpret
-    marshaller = get_marshaller([(int, str, float)])
+    marshaller1 = get_marshaller([(int, str, float)])
+    marshaller2 = get_marshaller([(int, str, int)])
     def f():
         buf = []
-        marshaller(buf, [(5, "hello", -0.5), (7, "world", 1E100)])
+        marshaller1(buf, [(5, "hello", -0.5), (7, "world", 1E100)])
+        marshaller2(buf, [(5, "hello", 1)])
         return ''.join(buf)
     res = interpret(f, [])
     res = ''.join(res.chars)
@@ -139,14 +141,20 @@
         assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00'
                        's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00'
                        'i\x07\x00\x00\x00s\x05\x00\x00\x00world'
-                       'f\x061e+100')
+                       'f\x061e+100'
+                       '[\x01\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00'
+                       's\x05\x00\x00\x00helloi\x01\x00\x00\x00')
     else:
         assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00'
                        'I\x05\x00\x00\x00\x00\x00\x00\x00'
                        's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00'
                        'I\x07\x00\x00\x00\x00\x00\x00\x00'
                        's\x05\x00\x00\x00world'
-                       'f\x061e+100')
+                       'f\x061e+100'
+                       '[\x01\x00\x00\x00(\x03\x00\x00\x00'
+                       'I\x05\x00\x00\x00\x00\x00\x00\x00'
+                       's\x05\x00\x00\x00hello'
+                       'I\x01\x00\x00\x00\x00\x00\x00\x00')
 
 def test_llinterp_unmarshal():
     from rpython.rtyper.test.test_llinterp import interpret

From pypy.commits at gmail.com  Thu Sep  1 05:50:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 02:50:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Don't put an __init__ in a Test class,
 it makes py.test skip(!?!!) that class
Message-ID: <57c7f9d4.94a51c0a.bea70.9ff3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86813:8e7fe0c9ab70
Date: 2016-09-01 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/8e7fe0c9ab70/

Log:	Don't put an __init__ in a Test class, it makes py.test skip(!?!!)
	that class

diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -286,7 +286,7 @@
 
 
 class TestUTF8Decoding(UnicodeTests):
-    def __init__(self):
+    def setup_method(self, meth):
         self.decoder = self.getdecoder('utf-8')
 
     def to_bytestring(self, bytes):

From pypy.commits at gmail.com  Thu Sep  1 05:55:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 02:55:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: Rename the exception that we call
 TestException to some class name that
Message-ID: <57c7fb03.822e1c0a.bb2f3.f2a6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86814:955998701207
Date: 2016-09-01 11:02 +0100
http://bitbucket.org/pypy/pypy/changeset/955998701207/

Log:	Rename the exception that we call TestException to some class name
	that doesn't start with Test, which confuses py.test

diff --git a/rpython/translator/c/test/test_exception.py b/rpython/translator/c/test/test_exception.py
--- a/rpython/translator/c/test/test_exception.py
+++ b/rpython/translator/c/test/test_exception.py
@@ -9,7 +9,7 @@
 getcompiledopt = test_backendoptimized.TestTypedOptimizedTestCase().getcompiled
 
 
-class TestException(Exception):
+class InTestException(Exception):
     pass
 
 class MyException(Exception):
@@ -18,7 +18,7 @@
 def test_simple1():
     def raise_(i):
         if i == 0:
-            raise TestException()
+            raise InTestException()
         elif i == 1:
             raise MyException()
         else:
@@ -29,7 +29,7 @@
             b = raise_(i) + 12
             c = raise_(i) + 13
             return a+b+c
-        except TestException:
+        except InTestException:
             return 7
         except MyException:
             return 123

From pypy.commits at gmail.com  Thu Sep  1 05:55:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 02:55:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: Hack at our included py.test to fail,
 not quietly skip, when it sees a
Message-ID: <57c7fb05.87adc20a.c9daa.4b81@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86815:0ace73a20a81
Date: 2016-09-01 11:03 +0100
http://bitbucket.org/pypy/pypy/changeset/0ace73a20a81/

Log:	Hack at our included py.test to fail, not quietly skip, when it sees
	a TestXxx class with an __init__() method

diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))

From pypy.commits at gmail.com  Thu Sep  1 06:36:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 03:36:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: Move the bit checking inside helpers,
 share it from the two places
Message-ID: <57c804c6.c186c20a.99002.8175@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86817:ee3a2fbec01a
Date: 2016-09-01 12:36 +0200
http://bitbucket.org/pypy/pypy/changeset/ee3a2fbec01a/

Log:	Move the bit checking inside helpers, share it from the two places

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -137,6 +137,25 @@
                                  result=result)
     return result.build(), pos
 
+def _invalid_cont_byte(ordch):
+    return ordch>>6 != 0x2    # 0b10
+
+_invalid_byte_2_of_2 = _invalid_cont_byte
+_invalid_byte_3_of_3 = _invalid_cont_byte
+_invalid_byte_3_of_4 = _invalid_cont_byte
+_invalid_byte_4_of_4 = _invalid_cont_byte
+
+def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xe0 and ordch2 < 0xa0)
+            # surrogates shouldn't be valid UTF-8!
+            or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f))
+
+def _invalid_byte_2_of_4(ordch1, ordch2):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xf0 and ordch2 < 0x90) or
+            (ordch1 == 0xf4 and ordch2 > 0x8f))
+
 @specialize.argtype(6)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
@@ -173,10 +192,7 @@
             ordch2 = ord(s[pos+1])
             if n == 3:
                 # 3-bytes seq with only a continuation byte
-                if (ordch2>>6 != 0x2 or   # 0b10
-                    (ordch1 == 0xe0 and ordch2 < 0xa0)
-                or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f)
-                    ):
+                if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                     # second byte invalid, take the first and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
@@ -192,16 +208,14 @@
                     continue
             elif n == 4:
                 # 4-bytes seq with 1 or 2 continuation bytes
-                if (ordch2>>6 != 0x2 or    # 0b10
-                    (ordch1 == 0xf0 and ordch2 < 0x90) or
-                    (ordch1 == 0xf4 and ordch2 > 0x8f)):
+                if _invalid_byte_2_of_4(ordch1, ordch2):
                     # second byte invalid, take the first and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
                                           s, pos, pos+1)
                     result.append(r)
                     continue
-                elif charsleft == 2 and ord(s[pos+2])>>6 != 0x2:   # 0b10
+                elif charsleft == 2 and _invalid_byte_3_of_4(ord(s[pos+2])):
                     # third byte invalid, take the first two and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
@@ -228,7 +242,7 @@
 
         elif n == 2:
             ordch2 = ord(s[pos+1])
-            if ordch2>>6 != 0x2:   # 0b10
+            if _invalid_byte_2_of_2(ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
@@ -242,17 +256,13 @@
         elif n == 3:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
-            if (ordch2>>6 != 0x2 or    # 0b10
-                (ordch1 == 0xe0 and ordch2 < 0xa0)
-                # surrogates shouldn't be valid UTF-8!
-                or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f)
-                ):
+            if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif ordch3>>6 != 0x2:     # 0b10
+            elif _invalid_byte_3_of_3(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
@@ -268,21 +278,19 @@
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
             ordch4 = ord(s[pos+3])
-            if (ordch2>>6 != 0x2 or     # 0b10
-                (ordch1 == 0xf0 and ordch2 < 0x90) or
-                (ordch1 == 0xf4 and ordch2 > 0x8f)):
+            if _invalid_byte_2_of_4(ordch1, ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif ordch3>>6 != 0x2:     # 0b10
+            elif _invalid_byte_3_of_4(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
-            elif ordch4>>6 != 0x2:     # 0b10
+            elif _invalid_byte_4_of_4(ordch4):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+3)

From pypy.commits at gmail.com  Thu Sep  1 06:36:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 03:36:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2389: the custom error handler
 may return a 'pos' that is smaller
Message-ID: <57c804c4.e2efc20a.1a16b.83ef@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86816:e9dd5882eed6
Date: 2016-09-01 12:23 +0200
http://bitbucket.org/pypy/pypy/changeset/e9dd5882eed6/

Log:	Issue #2389: the custom error handler may return a 'pos' that is
	smaller than 'size', in which case we need to continue looping

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -157,22 +157,26 @@
         if pos + n > size:
             if not final:
                 break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
             charsleft = size - pos - 1 # either 0, 1, 2
-            # note: when we get the 'unexpected end of data' we don't care
-            # about the pos anymore and we just ignore the value
+            # note: when we get the 'unexpected end of data' we need
+            # to care about the pos returned; it can be lower than size,
+            # in case we need to continue running this loop
             if not charsleft:
                 # there's only the start byte and nothing else
                 r, pos = errorhandler(errors, 'utf8',
                                       'unexpected end of data',
                                       s, pos, pos+1)
                 result.append(r)
-                break
+                continue
             ordch2 = ord(s[pos+1])
             if n == 3:
                 # 3-bytes seq with only a continuation byte
                 if (ordch2>>6 != 0x2 or   # 0b10
-                    (ordch1 == 0xe0 and ordch2 < 0xa0)):
-                    # or (ordch1 == 0xed and ordch2 > 0x9f)
+                    (ordch1 == 0xe0 and ordch2 < 0xa0)
+                or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f)
+                    ):
                     # second byte invalid, take the first and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
@@ -185,7 +189,7 @@
                                       'unexpected end of data',
                                       s, pos, pos+2)
                     result.append(r)
-                    break
+                    continue
             elif n == 4:
                 # 4-bytes seq with 1 or 2 continuation bytes
                 if (ordch2>>6 != 0x2 or    # 0b10
@@ -210,7 +214,8 @@
                                       'unexpected end of data',
                                       s, pos, pos+charsleft+1)
                     result.append(r)
-                    break
+                    continue
+            raise AssertionError("unreachable")
 
         if n == 0:
             r, pos = errorhandler(errors, 'utf8',
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -289,6 +289,12 @@
     def setup_method(self, meth):
         self.decoder = self.getdecoder('utf-8')
 
+    def custom_replace(self, errors, encoding, msg, s, startingpos, endingpos):
+        assert errors == 'custom'
+        # returns FOO, but consumes only one character (not up to endingpos)
+        FOO = u'\u1234'
+        return FOO, startingpos + 1
+
     def to_bytestring(self, bytes):
         return ''.join(chr(int(c, 16)) for c in bytes.split())
 
@@ -309,6 +315,7 @@
         E.g. <80> is a continuation byte and can appear only after a start byte.
         """
         FFFD = u'\ufffd'
+        FOO = u'\u1234'
         for byte in '\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF':
             py.test.raises(UnicodeDecodeError, self.decoder, byte, 1, None, final=True)
             self.checkdecodeerror(byte, 'utf-8', 0, 1, addstuff=False,
@@ -320,6 +327,11 @@
             assert self.decoder(byte, 1, 'ignore', final=True) == (u'', 1)
             assert (self.decoder('aaaa' + byte + 'bbbb', 9, 'ignore',
                         final=True) == (u'aaaabbbb', 9))
+            assert self.decoder(byte, 1, 'custom', final=True,
+                        errorhandler=self.custom_replace) == (FOO, 1)
+            assert (self.decoder('aaaa' + byte + 'bbbb', 9, 'custom',
+                        final=True, errorhandler=self.custom_replace) ==
+                        (u'aaaa'+ FOO + u'bbbb', 9))
 
     def test_unexpected_end_of_data(self):
         """
@@ -343,6 +355,7 @@
             'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF'
         ]
         FFFD = u'\ufffd'
+        FOO = u'\u1234'
         for seq in sequences:
             seq = self.to_bytestring(seq)
             py.test.raises(UnicodeDecodeError, self.decoder, seq, len(seq),
@@ -358,6 +371,12 @@
                                 ) == (u'', len(seq))
             assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'ignore',
                         final=True) == (u'aaaabbbb', len(seq) + 8))
+            assert (self.decoder(seq, len(seq), 'custom', final=True,
+                        errorhandler=self.custom_replace) == 
+                        (FOO * len(seq), len(seq)))
+            assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'custom',
+                        final=True, errorhandler=self.custom_replace) ==
+                        (u'aaaa'+ FOO * len(seq) + u'bbbb', len(seq) + 8))
 
     def test_invalid_cb_for_2bytes_seq(self):
         """

From pypy.commits at gmail.com  Thu Sep  1 11:11:09 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Sep 2016 08:11:09 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: implement view of
 __array_interface__, fixes issue #52, #53 on pypy/numpy
Message-ID: <57c8450d.87adc20a.c9daa.da07@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86818:4916eb438de5
Date: 2016-09-01 16:50 +0300
http://bitbucket.org/pypy/pypy/changeset/4916eb438de5/

Log:	implement view of __array_interface__, fixes issue #52, #53 on
	pypy/numpy

diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -82,13 +82,18 @@
             raise oefmt(space.w_ValueError,
                     "__array_interface__ could not decode dtype %R", w_dtype
                     )
-        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
+        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
+                                   space.isinstance_w(w_data, space.w_list)):
             data_w = space.listview(w_data)
             w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
-            read_only = True # XXX why not space.is_true(data_w[1])
+            read_only = space.is_true(data_w[1])
             offset = 0
+            w_base = w_object
+            if read_only:
+                w_base = None
             return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
-                                    dtype, strides=strides, start=offset), read_only
+                                dtype, w_base=w_base, strides=strides,
+                                start=offset), read_only
         if w_data is None:
             w_data = w_object
         w_offset = space.finditem(w_interface, space.wrap('offset'))

From pypy.commits at gmail.com  Thu Sep  1 11:11:11 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Sep 2016 08:11:11 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: extend and fix the
 already-existing test for __array_interfac__
Message-ID: <57c8450f.262ec20a.f163a.f81a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86819:8c587f75370f
Date: 2016-09-01 17:56 +0300
http://bitbucket.org/pypy/pypy/changeset/8c587f75370f/

Log:	extend and fix the already-existing test for __array_interfac__

diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -43,7 +43,7 @@
         raise oefmt(space.w_ValueError,
                     "object __array__ method not producing an array")
 
-def try_interface_method(space, w_object):
+def try_interface_method(space, w_object, copy):
     try:
         w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
         if w_interface is None:
@@ -86,7 +86,7 @@
                                    space.isinstance_w(w_data, space.w_list)):
             data_w = space.listview(w_data)
             w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
-            read_only = space.is_true(data_w[1])
+            read_only = space.is_true(data_w[1]) or copy
             offset = 0
             w_base = w_object
             if read_only:
@@ -203,7 +203,7 @@
             # use buffer interface
             w_object = _array_from_buffer_3118(space, w_object, dtype)
     if not isinstance(w_object, W_NDimArray):
-        w_array, _copy = try_interface_method(space, w_object)
+        w_array, _copy = try_interface_method(space, w_object, copy)
         if w_array is not None:
             w_object = w_array
             copy = _copy
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3215,7 +3215,9 @@
         raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)}))
 
         a = array([1, 2, 3])
-        b = array(Dummy(a.__array_interface__))
+        d = Dummy(a.__array_interface__)
+        b = array(d)
+        assert b.base is None
         b[1] = 200
         assert a[1] == 2 # upstream compatibility, is this a bug?
         interface_a = a.__array_interface__
@@ -3226,6 +3228,8 @@
         interface_b.pop('data')
         interface_a.pop('data')
         assert interface_a == interface_b
+        b = array(d, copy=False)
+        assert b.base is d
 
         b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
                          'data': 'a'*100}))

From pypy.commits at gmail.com  Thu Sep  1 11:45:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 01 Sep 2016 08:45:16 -0700 (PDT)
Subject: [pypy-commit] pypy redirect-assembler-jitlog: new tag tmp_callback
 to correctly make the connection between call_assembler <-> trace
Message-ID: <57c84d0c.94071c0a.1c406.43e9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: redirect-assembler-jitlog
Changeset: r86821:a2f692a70b2d
Date: 2016-09-01 17:44 +0200
http://bitbucket.org/pypy/pypy/changeset/a2f692a70b2d/

Log:	new tag tmp_callback to correctly make the connection between
	call_assembler <-> trace

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -7,6 +7,7 @@
 from rpython.rlib import rstack
 from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside
 from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.rlib.objectmodel import compute_unique_id
 from rpython.conftest import option
 
 from rpython.jit.metainterp.resoperation import ResOperation, rop,\
@@ -1156,6 +1157,9 @@
     operations[1].setfailargs([])
     operations = get_deep_immutable_oplist(operations)
     cpu.compile_loop(inputargs, operations, jitcell_token, log=False)
+
+    jl.tmp_callback(looptoken)
+
     if memory_manager is not None:    # for tests
         memory_manager.keep_loop_alive(jitcell_token)
     return jitcell_token
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -212,7 +212,7 @@
         return method
     return decor
 
-JITLOG_VERSION = 3
+JITLOG_VERSION = 4
 JITLOG_VERSION_16BIT_LE = struct.pack("<H", JITLOG_VERSION)
 
 marks = [
@@ -245,6 +245,7 @@
     ('ABORT_TRACE',),
     ('SOURCE_CODE',),
     ('REDIRECT_ASSEMBLER',),
+    ('TMP_CALLBACK',),
 ]
 
 start = 0x11
@@ -320,6 +321,12 @@
     content = ''.join(list)
     jitlog_write_marked(content, len(content))
 
+def tmp_callback(looptoken):
+    mark_tmp_callback = ''.join([
+        jl.MARK_TMP_CALLBACK,
+        encode_le_addr(compute_unique_id(looptoken)),
+        encode_le_64bit(looptoken.number)])
+    jl.jitlog_write_marked(mark_tmp_callback, len(mark_tmp_callback))
 
 class JitLogger(object):
     def __init__(self, cpu=None):

From pypy.commits at gmail.com  Thu Sep  1 11:45:14 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 01 Sep 2016 08:45:14 -0700 (PDT)
Subject: [pypy-commit] pypy redirect-assembler-jitlog: merge default
Message-ID: <57c84d0a.d41a1c0a.26683.37ac@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: redirect-assembler-jitlog
Changeset: r86820:a8a56628fefe
Date: 2016-09-01 15:57 +0200
http://bitbucket.org/pypy/pypy/changeset/a8a56628fefe/

Log:	merge default

diff too long, truncating to 2000 out of 212851 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,6 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,115 +360,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
-
-  Heinrich-Heine University, Germany 
-  Open End AB (formerly AB Strakt), Sweden
-  merlinux GmbH, Germany 
-  tismerysoft GmbH, Germany 
-  Logilab Paris, France 
-  DFKI GmbH, Germany 
-  Impara, Germany
-  Change Maker, Sweden 
-  University of California Berkeley, USA
-  Google Inc.
-  King's College London
-
-The PyPy Logo as used by http://speed.pypy.org and others was created
-by Samuel Reis and is distributed on terms of Creative Commons Share Alike
-License.
- 
-License for 'lib-python/2.7'
-============================
-
-Except when otherwise stated (look for LICENSE files or copyright/license
-information at the beginning of each file) the files in the 'lib-python/2.7'
-directory are all copyrighted by the Python Software Foundation and licensed
-under the terms that you can find here: https://docs.python.org/2/license.html
-
-License for 'pypy/module/unicodedata/'
-======================================
-
-The following files are from the website of The Unicode Consortium
-at http://www.unicode.org/.  For the terms of use of these files, see
-http://www.unicode.org/terms_of_use.html .  Or they are derived from
-files from the above website, and the same terms of use apply.
-
-    CompositionExclusions-*.txt
-    EastAsianWidth-*.txt
-    LineBreak-*.txt
-    UnicodeData-*.txt
-    UnihanNumeric-*.txt
-
-License for 'dotviewer/font/'
-=============================
-
-Copyright (C) 2008 The Android Open Source Project
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  
-     http://www.apache.org/licenses/LICENSE-2.0
-  
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Detailed license information is contained in the NOTICE file in the
-directory.
-
-
-Licenses and Acknowledgements for Incorporated Software
-=======================================================
-
-This section is an incomplete, but growing list of licenses and
-acknowledgements for third-party software incorporated in the PyPy
-distribution.
-
-License for 'Tcl/Tk'
---------------------
-
-This copy of PyPy contains library code that may, when used, result in
-the Tcl/Tk library to be loaded.  PyPy also includes code that may be
-regarded as being a copy of some parts of the Tcl/Tk header files.
-You may see a copy of the License for Tcl/Tk in the file
-`lib_pypy/_tkinter/license.terms` included here.
-
-License for 'bzip2'
--------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
-
-    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
-
-License for 'openssl'
----------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-openssl library.  You may see a copy of the License for OpenSSL at
-
-    https://www.openssl.org/source/license.html
-
-License for 'gdbm'
-------------------
-
-The gdbm module includes code from gdbm.h, which is distributed under
-the terms of the GPL license version 2 or any later version.  Thus the
-gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
-under the terms of the GPL license as well.
-
-License for 'rpython/rlib/rvmprof/src'
---------------------------------------
-
-The code is based on gperftools. You may see a copy of the License for it at
-
-    https://github.com/gperftools/gperftools/blob/master/COPYING
+  werat
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -137,6 +137,8 @@
             lib.gdbm_sync(self.__ll_dbm)
 
 def open(filename, flags='r', mode=0666):
+    if isinstance(filename, unicode):
+        filename = filename.encode()
     if flags[0] == 'r':
         iflags = lib.GDBM_READER
     elif flags[0] == 'w':
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,218 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,152 +1,9 @@
 ==========================
-What's new in PyPy2.7 5.3+
+What's new in PyPy2.7 5.4+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
-
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
-
-
-.. branch: const-fold-we-are-jitted
-
-Reduce the size of the generated C code by constant-folding ``we_are_jitted``
-in non-jitcode.
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,165 @@
+=========================
+What's new in PyPy2.7 5.4
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -208,7 +208,8 @@
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self, 
+                                        space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
@@ -216,7 +217,8 @@
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
@@ -224,7 +226,8 @@
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
@@ -232,7 +235,8 @@
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -23,6 +23,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -84,7 +92,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -38,6 +38,8 @@
 
 
 class W_ClassObject(W_Root):
+    _immutable_fields_ = ['bases_w?[*]', 'w_dict?']
+
     def __init__(self, space, w_name, bases, w_dict):
         self.name = space.str_w(w_name)
         make_sure_not_resized(bases)
@@ -75,6 +77,7 @@
                             "__bases__ items must be classes")
         self.bases_w = bases_w
 
+    @jit.unroll_safe
     def is_subclass_of(self, other):
         assert isinstance(other, W_ClassObject)
         if self is other:
@@ -313,7 +316,7 @@
         # This method ignores the instance dict and the __getattr__.
         # Returns None if not found.
         assert isinstance(name, str)
-        w_value = self.w_class.lookup(space, name)
+        w_value = jit.promote(self.w_class).lookup(space, name)
         if w_value is None:
             return None
         w_descr_get = space.lookup(w_value, '__get__')
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -250,6 +250,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -358,9 +358,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -460,7 +466,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -612,7 +618,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -151,6 +151,7 @@
 
     def test_readValues(self):
         from _winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from _winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -164,7 +165,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,8 +120,8 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
 for name in constant_names:
@@ -649,6 +649,7 @@
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
@@ -976,8 +977,10 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
     add_fork_hook('child', reinit_tls)
 
 def init_function(func):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,13 +1,17 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer)
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
+def PyObject_CheckBuffer(space, pyobj):
     """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
+    as_buffer = pyobj.c_ob_type.c_tp_as_buffer
+    flags = pyobj.c_ob_type.c_tp_flags
+    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
+        return 1
+    return 0  
 
 @cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
              rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -123,5 +123,4 @@
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
     space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname))
-    importing.exec_code_module(space, w_mod, code)
-    return w_mod
+    return importing.exec_code_module(space, w_mod, code, w_name)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.1-alpha0"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -12,7 +12,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    raise NotImplementedError
+    raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -3,15 +3,16 @@
 import re
 
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl)
+    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    readbufferproc, ssizessizeobjargproc)
+    readbufferproc, getbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -22,6 +23,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
@@ -298,11 +302,23 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj):
+    def __init__(self, ptr, size, w_obj, format='B', shape=None,
+                strides=None, ndim=1, itemsize=1, readonly=True):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        self.readonly = True
+        self.format = format
+        if not shape:
+            self.shape = [size]
+        else:
+            self.shape = shape
+        if not strides:
+            self.strides = [1]
+        else:
+            self.strides = strides
+        self.ndim = ndim 
+        self.itemsize = itemsize
+        self.readonly = readonly
 
     def getlength(self):
         return self.size
@@ -313,6 +329,15 @@
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
+    def getformat(self):
+        return self.format
+
+    def getshape(self):
+        return self.shape
+
+    def getitemsize(self):
+        return self.itemsize
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
@@ -322,6 +347,30 @@
             space.fromcache(State).check_and_raise_exception(always=True)
         return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
 
+def wrap_getbuffer(space, w_self, w_args, func):
+    func_target = rffi.cast(getbufferproc, func)
+    with lltype.scoped_alloc(Py_buffer) as pybuf:
+        _flags = 0
+        if space.len_w(w_args) > 0:
+            _flags = space.int_w(space.listview(w_args)[0])
+        flags = rffi.cast(rffi.INT_real,_flags)
+        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+        if widen(size) < 0:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        ptr = pybuf.c_buf
+        size = pybuf.c_len
+        ndim = widen(pybuf.c_ndim)
+        shape =   [pybuf.c_shape[i]   for i in range(ndim)]
+        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_format:
+            format = rffi.charp2str(pybuf.c_format)
+        else:
+            format = 'B'
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
+                            ndim=ndim, shape=shape, strides=strides,
+                            itemsize=pybuf.c_itemsize,
+                            readonly=widen(pybuf.c_readonly)))
+
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
         func_target = rffi.cast(richcmpfunc, func)
@@ -486,7 +535,6 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         api_func = slot_tp_getattro.api_func
-
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -542,6 +590,21 @@
                              w_stararg=w_args, w_starstararg=w_kwds)
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
+    elif name == 'tp_as_buffer.c_bf_getbuffer':
+        buff_fn = w_type.getdictvalue(space, '__buffer__')
+        if buff_fn is None:
+            return
+        @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
+                rffi.INT_real, header=None, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def buff_w(space, w_self, pybuf, flags):
+            # XXX this is wrong, needs a test
+            raise oefmt(space.w_NotImplemented, 
+                "calling bf_getbuffer on a builtin type not supported yet")
+            #args = Arguments(space, [w_self],
+            #                 w_stararg=w_args, w_starstararg=w_kwds)
+            #return space.call_args(space.get(buff_fn, w_self), args)
+        api_func = buff_w.api_func
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
@@ -850,11 +913,19 @@
 slotdefs = eval(slotdefs_str)
 # PyPy addition
 slotdefs += (
-    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    # XXX that might not be what we want!
+    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""),
 )
 
+if not PY3:
+    slotdefs += (
+        TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    )
+
+
 # partial sort to solve some slot conflicts:
 # Number slots before Mapping slots before Sequence slots.
+# also prefer the new buffer interface
 # These are the only conflicts between __name__ methods
 def slotdef_sort_key(slotdef):
     if slotdef.slot_name.startswith('tp_as_number'):
@@ -863,6 +934,10 @@
         return 2
     if slotdef.slot_name.startswith('tp_as_sequence'):
         return 3
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
+        return 100
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
+        return 101
     return 0
 slotdefs = sorted(slotdefs, key=slotdef_sort_key)
 
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -0,0 +1,248 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
+#include <Python.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/* 
+ * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol,
+ * which is copyright Jake Vanderplas and released under the BSD license
+ */
+
+/* Structure defines a 1-dimensional strided array */
+typedef struct{
+    int* arr;
+    Py_ssize_t length;
+} MyArray;
+
+/* initialize the array with integers 0...length */
+void initialize_MyArray(MyArray* a, long length){
+    int i;
+    a->length = length;
+    a->arr = (int*)malloc(length * sizeof(int));
+    for(i=0; i<length; i++){
+        a->arr[i] = i;
+    }
+}
+
+/* free the memory when finished */
+void deallocate_MyArray(MyArray* a){
+    free(a->arr);
+    a->arr = NULL;
+}
+
+/* tools to print the array */
+char* stringify(MyArray* a, int nmax){
+    char* output = (char*) malloc(nmax * 20);
+    int k, pos = sprintf(&output[0], "[");
+
+    for (k=0; k < a->length && k < nmax; k++){
+        pos += sprintf(&output[pos], " %d", a->arr[k]);
+    }
+    if(a->length > nmax)
+        pos += sprintf(&output[pos], "...");
+    sprintf(&output[pos], " ]");
+    return output;
+}
+
+void print_MyArray(MyArray* a, int nmax){
+    char* s = stringify(a, nmax);
+    printf("%s", s);
+    free(s);
+}
+
+/* This is where we define the PyMyArray object structure */
+typedef struct {
+    PyObject_HEAD
+    /* Type-specific fields go below. */
+    MyArray arr;
+} PyMyArray;
+
+
+/* This is the __init__ function, implemented in C */
+static int
+PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
+{
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
+    // init may have already been called
+    if (self->arr.arr != NULL) {
+        deallocate_MyArray(&self->arr);
+    }
+
+    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
+        return -1;
+
+    if (length < 0)
+        length = 0;
+
+    initialize_MyArray(&self->arr, length);
+
+    return 0;
+}
+
+
+/* this function is called when the object is deallocated */
+static void
+PyMyArray_dealloc(PyMyArray* self)
+{
+    deallocate_MyArray(&self->arr);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+
+/* This function returns the string representation of our object */
+static PyObject *
+PyMyArray_str(PyMyArray * self)
+{
+  char* s = stringify(&self->arr, 10);
+  PyObject* ret = PyUnicode_FromString(s);
+  free(s);
+  return ret;
+}
+
+/* Here is the buffer interface function */
+static int
+PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
+{
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
+  if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
+    PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
+    return -1;
+  }
+  if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
+    PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
+    return -1;
+  }
+
+  view->obj = (PyObject*)self;
+  view->buf = (void*)self->arr.arr;
+  view->len = self->arr.length * sizeof(int);
+  view->readonly = 0;
+  view->itemsize = sizeof(int);
+  view->format = "i";  // integer
+  view->ndim = 1;
+  view->shape = &self->arr.length;  // length-1 sequence of dimensions
+  view->strides = &view->itemsize;  // for the simple case we can do this
+  view->suboffsets = NULL;
+  view->internal = NULL;
+
+  Py_INCREF(self);  // need to increase the reference count
+  return 0;
+}
+
+static PyBufferProcs PyMyArray_as_buffer = {
+#if PY_MAJOR_VERSION < 3
+  (readbufferproc)0,
+  (writebufferproc)0,
+  (segcountproc)0,
+  (charbufferproc)0,
+#endif
+  (getbufferproc)PyMyArray_getbuffer,
+  (releasebufferproc)0,  // we do not require any special release function
+};
+
+
+/* Here is the type structure: we put the above functions in the appropriate place
+   in order to actually define the Python object type */
+static PyTypeObject PyMyArrayType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "pymyarray.PyMyArray",        /* tp_name */
+    sizeof(PyMyArray),            /* tp_basicsize */
+    0,                            /* tp_itemsize */
+    (destructor)PyMyArray_dealloc,/* tp_dealloc */
+    0,                            /* tp_print */
+    0,                            /* tp_getattr */
+    0,                            /* tp_setattr */
+    0,                            /* tp_reserved */
+    (reprfunc)PyMyArray_str,      /* tp_repr */
+    0,                            /* tp_as_number */
+    0,                            /* tp_as_sequence */
+    0,                            /* tp_as_mapping */
+    0,                            /* tp_hash  */
+    0,                            /* tp_call */
+    (reprfunc)PyMyArray_str,      /* tp_str */
+    0,                            /* tp_getattro */
+    0,                            /* tp_setattro */
+    &PyMyArray_as_buffer,         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */
+    "PyMyArray object",           /* tp_doc */
+    0,                            /* tp_traverse */
+    0,                            /* tp_clear */
+    0,                            /* tp_richcompare */
+    0,                            /* tp_weaklistoffset */
+    0,                            /* tp_iter */
+    0,                            /* tp_iternext */
+    0,                            /* tp_methods */
+    0,                            /* tp_members */
+    0,                            /* tp_getset */
+    0,                            /* tp_base */
+    0,                            /* tp_dict */
+    0,                            /* tp_descr_get */
+    0,                            /* tp_descr_set */
+    0,                            /* tp_dictoffset */
+    (initproc)PyMyArray_init,     /* tp_init */
+};
+
+static PyMethodDef buffer_functions[] = {
+    {NULL,        NULL}    /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "buffer_test",
+    "Module Doc",
+    -1,
+    buffer_functions,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+};
+#define INITERROR return NULL
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+PyInit_buffer_test(void)
+
+#else
+
+#define INITERROR return
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+#endif
+
+PyMODINIT_FUNC
+initbuffer_test(void)
+#endif
+{
+#if PY_MAJOR_VERSION >= 3
+    PyObject *m= PyModule_Create(&moduledef);
+#else
+    PyObject *m= Py_InitModule("buffer_test", buffer_functions);
+#endif
+    if (m == NULL)
+        INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&PyMyArrayType) < 0)
+        INITERROR;
+    Py_INCREF(&PyMyArrayType);
+    PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType);
+#if PY_MAJOR_VERSION >=3
+    return m;
+#endif
+}
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -87,4 +87,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
@@ -145,6 +151,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -433,21 +457,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,17 +1,26 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
+        assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
+        w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
-    @pytest.mark.skipif(True, reason='write a test for this')
-    def test_get_base_and_get_buffer(self, space, api):
-        assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+    def test_buffer_protocol(self):
+        import struct
+        module = self.import_module(name='buffer_test')
+        arr = module.PyMyArray(10)
+        y = memoryview(arr)
+        assert y.format == 'i'
+        assert y.shape == (10,)
+        s = y[3]
+        assert len(s) == struct.calcsize('i')
+        assert s == struct.pack('i', 3)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,9 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr)
+    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -385,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -608,6 +612,7 @@
                 bf_getwritebuffer.api_func.get_wrapper(space))
     pto.c_tp_as_buffer = c_buf
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
+    pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def type_dealloc(space, obj):
@@ -774,6 +779,8 @@
             pto.c_tp_setattro = base.c_tp_setattro
         if not pto.c_tp_getattro:
             pto.c_tp_getattro = base.c_tp_getattro
+        if not pto.c_tp_as_buffer:
+            pto.c_tp_as_buffer = base.c_tp_as_buffer
     finally:
         Py_DecRef(space, base_pyo)
 
@@ -810,8 +817,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -5,6 +5,7 @@
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
+from pypy.module.cpyext.api import Py_bufferP
 
 
 P, FT, PyO = Ptr, FuncType, PyObject
@@ -58,8 +59,7 @@
 writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
 segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
 charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
-## We don't support new buffer interface for now
-getbufferproc = rffi.VOIDP
+getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = rffi.VOIDP
 
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -597,6 +597,11 @@
 
 @jit.dont_look_inside
 def load_module(space, w_modulename, find_info, reuse=False):
+    """Like load_module() in CPython's import.c, this will normally
+    make a module object, store it in sys.modules, execute code in it,
+    and then fetch it again from sys.modules.  But this logic is not
+    used if we're calling a PEP302 loader.
+    """
     if find_info is None:
         return
 
@@ -625,17 +630,15 @@
 
         try:
             if find_info.modtype == PY_SOURCE:
-                load_source_module(
+                return load_source_module(
                     space, w_modulename, w_mod,
                     find_info.filename, find_info.stream.readall(),
                     find_info.stream.try_to_find_file_descriptor())
-                return w_mod
             elif find_info.modtype == PY_COMPILED:
                 magic = _r_long(find_info.stream)
                 timestamp = _r_long(find_info.stream)
-                load_compiled_module(space, w_modulename, w_mod, find_info.filename,
+                return load_compiled_module(space, w_modulename, w_mod, find_info.filename,
                                      magic, timestamp, find_info.stream.readall())
-                return w_mod
             elif find_info.modtype == PKG_DIRECTORY:
                 w_path = space.newlist([space.wrap(find_info.filename)])
                 space.setattr(w_mod, space.wrap('__path__'), w_path)
@@ -644,14 +647,13 @@
                 if find_info is None:
                     return w_mod
                 try:
-                    load_module(space, w_modulename, find_info, reuse=True)
+                    w_mod = load_module(space, w_modulename, find_info,
+                                        reuse=True)
                 finally:
                     try:
                         find_info.stream.close()
                     except StreamErrors:
                         pass
-                # fetch the module again, in case of "substitution"
-                w_mod = check_sys_modules(space, w_modulename)
                 return w_mod
             elif find_info.modtype == C_EXTENSION and has_so_extension(space):
                 load_c_extension(space, find_info.filename, space.str_w(w_modulename))
@@ -677,13 +679,6 @@
         try:
             if find_info:
                 w_mod = load_module(space, w_modulename, find_info)
-                try:
-                    w_mod = space.getitem(space.sys.get("modules"),
-                                          w_modulename)
-                except OperationError as oe:
-                    if not oe.match(space, space.w_KeyError):
-                        raise
-                    raise OperationError(space.w_ImportError, w_modulename)
                 if w_parent is not None:
                     space.setattr(w_parent, space.wrap(partname), w_mod)
                 return w_mod
@@ -875,20 +870,32 @@
     pycode = ec.compiler.compile(source, pathname, 'exec', 0)
     return pycode
 
-def exec_code_module(space, w_mod, code_w):
+def exec_code_module(space, w_mod, code_w, w_modulename, check_afterwards=True):
+    """
+    Execute a code object in the module's dict.  Returns
+    'sys.modules[modulename]', which must exist.
+    """
     w_dict = space.getattr(w_mod, space.wrap('__dict__'))
     space.call_method(w_dict, 'setdefault',
                       space.wrap('__builtins__'),
                       space.wrap(space.builtin))
     code_w.exec_code(space, w_dict, w_dict)
 
+    if check_afterwards:
+        w_mod = check_sys_modules(space, w_modulename)
+        if w_mod is None:
+            raise oefmt(space.w_ImportError,

From pypy.commits at gmail.com  Thu Sep  1 12:00:54 2016
From: pypy.commits at gmail.com (vext01)
Date: Thu, 01 Sep 2016 09:00:54 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Fix translation.
Message-ID: <57c850b6.c41f1c0a.c732d.4489@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86822:f777170ec79b
Date: 2016-09-01 17:00 +0100
http://bitbucket.org/pypy/pypy/changeset/f777170ec79b/

Log:	Fix translation.

diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -746,20 +746,20 @@
     def set_pages_executable(addr, size):
         assert lltype.typeOf(addr) == rffi.CCHARP
         assert isinstance(size, int)
-        #assert size >= 0
         rv = mprotect(addr, size, PROT_EXEC | PROT_READ)
         if int(rv) < 0:
             from rpython.rlib import debug
             debug.fatalerror_notb("set_pages_executable failed")
+    set_pages_executable._annenforceargs_ = (None, int)
 
     def set_pages_writable(addr, size):
         assert lltype.typeOf(addr) == rffi.CCHARP
         assert isinstance(size, int)
-        #assert size >= 0
         rv = mprotect(addr, size, PROT_WRITE | PROT_READ)
         if int(rv) < 0:
             from rpython.rlib import debug
             debug.fatalerror_notb("set_pages_writable failed")
+    set_pages_writable._annenforceargs_ = (None, int)
 
     def clear_large_memory_chunk_aligned(addr, map_size):
         addr = rffi.cast(PTR, addr)

From pypy.commits at gmail.com  Thu Sep  1 13:40:26 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 01 Sep 2016 10:40:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: Improve error message when attempting
 to modify annotations in a fixed graph
Message-ID: <57c8680a.262ec20a.f163a.308c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86823:30767c452330
Date: 2016-09-01 18:39 +0100
http://bitbucket.org/pypy/pypy/changeset/30767c452330/

Log:	Improve error message when attempting to modify annotations in a
	fixed graph

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -164,8 +164,13 @@
             # annotations that are passed in, and don't annotate the old
             # graph -- it's already low-level operations!
             for a, s_newarg in zip(block.inputargs, cells):
-                s_oldarg = self.binding(a)
-                assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
+                s_oldarg = a.annotation
+                if not s_oldarg.contains(s_newarg):
+                    raise annmodel.AnnotatorError(
+                        "Late-stage annotation is not allowed to modify the "
+                        "existing annotation for variable %s: %s" %
+                            (a, s_oldarg))
+
         else:
             assert not self.frozen
             if block not in self.annotated:

From pypy.commits at gmail.com  Thu Sep  1 15:19:37 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 01 Sep 2016 12:19:37 -0700 (PDT)
Subject: [pypy-commit] pypy redirect-assembler-jitlog: translation issue,
 assigning unique number to tmp_callback jit cell token
Message-ID: <57c87f49.cb7f1c0a.fa27d.922f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: redirect-assembler-jitlog
Changeset: r86824:cd97133d768d
Date: 2016-09-01 21:18 +0200
http://bitbucket.org/pypy/pypy/changeset/cd97133d768d/

Log:	translation issue, assigning unique number to tmp_callback jit cell
	token

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -540,9 +540,9 @@
             looptoken._x86_ops_offset = ops_offset
         looptoken._ll_function_addr = rawstart + functionpos
 
-        if logger:
-            log = logger.log_trace(jl.MARK_TRACE_ASM, None, self.mc)
-            log.write(inputargs, operations, ops_offset=ops_offset)
+        if log and logger:
+            l = logger.log_trace(jl.MARK_TRACE_ASM, None, self.mc)
+            l.write(inputargs, operations, ops_offset=ops_offset)
 
             # legacy
             if logger.logger_ops:
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1122,6 +1122,11 @@
     version of the code may end up replacing it.
     """
     jitcell_token = make_jitcell_token(jitdriver_sd)
+    #
+    logger = jitdriver_sd.metainterp_sd.jitlog
+    jitcell_token.number = logger.next_id()
+    jl.tmp_callback(jitcell_token)
+    #
     nb_red_args = jitdriver_sd.num_red_args
     assert len(redargtypes) == nb_red_args
     inputargs = []
@@ -1158,8 +1163,6 @@
     operations = get_deep_immutable_oplist(operations)
     cpu.compile_loop(inputargs, operations, jitcell_token, log=False)
 
-    jl.tmp_callback(looptoken)
-
     if memory_manager is not None:    # for tests
         memory_manager.keep_loop_alive(jitcell_token)
     return jitcell_token
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -350,6 +350,10 @@
     def finish(self):
         jitlog_teardown()
 
+    def next_id(self):
+        self.trace_id += 1
+        return self.trace_id
+
     def start_new_trace(self, metainterp_sd, faildescr=None, entry_bridge=False, jd_name=""):
         # even if the logger is not enabled, increment the trace id
         self.trace_id += 1

From pypy.commits at gmail.com  Thu Sep  1 17:45:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 14:45:19 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57c8a16f.8f8e1c0a.41a0.1e2d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r784:8cea640f2073
Date: 2016-09-01 23:45 +0200
http://bitbucket.org/pypy/pypy.org/changeset/8cea640f2073/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 61.8
+      value: 61.9
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $64931 of $105000 (61.8%)
+   $64959 of $105000 (61.9%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5400 left
+    we have $5426 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Thu Sep  1 18:03:29 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 01 Sep 2016 15:03:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix translation and add a warning in
 annmodel.unionof
Message-ID: <57c8a5b1.87d11c0a.2592e.27bb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86825:82980a978280
Date: 2016-09-01 23:02 +0100
http://bitbucket.org/pypy/pypy/changeset/82980a978280/

Log:	Fix translation and add a warning in annmodel.unionof

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -165,7 +165,9 @@
             # graph -- it's already low-level operations!
             for a, s_newarg in zip(block.inputargs, cells):
                 s_oldarg = a.annotation
-                if not s_oldarg.contains(s_newarg):
+                # XXX: Should use s_oldarg.contains(s_newarg) but that breaks
+                # PyPy translation
+                if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg:
                     raise annmodel.AnnotatorError(
                         "Late-stage annotation is not allowed to modify the "
                         "existing annotation for variable %s: %s" %
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -750,6 +750,7 @@
                 s1 = pair(s1, s2).union()
     else:
         # this is just a performance shortcut
+        # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
         if s1 != s2:
             s1 = pair(s1, s2).union()
     return s1

From pypy.commits at gmail.com  Thu Sep  1 19:03:28 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 01 Sep 2016 16:03:28 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Merge
Message-ID: <57c8b3c0.c3f0c20a.54624.9728@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86826:5649730037a2
Date: 2016-09-01 15:17 -0400
http://bitbucket.org/pypy/pypy/changeset/5649730037a2/

Log:	Merge

diff too long, truncating to 2000 out of 213792 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,6 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,115 +360,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
-
-  Heinrich-Heine University, Germany 
-  Open End AB (formerly AB Strakt), Sweden
-  merlinux GmbH, Germany 
-  tismerysoft GmbH, Germany 
-  Logilab Paris, France 
-  DFKI GmbH, Germany 
-  Impara, Germany
-  Change Maker, Sweden 
-  University of California Berkeley, USA
-  Google Inc.
-  King's College London
-
-The PyPy Logo as used by http://speed.pypy.org and others was created
-by Samuel Reis and is distributed on terms of Creative Commons Share Alike
-License.
- 
-License for 'lib-python/2.7'
-============================
-
-Except when otherwise stated (look for LICENSE files or copyright/license
-information at the beginning of each file) the files in the 'lib-python/2.7'
-directory are all copyrighted by the Python Software Foundation and licensed
-under the terms that you can find here: https://docs.python.org/2/license.html
-
-License for 'pypy/module/unicodedata/'
-======================================
-
-The following files are from the website of The Unicode Consortium
-at http://www.unicode.org/.  For the terms of use of these files, see
-http://www.unicode.org/terms_of_use.html .  Or they are derived from
-files from the above website, and the same terms of use apply.
-
-    CompositionExclusions-*.txt
-    EastAsianWidth-*.txt
-    LineBreak-*.txt
-    UnicodeData-*.txt
-    UnihanNumeric-*.txt
-
-License for 'dotviewer/font/'
-=============================
-
-Copyright (C) 2008 The Android Open Source Project
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  
-     http://www.apache.org/licenses/LICENSE-2.0
-  
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Detailed license information is contained in the NOTICE file in the
-directory.
-
-
-Licenses and Acknowledgements for Incorporated Software
-=======================================================
-
-This section is an incomplete, but growing list of licenses and
-acknowledgements for third-party software incorporated in the PyPy
-distribution.
-
-License for 'Tcl/Tk'
---------------------
-
-This copy of PyPy contains library code that may, when used, result in
-the Tcl/Tk library to be loaded.  PyPy also includes code that may be
-regarded as being a copy of some parts of the Tcl/Tk header files.
-You may see a copy of the License for Tcl/Tk in the file
-`lib_pypy/_tkinter/license.terms` included here.
-
-License for 'bzip2'
--------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
-
-    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
-
-License for 'openssl'
----------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-openssl library.  You may see a copy of the License for OpenSSL at
-
-    https://www.openssl.org/source/license.html
-
-License for 'gdbm'
-------------------
-
-The gdbm module includes code from gdbm.h, which is distributed under
-the terms of the GPL license version 2 or any later version.  Thus the
-gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
-under the terms of the GPL license as well.
-
-License for 'rpython/rlib/rvmprof/src'
---------------------------------------
-
-The code is based on gperftools. You may see a copy of the License for it at
-
-    https://github.com/gperftools/gperftools/blob/master/COPYING
+  werat
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -137,6 +137,8 @@
             lib.gdbm_sync(self.__ll_dbm)
 
 def open(filename, flags='r', mode=0666):
+    if isinstance(filename, unicode):
+        filename = filename.encode()
     if flags[0] == 'r':
         iflags = lib.GDBM_READER
     elif flags[0] == 'w':
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,218 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,146 +1,9 @@
 ==========================
-What's new in PyPy2.7 5.3+
+What's new in PyPy2.7 5.4+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
-
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,165 @@
+=========================
+What's new in PyPy2.7 5.4
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -208,7 +208,8 @@
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self, 
+                                        space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
@@ -216,7 +217,8 @@
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
@@ -224,7 +226,8 @@
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
@@ -232,7 +235,8 @@
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -23,6 +23,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -84,7 +92,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -38,6 +38,8 @@
 
 
 class W_ClassObject(W_Root):
+    _immutable_fields_ = ['bases_w?[*]', 'w_dict?']
+
     def __init__(self, space, w_name, bases, w_dict):
         self.name = space.str_w(w_name)
         make_sure_not_resized(bases)
@@ -75,6 +77,7 @@
                             "__bases__ items must be classes")
         self.bases_w = bases_w
 
+    @jit.unroll_safe
     def is_subclass_of(self, other):
         assert isinstance(other, W_ClassObject)
         if self is other:
@@ -313,7 +316,7 @@
         # This method ignores the instance dict and the __getattr__.
         # Returns None if not found.
         assert isinstance(name, str)
-        w_value = self.w_class.lookup(space, name)
+        w_value = jit.promote(self.w_class).lookup(space, name)
         if w_value is None:
             return None
         w_descr_get = space.lookup(w_value, '__get__')
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -250,6 +250,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -358,9 +358,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -460,7 +466,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -612,7 +618,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -151,6 +151,7 @@
 
     def test_readValues(self):
         from _winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from _winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -164,7 +165,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,8 +120,8 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
 for name in constant_names:
@@ -649,6 +649,7 @@
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
@@ -976,8 +977,10 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
     add_fork_hook('child', reinit_tls)
 
 def init_function(func):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,13 +1,17 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer)
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
+def PyObject_CheckBuffer(space, pyobj):
     """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
+    as_buffer = pyobj.c_ob_type.c_tp_as_buffer
+    flags = pyobj.c_ob_type.c_tp_flags
+    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
+        return 1
+    return 0  
 
 @cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
              rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -123,5 +123,4 @@
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
     space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname))
-    importing.exec_code_module(space, w_mod, code)
-    return w_mod
+    return importing.exec_code_module(space, w_mod, code, w_name)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.4.1-alpha0"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -12,7 +12,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    raise NotImplementedError
+    raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -3,15 +3,16 @@
 import re
 
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl)
+    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    readbufferproc, ssizessizeobjargproc)
+    readbufferproc, getbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -22,6 +23,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
@@ -298,11 +302,23 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj):
+    def __init__(self, ptr, size, w_obj, format='B', shape=None,
+                strides=None, ndim=1, itemsize=1, readonly=True):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        self.readonly = True
+        self.format = format
+        if not shape:
+            self.shape = [size]
+        else:
+            self.shape = shape
+        if not strides:
+            self.strides = [1]
+        else:
+            self.strides = strides
+        self.ndim = ndim 
+        self.itemsize = itemsize
+        self.readonly = readonly
 
     def getlength(self):
         return self.size
@@ -313,6 +329,15 @@
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
+    def getformat(self):
+        return self.format
+
+    def getshape(self):
+        return self.shape
+
+    def getitemsize(self):
+        return self.itemsize
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
@@ -322,6 +347,30 @@
             space.fromcache(State).check_and_raise_exception(always=True)
         return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
 
+def wrap_getbuffer(space, w_self, w_args, func):
+    func_target = rffi.cast(getbufferproc, func)
+    with lltype.scoped_alloc(Py_buffer) as pybuf:
+        _flags = 0
+        if space.len_w(w_args) > 0:
+            _flags = space.int_w(space.listview(w_args)[0])
+        flags = rffi.cast(rffi.INT_real,_flags)
+        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+        if widen(size) < 0:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        ptr = pybuf.c_buf
+        size = pybuf.c_len
+        ndim = widen(pybuf.c_ndim)
+        shape =   [pybuf.c_shape[i]   for i in range(ndim)]
+        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_format:
+            format = rffi.charp2str(pybuf.c_format)
+        else:
+            format = 'B'
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
+                            ndim=ndim, shape=shape, strides=strides,
+                            itemsize=pybuf.c_itemsize,
+                            readonly=widen(pybuf.c_readonly)))
+
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
         func_target = rffi.cast(richcmpfunc, func)
@@ -486,7 +535,6 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         api_func = slot_tp_getattro.api_func
-
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -542,6 +590,21 @@
                              w_stararg=w_args, w_starstararg=w_kwds)
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
+    elif name == 'tp_as_buffer.c_bf_getbuffer':
+        buff_fn = w_type.getdictvalue(space, '__buffer__')
+        if buff_fn is None:
+            return
+        @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
+                rffi.INT_real, header=None, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def buff_w(space, w_self, pybuf, flags):
+            # XXX this is wrong, needs a test
+            raise oefmt(space.w_NotImplemented, 
+                "calling bf_getbuffer on a builtin type not supported yet")
+            #args = Arguments(space, [w_self],
+            #                 w_stararg=w_args, w_starstararg=w_kwds)
+            #return space.call_args(space.get(buff_fn, w_self), args)
+        api_func = buff_w.api_func
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
@@ -850,11 +913,19 @@
 slotdefs = eval(slotdefs_str)
 # PyPy addition
 slotdefs += (
-    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    # XXX that might not be what we want!
+    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""),
 )
 
+if not PY3:
+    slotdefs += (
+        TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    )
+
+
 # partial sort to solve some slot conflicts:
 # Number slots before Mapping slots before Sequence slots.
+# also prefer the new buffer interface
 # These are the only conflicts between __name__ methods
 def slotdef_sort_key(slotdef):
     if slotdef.slot_name.startswith('tp_as_number'):
@@ -863,6 +934,10 @@
         return 2
     if slotdef.slot_name.startswith('tp_as_sequence'):
         return 3
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
+        return 100
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
+        return 101
     return 0
 slotdefs = sorted(slotdefs, key=slotdef_sort_key)
 
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -0,0 +1,248 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
+#include <Python.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/* 
+ * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol,
+ * which is copyright Jake Vanderplas and released under the BSD license
+ */
+
+/* Structure defines a 1-dimensional strided array */
+typedef struct{
+    int* arr;
+    Py_ssize_t length;
+} MyArray;
+
+/* initialize the array with integers 0...length */
+void initialize_MyArray(MyArray* a, long length){
+    int i;
+    a->length = length;
+    a->arr = (int*)malloc(length * sizeof(int));
+    for(i=0; i<length; i++){
+        a->arr[i] = i;
+    }
+}
+
+/* free the memory when finished */
+void deallocate_MyArray(MyArray* a){
+    free(a->arr);
+    a->arr = NULL;
+}
+
+/* tools to print the array */
+char* stringify(MyArray* a, int nmax){
+    char* output = (char*) malloc(nmax * 20);
+    int k, pos = sprintf(&output[0], "[");
+
+    for (k=0; k < a->length && k < nmax; k++){
+        pos += sprintf(&output[pos], " %d", a->arr[k]);
+    }
+    if(a->length > nmax)
+        pos += sprintf(&output[pos], "...");
+    sprintf(&output[pos], " ]");
+    return output;
+}
+
+void print_MyArray(MyArray* a, int nmax){
+    char* s = stringify(a, nmax);
+    printf("%s", s);
+    free(s);
+}
+
+/* This is where we define the PyMyArray object structure */
+typedef struct {
+    PyObject_HEAD
+    /* Type-specific fields go below. */
+    MyArray arr;
+} PyMyArray;
+
+
+/* This is the __init__ function, implemented in C */
+static int
+PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
+{
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
+    // init may have already been called
+    if (self->arr.arr != NULL) {
+        deallocate_MyArray(&self->arr);
+    }
+
+    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
+        return -1;
+
+    if (length < 0)
+        length = 0;
+
+    initialize_MyArray(&self->arr, length);
+
+    return 0;
+}
+
+
+/* this function is called when the object is deallocated */
+static void
+PyMyArray_dealloc(PyMyArray* self)
+{
+    deallocate_MyArray(&self->arr);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+
+/* This function returns the string representation of our object */
+static PyObject *
+PyMyArray_str(PyMyArray * self)
+{
+  char* s = stringify(&self->arr, 10);
+  PyObject* ret = PyUnicode_FromString(s);
+  free(s);
+  return ret;
+}
+
+/* Here is the buffer interface function */
+static int
+PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
+{
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
+  if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
+    PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
+    return -1;
+  }
+  if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
+    PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
+    return -1;
+  }
+
+  view->obj = (PyObject*)self;
+  view->buf = (void*)self->arr.arr;
+  view->len = self->arr.length * sizeof(int);
+  view->readonly = 0;
+  view->itemsize = sizeof(int);
+  view->format = "i";  // integer
+  view->ndim = 1;
+  view->shape = &self->arr.length;  // length-1 sequence of dimensions
+  view->strides = &view->itemsize;  // for the simple case we can do this
+  view->suboffsets = NULL;
+  view->internal = NULL;
+
+  Py_INCREF(self);  // need to increase the reference count
+  return 0;
+}
+
+static PyBufferProcs PyMyArray_as_buffer = {
+#if PY_MAJOR_VERSION < 3
+  (readbufferproc)0,
+  (writebufferproc)0,
+  (segcountproc)0,
+  (charbufferproc)0,
+#endif
+  (getbufferproc)PyMyArray_getbuffer,
+  (releasebufferproc)0,  // we do not require any special release function
+};
+
+
+/* Here is the type structure: we put the above functions in the appropriate place
+   in order to actually define the Python object type */
+static PyTypeObject PyMyArrayType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "pymyarray.PyMyArray",        /* tp_name */
+    sizeof(PyMyArray),            /* tp_basicsize */
+    0,                            /* tp_itemsize */
+    (destructor)PyMyArray_dealloc,/* tp_dealloc */
+    0,                            /* tp_print */
+    0,                            /* tp_getattr */
+    0,                            /* tp_setattr */
+    0,                            /* tp_reserved */
+    (reprfunc)PyMyArray_str,      /* tp_repr */
+    0,                            /* tp_as_number */
+    0,                            /* tp_as_sequence */
+    0,                            /* tp_as_mapping */
+    0,                            /* tp_hash  */
+    0,                            /* tp_call */
+    (reprfunc)PyMyArray_str,      /* tp_str */
+    0,                            /* tp_getattro */
+    0,                            /* tp_setattro */
+    &PyMyArray_as_buffer,         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */
+    "PyMyArray object",           /* tp_doc */
+    0,                            /* tp_traverse */
+    0,                            /* tp_clear */
+    0,                            /* tp_richcompare */
+    0,                            /* tp_weaklistoffset */
+    0,                            /* tp_iter */
+    0,                            /* tp_iternext */
+    0,                            /* tp_methods */
+    0,                            /* tp_members */
+    0,                            /* tp_getset */
+    0,                            /* tp_base */
+    0,                            /* tp_dict */
+    0,                            /* tp_descr_get */
+    0,                            /* tp_descr_set */
+    0,                            /* tp_dictoffset */
+    (initproc)PyMyArray_init,     /* tp_init */
+};
+
+static PyMethodDef buffer_functions[] = {
+    {NULL,        NULL}    /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "buffer_test",
+    "Module Doc",
+    -1,
+    buffer_functions,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+};
+#define INITERROR return NULL
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+PyInit_buffer_test(void)
+
+#else
+
+#define INITERROR return
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+#endif
+
+PyMODINIT_FUNC
+initbuffer_test(void)
+#endif
+{
+#if PY_MAJOR_VERSION >= 3
+    PyObject *m= PyModule_Create(&moduledef);
+#else
+    PyObject *m= Py_InitModule("buffer_test", buffer_functions);
+#endif
+    if (m == NULL)
+        INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&PyMyArrayType) < 0)
+        INITERROR;
+    Py_INCREF(&PyMyArrayType);
+    PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType);
+#if PY_MAJOR_VERSION >=3
+    return m;
+#endif
+}
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -87,4 +87,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
@@ -145,6 +151,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -433,21 +457,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,17 +1,26 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
+        assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
+        w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
-    @pytest.mark.skipif(True, reason='write a test for this')
-    def test_get_base_and_get_buffer(self, space, api):
-        assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+    def test_buffer_protocol(self):
+        import struct
+        module = self.import_module(name='buffer_test')
+        arr = module.PyMyArray(10)
+        y = memoryview(arr)
+        assert y.format == 'i'
+        assert y.shape == (10,)
+        s = y[3]
+        assert len(s) == struct.calcsize('i')
+        assert s == struct.pack('i', 3)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,9 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr)
+    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -385,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -608,6 +612,7 @@
                 bf_getwritebuffer.api_func.get_wrapper(space))
     pto.c_tp_as_buffer = c_buf
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
+    pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def type_dealloc(space, obj):
@@ -774,6 +779,8 @@
             pto.c_tp_setattro = base.c_tp_setattro
         if not pto.c_tp_getattro:
             pto.c_tp_getattro = base.c_tp_getattro
+        if not pto.c_tp_as_buffer:
+            pto.c_tp_as_buffer = base.c_tp_as_buffer
     finally:
         Py_DecRef(space, base_pyo)
 
@@ -810,8 +817,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -5,6 +5,7 @@
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
+from pypy.module.cpyext.api import Py_bufferP
 
 
 P, FT, PyO = Ptr, FuncType, PyObject
@@ -58,8 +59,7 @@
 writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
 segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
 charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
-## We don't support new buffer interface for now
-getbufferproc = rffi.VOIDP
+getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = rffi.VOIDP
 
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -597,6 +597,11 @@
 
 @jit.dont_look_inside
 def load_module(space, w_modulename, find_info, reuse=False):
+    """Like load_module() in CPython's import.c, this will normally
+    make a module object, store it in sys.modules, execute code in it,
+    and then fetch it again from sys.modules.  But this logic is not
+    used if we're calling a PEP302 loader.
+    """
     if find_info is None:
         return
 
@@ -625,17 +630,15 @@
 
         try:
             if find_info.modtype == PY_SOURCE:
-                load_source_module(
+                return load_source_module(
                     space, w_modulename, w_mod,
                     find_info.filename, find_info.stream.readall(),
                     find_info.stream.try_to_find_file_descriptor())
-                return w_mod
             elif find_info.modtype == PY_COMPILED:
                 magic = _r_long(find_info.stream)
                 timestamp = _r_long(find_info.stream)
-                load_compiled_module(space, w_modulename, w_mod, find_info.filename,
+                return load_compiled_module(space, w_modulename, w_mod, find_info.filename,
                                      magic, timestamp, find_info.stream.readall())
-                return w_mod
             elif find_info.modtype == PKG_DIRECTORY:
                 w_path = space.newlist([space.wrap(find_info.filename)])
                 space.setattr(w_mod, space.wrap('__path__'), w_path)
@@ -644,14 +647,13 @@
                 if find_info is None:
                     return w_mod
                 try:
-                    load_module(space, w_modulename, find_info, reuse=True)
+                    w_mod = load_module(space, w_modulename, find_info,
+                                        reuse=True)
                 finally:
                     try:
                         find_info.stream.close()
                     except StreamErrors:
                         pass
-                # fetch the module again, in case of "substitution"
-                w_mod = check_sys_modules(space, w_modulename)
                 return w_mod
             elif find_info.modtype == C_EXTENSION and has_so_extension(space):
                 load_c_extension(space, find_info.filename, space.str_w(w_modulename))
@@ -677,13 +679,6 @@
         try:
             if find_info:
                 w_mod = load_module(space, w_modulename, find_info)
-                try:
-                    w_mod = space.getitem(space.sys.get("modules"),
-                                          w_modulename)

From pypy.commits at gmail.com  Thu Sep  1 23:36:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 20:36:35 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: haaaack on top of previous hacks
Message-ID: <57c8f3c3.262ec20a.f163a.c930@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-5.x
Changeset: r86827:12bd393e0411
Date: 2016-09-01 20:35 -0700
http://bitbucket.org/pypy/pypy/changeset/12bd393e0411/

Log:	haaaack on top of previous hacks

diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -98,8 +98,15 @@
         try:
             ctypes.CDLL(name)
         except OSError as e:
+            # common case: ctypes fails too, with the real dlerror()
+            # message in str(e).  Return that error message.
             return str(e)
         else:
+            # uncommon case: may happen if 'name' is a linker script
+            # (which the C-level dlopen() can't handle) and we are
+            # directly running on pypy (whose implementation of ctypes
+            # or cffi will resolve linker scripts).  In that case, 
+            # unsure what we can do.
             return ("opening %r with ctypes.CDLL() works, "
                     "but not with c_dlopen()??" % (name,))
 
@@ -160,6 +167,13 @@
             mode = _dlopen_default_mode()
         elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
+        #
+        # haaaack for 'pypy py.test -A' if libm.so is a linker script
+        # (see reason in _dlerror_on_dlopen_untranslated())
+        if not we_are_translated() and platform.name == "linux":
+            if rffi.charp2str(name) == 'libm.so':
+                name = rffi.str2charp('libm.so.6', track_allocation=False)
+        #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
         if not res:
             if not we_are_translated():

From pypy.commits at gmail.com  Thu Sep  1 23:36:37 2016
From: pypy.commits at gmail.com (stefanor)
Date: Thu, 01 Sep 2016 20:36:37 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: name can be None
Message-ID: <57c8f3c5.85c11c0a.ed820.5fbe@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: release-5.x
Changeset: r86828:3806b361cac3
Date: 2016-09-01 20:35 -0700
http://bitbucket.org/pypy/pypy/changeset/3806b361cac3/

Log:	name can be None

diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -171,7 +171,7 @@
         # haaaack for 'pypy py.test -A' if libm.so is a linker script
         # (see reason in _dlerror_on_dlopen_untranslated())
         if not we_are_translated() and platform.name == "linux":
-            if rffi.charp2str(name) == 'libm.so':
+            if name and rffi.charp2str(name) == 'libm.so':
                 name = rffi.str2charp('libm.so.6', track_allocation=False)
         #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))

From pypy.commits at gmail.com  Thu Sep  1 23:36:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 01 Sep 2016 20:36:39 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Add a failing (skipped) whitebox
 test and a test about ctypes that fails
Message-ID: <57c8f3c7.a717c20a.dd7b5.d15f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-5.x
Changeset: r86829:2cab10de65ad
Date: 2016-09-01 20:35 -0700
http://bitbucket.org/pypy/pypy/changeset/2cab10de65ad/

Log:	Add a failing (skipped) whitebox test and a test about ctypes that
	fails on -A (xfailed)

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
@@ -195,6 +195,29 @@
             _fields_ = [('t', enum)]
         assert isinstance(S().t, enum)
 
+    def test_no_missing_shape_to_ffi_type(self):
+        # whitebox test
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("only for pypy's ctypes")
+        skip("re-enable after adding 'g' to _shape_to_ffi_type.typemap, "
+             "which I think needs fighting all the way up from "
+             "rpython.rlib.libffi")
+        from _ctypes.basics import _shape_to_ffi_type
+        from _rawffi import Array
+        for i in range(1, 256):
+            try:
+                Array(chr(i))
+            except ValueError:
+                pass
+            else:
+                assert chr(i) in _shape_to_ffi_type.typemap
+
+    @py.test.mark.xfail
+    def test_pointer_to_long_double(self):
+        import ctypes
+        ctypes.POINTER(ctypes.c_longdouble)
+
 ##    def test_perf(self):
 ##        check_perf()
 
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -47,6 +47,8 @@
         cls.ulonglong = clibffi.cast_type_to_ffitype(rffi.ULONGLONG)
         cls.signed = clibffi.cast_type_to_ffitype(rffi.SIGNED)
         cls.wchar_t = clibffi.cast_type_to_ffitype(lltype.UniChar)
+        # XXX long double support: clibffi.ffi_type_longdouble, but then
+        # XXX fix the whole rest of this file to add a case for long double
         del cls._import
 
     @staticmethod

From pypy.commits at gmail.com  Thu Sep  1 23:36:42 2016
From: pypy.commits at gmail.com (stefanor)
Date: Thu, 01 Sep 2016 20:36:42 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Avoid blowing up with results that
 are ~0 but <0 due to floating point imprecision
Message-ID: <57c8f3ca.82cbc20a.81584.c590@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: release-5.x
Changeset: r86831:5169ca3e696d
Date: 2016-09-01 20:35 -0700
http://bitbucket.org/pypy/pypy/changeset/5169ca3e696d/

Log:	Avoid blowing up with results that are ~0 but <0 due to floating
	point imprecision

diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -532,8 +532,7 @@
         return sys.maxint
     else:
         res = Solution[blockmap[graph.startblock]]
-        assert res >= 0
-        return res
+        return max(res, 0.0)
 
 def static_instruction_count(graph):
     count = 0

From pypy.commits at gmail.com  Thu Sep  1 23:36:41 2016
From: pypy.commits at gmail.com (stefanor)
Date: Thu, 01 Sep 2016 20:36:41 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Bump recursionlimit,
 for translating with cpython
Message-ID: <57c8f3c9.05d71c0a.db689.689c@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: release-5.x
Changeset: r86830:a5db0f4359ab
Date: 2016-09-01 20:35 -0700
http://bitbucket.org/pypy/pypy/changeset/a5db0f4359ab/

Log:	Bump recursionlimit, for translating with cpython

diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -213,6 +213,7 @@
         log.WARNING(warning)
 
 def main():
+    sys.setrecursionlimit(2000)  # PyPy can't translate within cpython's 1k limit
     targetspec_dic, translateconfig, config, args = parse_options_and_load_target()
     from rpython.translator import translator
     from rpython.translator import driver

From pypy.commits at gmail.com  Fri Sep  2 01:11:55 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Sep 2016 22:11:55 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: pfff another hack on top of this
 pile of hacks
Message-ID: <57c90a1b.81091c0a.d9c91.7c58@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86832:3e411b32904e
Date: 2016-09-02 08:10 +0300
http://bitbucket.org/pypy/pypy/changeset/3e411b32904e/

Log:	pfff another hack on top of this pile of hacks (grafted from
	76e37b5f30ae446c64388d1d7cd2d2f01999621e)

diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -170,11 +170,15 @@
         #
         # haaaack for 'pypy py.test -A' if libm.so is a linker script
         # (see reason in _dlerror_on_dlopen_untranslated())
+        must_free = False
         if not we_are_translated() and platform.name == "linux":
             if name and rffi.charp2str(name) == 'libm.so':
-                name = rffi.str2charp('libm.so.6', track_allocation=False)
+                name = rffi.str2charp('libm.so.6')
+                must_free = True
         #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
+        if must_free:
+            rffi.free_charp(name)
         if not res:
             if not we_are_translated():
                 err = _dlerror_on_dlopen_untranslated(name)

From pypy.commits at gmail.com  Fri Sep  2 01:44:30 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Sep 2016 22:44:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix b127faf95f86 which updated
 contributors but removed the rest of LICENSE
Message-ID: <57c911be.94a51c0a.94102.7fea@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86833:d6a8fc8b6b27
Date: 2016-09-02 08:42 +0300
http://bitbucket.org/pypy/pypy/changeset/d6a8fc8b6b27/

Log:	fix b127faf95f86 which updated contributors but removed the rest of
	LICENSE

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -369,3 +369,109 @@
   Roman Podoliaka
   Dan Loewenherz
   werat
+
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
+
+The PyPy Logo as used by http://speed.pypy.org and others was created
+by Samuel Reis and is distributed on terms of Creative Commons Share Alike
+License.
+ 
+License for 'lib-python/2.7'
+============================
+
+Except when otherwise stated (look for LICENSE files or copyright/license
+information at the beginning of each file) the files in the 'lib-python/2.7'
+directory are all copyrighted by the Python Software Foundation and licensed
+under the terms that you can find here: https://docs.python.org/2/license.html
+
+License for 'pypy/module/unicodedata/'
+======================================
+
+The following files are from the website of The Unicode Consortium
+at http://www.unicode.org/.  For the terms of use of these files, see
+http://www.unicode.org/terms_of_use.html .  Or they are derived from
+files from the above website, and the same terms of use apply.
+
+    CompositionExclusions-*.txt
+    EastAsianWidth-*.txt
+    LineBreak-*.txt
+    UnicodeData-*.txt
+    UnihanNumeric-*.txt
+
+License for 'dotviewer/font/'
+=============================
+
+Copyright (C) 2008 The Android Open Source Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+  
+     http://www.apache.org/licenses/LICENSE-2.0
+  
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Detailed license information is contained in the NOTICE file in the
+directory.
+
+
+Licenses and Acknowledgements for Incorporated Software
+=======================================================
+
+This section is an incomplete, but growing list of licenses and
+acknowledgements for third-party software incorporated in the PyPy
+distribution.
+
+License for 'Tcl/Tk'
+--------------------
+
+This copy of PyPy contains library code that may, when used, result in
+the Tcl/Tk library to be loaded.  PyPy also includes code that may be
+regarded as being a copy of some parts of the Tcl/Tk header files.
+You may see a copy of the License for Tcl/Tk in the file
+`lib_pypy/_tkinter/license.terms` included here.
+
+License for 'bzip2'
+-------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
+
+    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
+
+License for 'openssl'
+---------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+openssl library.  You may see a copy of the License for OpenSSL at
+
+    https://www.openssl.org/source/license.html
+
+License for 'gdbm'
+------------------
+
+The gdbm module includes code from gdbm.h, which is distributed under
+the terms of the GPL license version 2 or any later version.  Thus the
+gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
+under the terms of the GPL license as well.
+
+License for 'rpython/rlib/rvmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://github.com/gperftools/gperftools/blob/master/COPYING

From pypy.commits at gmail.com  Fri Sep  2 01:44:32 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Sep 2016 22:44:32 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: fix b127faf95f86 which updated
 contributors but removed the rest of LICENSE
Message-ID: <57c911c0.82cbc20a.81584.e2e9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86834:d68da509772b
Date: 2016-09-02 08:43 +0300
http://bitbucket.org/pypy/pypy/changeset/d68da509772b/

Log:	fix b127faf95f86 which updated contributors but removed the rest of
	LICENSE (grafted from d6a8fc8b6b278362d6a4c7602135354b91ac2e9a)

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -369,3 +369,109 @@
   Roman Podoliaka
   Dan Loewenherz
   werat
+
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
+
+The PyPy Logo as used by http://speed.pypy.org and others was created
+by Samuel Reis and is distributed on terms of Creative Commons Share Alike
+License.
+ 
+License for 'lib-python/2.7'
+============================
+
+Except when otherwise stated (look for LICENSE files or copyright/license
+information at the beginning of each file) the files in the 'lib-python/2.7'
+directory are all copyrighted by the Python Software Foundation and licensed
+under the terms that you can find here: https://docs.python.org/2/license.html
+
+License for 'pypy/module/unicodedata/'
+======================================
+
+The following files are from the website of The Unicode Consortium
+at http://www.unicode.org/.  For the terms of use of these files, see
+http://www.unicode.org/terms_of_use.html .  Or they are derived from
+files from the above website, and the same terms of use apply.
+
+    CompositionExclusions-*.txt
+    EastAsianWidth-*.txt
+    LineBreak-*.txt
+    UnicodeData-*.txt
+    UnihanNumeric-*.txt
+
+License for 'dotviewer/font/'
+=============================
+
+Copyright (C) 2008 The Android Open Source Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+  
+     http://www.apache.org/licenses/LICENSE-2.0
+  
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Detailed license information is contained in the NOTICE file in the
+directory.
+
+
+Licenses and Acknowledgements for Incorporated Software
+=======================================================
+
+This section is an incomplete, but growing list of licenses and
+acknowledgements for third-party software incorporated in the PyPy
+distribution.
+
+License for 'Tcl/Tk'
+--------------------
+
+This copy of PyPy contains library code that may, when used, result in
+the Tcl/Tk library to be loaded.  PyPy also includes code that may be
+regarded as being a copy of some parts of the Tcl/Tk header files.
+You may see a copy of the License for Tcl/Tk in the file
+`lib_pypy/_tkinter/license.terms` included here.
+
+License for 'bzip2'
+-------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
+
+    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
+
+License for 'openssl'
+---------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+openssl library.  You may see a copy of the License for OpenSSL at
+
+    https://www.openssl.org/source/license.html
+
+License for 'gdbm'
+------------------
+
+The gdbm module includes code from gdbm.h, which is distributed under
+the terms of the GPL license version 2 or any later version.  Thus the
+gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
+under the terms of the GPL license as well.
+
+License for 'rpython/rlib/rvmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://github.com/gperftools/gperftools/blob/master/COPYING

From pypy.commits at gmail.com  Fri Sep  2 02:49:19 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Sep 2016 23:49:19 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: merge default into branch
Message-ID: <57c920ef.4152c20a.7f40d.0401@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86836:802bf4e38a0d
Date: 2016-09-02 09:48 +0300
http://bitbucket.org/pypy/pypy/changeset/802bf4e38a0d/

Log:	merge default into branch

diff too long, truncating to 2000 out of 210666 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,6 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,12 +360,15 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
 
   Heinrich-Heine University, Germany 
   Open End AB (formerly AB Strakt), Sweden
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -3,7 +3,8 @@
 ============
 
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
-This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
 which now supports the "limited API" mode for c-extensions on 
 CPython >=3.2.
@@ -12,9 +13,7 @@
 support to OpenBSD and Dragon Fly BSD
 
 As always, this release fixed many issues and bugs raised by the
-growing community of PyPy users. 
-
-XXXXX MORE ???
+growing community of PyPy users. We strongly recommend updating.
 
 You can download the PyPy2.7 v5.4 release here:
 
@@ -110,8 +109,8 @@
 
   * (RPython) add `rposix_scandir` portably, needed for Python 3.5
 
-  * Support for memoryview attributes (format, itemsize, ...) which also
-    adds support for `PyMemoryView_FromObject`
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
 
 * Bug Fixes
 
@@ -153,10 +152,6 @@
   * Make `hash(-1)` return -2, as CPython does, and fix all the
     ancilary places this matters
 
-  * Issues reported with our previous release were resolved_ after
-    reports from users on our issue tracker at
-    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
-
   * Fix `PyNumber_Check()` to behave more like CPython
 
   * (VMProf) Try hard to not miss any Python-level frame in the
@@ -169,6 +164,10 @@
   * Fix the mapdict cache for subclasses of builtin types that
     provide a dict
 
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,5 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
-
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -23,6 +23,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -84,7 +92,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -38,6 +38,8 @@
 
 
 class W_ClassObject(W_Root):
+    _immutable_fields_ = ['bases_w?[*]', 'w_dict?']
+
     def __init__(self, space, w_name, bases, w_dict):
         self.name = space.str_w(w_name)
         make_sure_not_resized(bases)
@@ -75,6 +77,7 @@
                             "__bases__ items must be classes")
         self.bases_w = bases_w
 
+    @jit.unroll_safe
     def is_subclass_of(self, other):
         assert isinstance(other, W_ClassObject)
         if self is other:
@@ -313,7 +316,7 @@
         # This method ignores the instance dict and the __getattr__.
         # Returns None if not found.
         assert isinstance(name, str)
-        w_value = self.w_class.lookup(space, name)
+        w_value = jit.promote(self.w_class).lookup(space, name)
         if w_value is None:
             return None
         w_descr_get = space.lookup(w_value, '__get__')
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -250,6 +250,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -123,5 +123,4 @@
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
     space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname))
-    importing.exec_code_module(space, w_mod, code)
-    return w_mod
+    return importing.exec_code_module(space, w_mod, code, w_name)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
@@ -145,6 +151,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -433,21 +457,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -597,6 +597,11 @@
 
 @jit.dont_look_inside
 def load_module(space, w_modulename, find_info, reuse=False):
+    """Like load_module() in CPython's import.c, this will normally
+    make a module object, store it in sys.modules, execute code in it,
+    and then fetch it again from sys.modules.  But this logic is not
+    used if we're calling a PEP302 loader.
+    """
     if find_info is None:
         return
 
@@ -625,17 +630,15 @@
 
         try:
             if find_info.modtype == PY_SOURCE:
-                load_source_module(
+                return load_source_module(
                     space, w_modulename, w_mod,
                     find_info.filename, find_info.stream.readall(),
                     find_info.stream.try_to_find_file_descriptor())
-                return w_mod
             elif find_info.modtype == PY_COMPILED:
                 magic = _r_long(find_info.stream)
                 timestamp = _r_long(find_info.stream)
-                load_compiled_module(space, w_modulename, w_mod, find_info.filename,
+                return load_compiled_module(space, w_modulename, w_mod, find_info.filename,
                                      magic, timestamp, find_info.stream.readall())
-                return w_mod
             elif find_info.modtype == PKG_DIRECTORY:
                 w_path = space.newlist([space.wrap(find_info.filename)])
                 space.setattr(w_mod, space.wrap('__path__'), w_path)
@@ -644,14 +647,13 @@
                 if find_info is None:
                     return w_mod
                 try:
-                    load_module(space, w_modulename, find_info, reuse=True)
+                    w_mod = load_module(space, w_modulename, find_info,
+                                        reuse=True)
                 finally:
                     try:
                         find_info.stream.close()
                     except StreamErrors:
                         pass
-                # fetch the module again, in case of "substitution"
-                w_mod = check_sys_modules(space, w_modulename)
                 return w_mod
             elif find_info.modtype == C_EXTENSION and has_so_extension(space):
                 load_c_extension(space, find_info.filename, space.str_w(w_modulename))
@@ -677,13 +679,6 @@
         try:
             if find_info:
                 w_mod = load_module(space, w_modulename, find_info)
-                try:
-                    w_mod = space.getitem(space.sys.get("modules"),
-                                          w_modulename)
-                except OperationError as oe:
-                    if not oe.match(space, space.w_KeyError):
-                        raise
-                    raise OperationError(space.w_ImportError, w_modulename)
                 if w_parent is not None:
                     space.setattr(w_parent, space.wrap(partname), w_mod)
                 return w_mod
@@ -875,20 +870,32 @@
     pycode = ec.compiler.compile(source, pathname, 'exec', 0)
     return pycode
 
-def exec_code_module(space, w_mod, code_w):
+def exec_code_module(space, w_mod, code_w, w_modulename, check_afterwards=True):
+    """
+    Execute a code object in the module's dict.  Returns
+    'sys.modules[modulename]', which must exist.
+    """
     w_dict = space.getattr(w_mod, space.wrap('__dict__'))
     space.call_method(w_dict, 'setdefault',
                       space.wrap('__builtins__'),
                       space.wrap(space.builtin))
     code_w.exec_code(space, w_dict, w_dict)
 
+    if check_afterwards:
+        w_mod = check_sys_modules(space, w_modulename)
+        if w_mod is None:
+            raise oefmt(space.w_ImportError,
+                        "Loaded module %R not found in sys.modules",
+                        w_modulename)
+    return w_mod
+
 
 @jit.dont_look_inside
 def load_source_module(space, w_modulename, w_mod, pathname, source, fd,
-                       write_pyc=True):
+                       write_pyc=True, check_afterwards=True):
     """
-    Load a source module from a given file and return its module
-    object.
+    Load a source module from a given file.  Returns the result
+    of sys.modules[modulename], which must exist.
     """
     w = space.wrap
 
@@ -927,9 +934,8 @@
         code_w.remove_docstrings(space)
 
     update_code_filenames(space, code_w, pathname)
-    exec_code_module(space, w_mod, code_w)
-
-    return w_mod
+    return exec_code_module(space, w_mod, code_w, w_modulename,
+                            check_afterwards=check_afterwards)
 
 def update_code_filenames(space, code_w, pathname, oldname=None):
     assert isinstance(code_w, PyCode)
@@ -1012,10 +1018,10 @@
 
 @jit.dont_look_inside
 def load_compiled_module(space, w_modulename, w_mod, cpathname, magic,
-                         timestamp, source):
+                         timestamp, source, check_afterwards=True):
     """
-    Load a module from a compiled file, execute it, and return its
-    module object.
+    Load a module from a compiled file and execute it.  Returns
+    'sys.modules[modulename]', which must exist.
     """
     log_pyverbose(space, 1, "import %s # compiled from %s\n" %
                   (space.str_w(w_modulename), cpathname))
@@ -1032,9 +1038,8 @@
     if optimize >= 2:
         code_w.remove_docstrings(space)
 
-    exec_code_module(space, w_mod, code_w)
-
-    return w_mod
+    return exec_code_module(space, w_mod, code_w, w_modulename,
+                            check_afterwards=check_afterwards)
 
 def open_exclusive(space, cpathname, mode):
     try:
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -98,33 +98,35 @@
     w_mod = space.wrap(Module(space, w_modulename))
     importing._prepare_module(space, w_mod, filename, None)
 
-    importing.load_source_module(
+    w_mod = importing.load_source_module(
         space, w_modulename, w_mod,
         filename, stream.readall(), stream.try_to_find_file_descriptor())
     if space.is_none(w_file):
         stream.close()
     return w_mod
 
- at unwrap_spec(filename='str0')
-def _run_compiled_module(space, w_modulename, filename, w_file, w_module):
+ at unwrap_spec(filename='str0', check_afterwards=int)
+def _run_compiled_module(space, w_modulename, filename, w_file, w_module,
+                         check_afterwards=False):
     # the function 'imp._run_compiled_module' is a pypy-only extension
     stream = get_file(space, w_file, filename, 'rb')
 
     magic = importing._r_long(stream)
     timestamp = importing._r_long(stream)
 
-    importing.load_compiled_module(
+    w_mod = importing.load_compiled_module(
         space, w_modulename, w_module, filename, magic, timestamp,
-        stream.readall())
+        stream.readall(), check_afterwards=check_afterwards)
     if space.is_none(w_file):
         stream.close()
+    return w_mod
 
 @unwrap_spec(filename='str0')
 def load_compiled(space, w_modulename, filename, w_file=None):
     w_mod = space.wrap(Module(space, w_modulename))
     importing._prepare_module(space, w_mod, filename, None)
-    _run_compiled_module(space, w_modulename, filename, w_file, w_mod)
-    return w_mod
+    return _run_compiled_module(space, w_modulename, filename, w_file, w_mod,
+                                check_afterwards=True)
 
 @unwrap_spec(filename=str)
 def load_dynamic(space, w_modulename, filename, w_file=None):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -118,7 +118,7 @@
         filename = str(p.join("x.py"))
         stream = streamio.open_file_as_stream(filename, "r")
         try:
-            importing.load_source_module(
+            _load_source_module(
                 space, w_modname, w(importing.Module(space, w_modname)),
                 filename, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -139,6 +139,15 @@
 
     return str(root)
 
+def _load_source_module(space, w_modname, w_mod, *args, **kwds):
+    kwds.setdefault('check_afterwards', False)
+    return importing.load_source_module(space, w_modname, w_mod, *args, **kwds)
+
+def _load_compiled_module(space, w_modname, w_mod, *args, **kwds):
+    kwds.setdefault('check_afterwards', False)
+    return importing.load_compiled_module(space, w_modname, w_mod,
+                                          *args, **kwds)
+
 
 def _setup(space):
     dn = setup_directory_structure(space)
@@ -887,8 +896,7 @@
             w_mod = space.wrap(Module(space, w_modulename))
             magic = importing._r_long(stream)
             timestamp = importing._r_long(stream)
-            w_ret = importing.load_compiled_module(space,
-                                                   w_modulename,
+            w_ret = _load_compiled_module(space,   w_modulename,
                                                    w_mod,
                                                    cpathname,
                                                    magic,
@@ -946,7 +954,7 @@
         pathname = _testfilesource()
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -968,7 +976,7 @@
         pathname = _testfilesource()
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor(),
@@ -987,7 +995,7 @@
         try:
             space.setattr(space.sys, space.wrap('dont_write_bytecode'),
                           space.w_True)
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1006,7 +1014,7 @@
         pathname = _testfilesource(source="<Syntax Error>")
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1026,7 +1034,7 @@
         pathname = _testfilesource(source="a = unknown_name")
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1114,7 +1122,7 @@
                     magic = importing._r_long(stream)
                     timestamp = importing._r_long(stream)
                     space2.raises_w(space2.w_ImportError,
-                                    importing.load_compiled_module,
+                                    _load_compiled_module,
                                     space2,
                                     w_modulename,
                                     w_mod,
@@ -1326,10 +1334,7 @@
         # use an import hook that doesn't update sys.modules, then the
         # import succeeds; but at the same time, you can have the same
         # result without an import hook (see test_del_from_sys_modules)
-        # and then the import fails.  This looks like even more mess
-        # to replicate, so we ignore it until someone really hits this
-        # case...
-        skip("looks like an inconsistency in CPython")
+        # and then the import fails.  Mess mess mess.
 
         class ImportHook(object):
             def find_module(self, fullname, path=None):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -187,6 +187,43 @@
 
         """)
 
+    def test_oldstyle_methcall(self):
+        def main():
+            def g(): pass
+            class A:
+                def f(self):
+                    return self.x + 1
+            class I(A):
+                pass
+            class J(I):
+                pass
+
+
+            class B(J):
+                def __init__(self, x):
+                    self.x = x
+
+            i = 0
+            b = B(1)
+            while i < 1000:
+                g()
+                v = b.f() # ID: meth
+                i += v
+            return i
+
+        log = self.run(main, [], threshold=80)
+        loop, = log.loops_by_filename(self.filepath, is_entry_bridge=True)
+        assert loop.match_by_id('meth',
+        '''
+    guard_nonnull_class(p18, ..., descr=...)
+    p52 = getfield_gc_r(p18, descr=...) # read map
+    guard_value(p52, ConstPtr(ptr53), descr=...)
+    p54 = getfield_gc_r(p18, descr=...) # read class
+    guard_value(p54, ConstPtr(ptr55), descr=...)
+    p56 = force_token() # done
+        ''')
+
+
     def test_oldstyle_newstyle_mix(self):
         def main():
             class A:
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -64,7 +64,7 @@
         AsyncAction.__init__(self, space)
         self.pending_signal = -1
         self.fire_in_another_thread = False
-        #
+
         @rgc.no_collect
         def _after_thread_switch():
             if self.fire_in_another_thread:
@@ -251,7 +251,7 @@
         except OSError as e:
             if e.errno == errno.EBADF:
                 raise oefmt(space.w_ValueError, "invalid fd")
-    old_fd = pypysig_set_wakeup_fd(fd)
+    old_fd = pypysig_set_wakeup_fd(fd, True)
     return space.wrap(intmask(old_fd))
 
 
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
@@ -195,6 +195,29 @@
             _fields_ = [('t', enum)]
         assert isinstance(S().t, enum)
 
+    def test_no_missing_shape_to_ffi_type(self):
+        # whitebox test
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("only for pypy's ctypes")
+        skip("re-enable after adding 'g' to _shape_to_ffi_type.typemap, "
+             "which I think needs fighting all the way up from "
+             "rpython.rlib.libffi")
+        from _ctypes.basics import _shape_to_ffi_type
+        from _rawffi import Array
+        for i in range(1, 256):
+            try:
+                Array(chr(i))
+            except ValueError:
+                pass
+            else:
+                assert chr(i) in _shape_to_ffi_type.typemap
+
+    @py.test.mark.xfail
+    def test_pointer_to_long_double(self):
+        import ctypes
+        ctypes.POINTER(ctypes.c_longdouble)
+
 ##    def test_perf(self):
 ##        check_perf()
 
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -152,8 +152,7 @@
         importing._prepare_module(space, w_mod, real_name, pkgpath)
         co_filename = self.make_co_filename(filename)
         code_w = importing.parse_source_module(space, co_filename, buf)
-        importing.exec_code_module(space, w_mod, code_w)
-        return w_mod
+        return importing.exec_code_module(space, w_mod, code_w, w(modname))
 
     def _parse_mtime(self, space, filename):
         w = space.wrap
@@ -205,10 +204,10 @@
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
         space.setattr(w_mod, w('__loader__'), space.wrap(self))
         importing._prepare_module(space, w_mod, real_name, pkgpath)
-        result = importing.load_compiled_module(space, w(modname), w_mod,
+        w_result = importing.load_compiled_module(space, w(modname), w_mod,
                                                 filename, magic, timestamp,
                                                 buf)
-        return result
+        return w_result
 
     def have_modulefile(self, space, filename):
         if ZIPSEP != os.path.sep:
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -58,6 +58,20 @@
     return w_iter
 tuple_iter._annspecialcase_ = 'specialize:memo'
 
+def str_getitem(space):
+    "Utility that returns the app-level descriptor str.__getitem__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_str,
+                                               '__getitem__')
+    return w_iter
+str_getitem._annspecialcase_ = 'specialize:memo'
+
+def unicode_getitem(space):
+    "Utility that returns the app-level descriptor unicode.__getitem__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
+                                               '__getitem__')
+    return w_iter
+unicode_getitem._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:
         raise oefmt(space.w_AttributeError,
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -8,6 +8,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rfloat import copysign, formatd
 from rpython.rlib.rarithmetic import r_uint, intmask
+from pypy.interpreter.signature import Signature
 
 
 @specialize.argtype(1)
@@ -40,19 +41,32 @@
 ANS_MANUAL = 3
 
 
-def make_template_formatting_class():
+format_signature = Signature([], 'args', 'kwargs')
+
+
+def make_template_formatting_class(for_unicode):
     class TemplateFormatter(object):
+        is_unicode = for_unicode
 
         parser_list_w = None
 
-        def __init__(self, space, is_unicode, template):
+        def __init__(self, space, template):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.template = template
 
         def build(self, args):
-            self.args, self.kwargs = args.unpack()
+            if self.is_unicode:
+                # for unicode, use the slower parse_obj() to get self.w_kwargs
+                # as a wrapped dictionary that may contain full-range unicode
+                # keys.  See test_non_latin1_key
+                space = self.space
+                w_args, w_kwds = args.parse_obj(None, 'format',
+                                                format_signature)
+                self.args = space.listview(w_args)
+                self.w_kwargs = w_kwds
+            else:
+                self.args, self.kwargs = args.unpack()
             self.auto_numbering = 0
             self.auto_numbering_state = ANS_INIT
             return self._build_string(0, len(self.template), 2)
@@ -197,17 +211,13 @@
             if index == -1:
                 kwarg = name[:i]
                 if self.is_unicode:
+                    w_arg = space.getitem(self.w_kwargs, space.wrap(kwarg))
+                else:
                     try:
-                        arg_key = kwarg.encode("latin-1")
-                    except UnicodeEncodeError:
-                        # Not going to be found in a dict of strings.
-                        raise OperationError(space.w_KeyError, space.wrap(kwarg))
-                else:
-                    arg_key = kwarg
-                try:
-                    w_arg = self.kwargs[arg_key]
-                except KeyError:
-                    raise OperationError(space.w_KeyError, space.wrap(arg_key))
+                        w_arg = self.kwargs[kwarg]
+                    except KeyError:
+                        raise OperationError(space.w_KeyError,
+                                             space.wrap(kwarg))
             else:
                 try:
                     w_arg = self.args[index]
@@ -351,14 +361,8 @@
             return space.iter(space.newlist(self.parser_list_w))
     return TemplateFormatter
 
-StrTemplateFormatter = make_template_formatting_class()
-UnicodeTemplateFormatter = make_template_formatting_class()
-
-def str_template_formatter(space, template):
-    return StrTemplateFormatter(space, False, template)
-
-def unicode_template_formatter(space, template):
-    return UnicodeTemplateFormatter(space, True, template)
+str_template_formatter = make_template_formatting_class(for_unicode=False)
+unicode_template_formatter = make_template_formatting_class(for_unicode=True)
 
 
 def format_method(space, w_string, args, is_unicode):
@@ -395,16 +399,16 @@
 
 LONG_DIGITS = string.digits + string.ascii_lowercase
 
-def make_formatting_class():
+def make_formatting_class(for_unicode):
     class Formatter(BaseFormatter):
         """__format__ implementation for builtin types."""
 
+        is_unicode = for_unicode
         _grouped_digits = None
 
-        def __init__(self, space, is_unicode, spec):
+        def __init__(self, space, spec):
             self.space = space
-            self.is_unicode = is_unicode
-            self.empty = u"" if is_unicode else ""
+            self.empty = u"" if self.is_unicode else ""
             self.spec = spec
 
         def _is_alignment(self, c):
@@ -1138,15 +1142,8 @@
             self._unknown_presentation("complex")
     return Formatter
 
-StrFormatter = make_formatting_class()
-UnicodeFormatter = make_formatting_class()
-
-
-def unicode_formatter(space, spec):
-    return StrFormatter(space, True, spec)
-
-def str_formatter(space, spec):
-    return UnicodeFormatter(space, False, spec)
+str_formatter = make_formatting_class(for_unicode=False)
+unicode_formatter = make_formatting_class(for_unicode=True)
 
 
 @specialize.arg(2)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -445,7 +445,7 @@
             return w_obj.listview_bytes()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_bytes()
-        if isinstance(w_obj, W_BytesObject) and self._uses_no_iter(w_obj):
+        if isinstance(w_obj, W_BytesObject) and self._str_uses_no_iter(w_obj):
             return w_obj.listview_bytes()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_bytes()
@@ -460,7 +460,7 @@
             return w_obj.listview_unicode()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_unicode()
-        if isinstance(w_obj, W_UnicodeObject) and self._uses_no_iter(w_obj):
+        if isinstance(w_obj, W_UnicodeObject) and self._uni_uses_no_iter(w_obj):
             return w_obj.listview_unicode()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_unicode()
@@ -504,8 +504,15 @@
         from pypy.objspace.descroperation import tuple_iter
         return self.lookup(w_obj, '__iter__') is tuple_iter(self)
 
-    def _uses_no_iter(self, w_obj):
-        return self.lookup(w_obj, '__iter__') is None
+    def _str_uses_no_iter(self, w_obj):
+        from pypy.objspace.descroperation import str_getitem
+        return (self.lookup(w_obj, '__iter__') is None and
+                self.lookup(w_obj, '__getitem__') is str_getitem(self))
+
+    def _uni_uses_no_iter(self, w_obj):
+        from pypy.objspace.descroperation import unicode_getitem
+        return (self.lookup(w_obj, '__iter__') is None and
+                self.lookup(w_obj, '__getitem__') is unicode_getitem(self))
 
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -432,7 +432,7 @@
 
 
 class AppTestListObject(object):
-    spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
+    #spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
 
     def setup_class(cls):
         import platform
@@ -1518,6 +1518,16 @@
             def __iter__(self):
                 yield "ok"
         assert list(U(u"don't see me")) == ["ok"]
+        #
+        class S(str):
+            def __getitem__(self, index):
+                return str.__getitem__(self, index).upper()
+        assert list(S("abc")) == list("ABC")
+        #
+        class U(unicode):
+            def __getitem__(self, index):
+                return unicode.__getitem__(self, index).upper()
+        assert list(U(u"abc")) == list(u"ABC")
 
     def test_extend_from_nonempty_list_with_subclasses(self):
         l = ["hi!"]
@@ -1543,6 +1553,20 @@
         l.extend(U(u"don't see me"))
         #
         assert l == ["hi!", "okT", "okL", "okL", "okS", "okU"]
+        #
+        class S(str):
+            def __getitem__(self, index):
+                return str.__getitem__(self, index).upper()
+        l = []
+        l.extend(S("abc"))
+        assert l == list("ABC")
+        #
+        class U(unicode):
+            def __getitem__(self, index):
+                return unicode.__getitem__(self, index).upper()
+        l = []
+        l.extend(U(u"abc"))
+        assert l == list(u"ABC")
 
     def test_no_len_on_range_iter(self):
         iterable = range(10)
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -215,7 +215,9 @@
         assert self.s("{!r}").format(x()) == self.s("32")
 
     def test_non_latin1_key(self):
-        raises(KeyError, self.s("{\u1000}").format)
+        raises(KeyError, u"{\u1000}".format)
+        d = {u"\u1000": u"foo"}
+        assert u"{\u1000}".format(**d) == u"foo"
 
 
 class AppTestStringFormat(BaseStringFormatTests):
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=3
-rev=1
+min=4
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -164,8 +164,15 @@
             # annotations that are passed in, and don't annotate the old
             # graph -- it's already low-level operations!
             for a, s_newarg in zip(block.inputargs, cells):
-                s_oldarg = self.binding(a)
-                assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
+                s_oldarg = a.annotation
+                # XXX: Should use s_oldarg.contains(s_newarg) but that breaks
+                # PyPy translation
+                if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg:
+                    raise annmodel.AnnotatorError(
+                        "Late-stage annotation is not allowed to modify the "
+                        "existing annotation for variable %s: %s" %
+                            (a, s_oldarg))
+
         else:
             assert not self.frozen
             if block not in self.annotated:
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -17,7 +17,7 @@
 from rpython.flowspace.model import Variable, Constant, const
 from rpython.flowspace.operation import op
 from rpython.rlib import rarithmetic
-from rpython.annotator.model import AnnotatorError
+from rpython.annotator.model import AnnotatorError, TLS
 
 BINARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values()
                         if oper.dispatch == 2])
@@ -436,6 +436,11 @@
 class __extend__(pairtype(SomeFloat, SomeFloat)):
 
     def union((flt1, flt2)):
+        if not TLS.allow_int_to_float:
+            # in this mode, if one of the two is actually the
+            # subclass SomeInteger, complain
+            if isinstance(flt1, SomeInteger) or isinstance(flt2, SomeInteger):
+                raise UnionError(flt1, flt2)
         return SomeFloat()
 
     add = sub = mul = union
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -44,6 +44,7 @@
     # A global attribute :-(  Patch it with 'True' to enable checking of
     # the no_nul attribute...
     check_str_without_nul = False
+    allow_int_to_float = True
 TLS = State()
 
 class SomeObject(object):
@@ -749,6 +750,7 @@
                 s1 = pair(s1, s2).union()
     else:
         # this is just a performance shortcut
+        # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
         if s1 != s2:
             s1 = pair(s1, s2).union()
     return s1
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -176,6 +176,7 @@
 
 
 SOCK_DGRAM SOCK_RAW SOCK_RDM SOCK_SEQPACKET SOCK_STREAM
+SOCK_CLOEXEC
 
 SOL_SOCKET SOL_IPX SOL_AX25 SOL_ATALK SOL_NETROM SOL_ROSE
 
@@ -319,6 +320,8 @@
                                           [('p_proto', rffi.INT),
                                            ])
 
+CConfig.HAVE_ACCEPT4 = platform.Has('accept4')
+
 if _POSIX:
     CConfig.nfds_t = platform.SimpleType('nfds_t')
     CConfig.pollfd = platform.Struct('struct pollfd',
@@ -541,6 +544,12 @@
 socketaccept = external('accept', [socketfd_type, sockaddr_ptr,
                                    socklen_t_ptr], socketfd_type,
                         save_err=SAVE_ERR)
+HAVE_ACCEPT4 = cConfig.HAVE_ACCEPT4
+if HAVE_ACCEPT4:
+    socketaccept4 = external('accept4', [socketfd_type, sockaddr_ptr,
+                                         socklen_t_ptr, rffi.INT],
+                                        socketfd_type,
+                             save_err=SAVE_ERR)
 socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t],
                               rffi.INT, save_err=SAVE_ERR)
 socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT,
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -10,6 +10,7 @@
     _immutable_ = True
 
     def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
         raise NotImplementedError
 
     def __len__(self):
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -47,6 +47,8 @@
         cls.ulonglong = clibffi.cast_type_to_ffitype(rffi.ULONGLONG)
         cls.signed = clibffi.cast_type_to_ffitype(rffi.SIGNED)
         cls.wchar_t = clibffi.cast_type_to_ffitype(lltype.UniChar)
+        # XXX long double support: clibffi.ffi_type_longdouble, but then
+        # XXX fix the whole rest of this file to add a case for long double
         del cls._import
 
     @staticmethod
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1222,6 +1222,9 @@
         # base is supposed to be positive or 0.0, which means we use e
         if base == 10.0:
             return _loghelper(math.log10, self)
+        if base == 2.0:
+            from rpython.rlib import rfloat
+            return _loghelper(rfloat.log2, self)
         ret = _loghelper(math.log, self)
         if base != 0.0:
             ret /= math.log(base)
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -98,8 +98,15 @@
         try:
             ctypes.CDLL(name)
         except OSError as e:
+            # common case: ctypes fails too, with the real dlerror()
+            # message in str(e).  Return that error message.
             return str(e)
         else:
+            # uncommon case: may happen if 'name' is a linker script
+            # (which the C-level dlopen() can't handle) and we are
+            # directly running on pypy (whose implementation of ctypes
+            # or cffi will resolve linker scripts).  In that case, 
+            # unsure what we can do.
             return ("opening %r with ctypes.CDLL() works, "
                     "but not with c_dlopen()??" % (name,))
 
@@ -160,7 +167,18 @@
             mode = _dlopen_default_mode()
         elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
+        #
+        # haaaack for 'pypy py.test -A' if libm.so is a linker script
+        # (see reason in _dlerror_on_dlopen_untranslated())
+        must_free = False
+        if not we_are_translated() and platform.name == "linux":
+            if name and rffi.charp2str(name) == 'libm.so':
+                name = rffi.str2charp('libm.so.6')
+                must_free = True
+        #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
+        if must_free:
+            rffi.free_charp(name)
         if not res:
             if not we_are_translated():
                 err = _dlerror_on_dlopen_untranslated(name)
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -430,9 +430,8 @@
 
 def encode_merge_point(log, compressor, values):
     line = []
-    unrolled = unrolling_iterable(values)
     i = 0
-    for value in unrolled:
+    for value in values:
         line.append(value.encode(log,i,compressor))
         i += 1
     return ''.join(line)
diff --git a/rpython/rlib/rmarshal.py b/rpython/rlib/rmarshal.py
--- a/rpython/rlib/rmarshal.py
+++ b/rpython/rlib/rmarshal.py
@@ -346,11 +346,15 @@
     # on s_bigger.  It relies on the fact that s_bigger was created with
     # an expression like 'annotation([s_item])' which returns a ListDef with
     # no bookkeeper, on which side-effects are not allowed.
+    saved = annmodel.TLS.allow_int_to_float
     try:
+        annmodel.TLS.allow_int_to_float = False
         s_union = annmodel.unionof(s_bigger, s_smaller)
         return s_bigger.contains(s_union)
     except (annmodel.UnionError, TooLateForChange):
         return False
+    finally:
+        annmodel.TLS.allow_int_to_float = saved
 
 
 class __extend__(pairtype(MTag, annmodel.SomeObject)):
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -126,6 +126,10 @@
         "SSL_OP_NO_COMPRESSION")
     SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger(
         "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS")
+    SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger(
+        "SSL_OP_CIPHER_SERVER_PREFERENCE")
+    SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger(
+        "SSL_OP_SINGLE_DH_USE")
     HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME")
     HAS_NPN = rffi_platform.Defined("OPENSSL_NPN_NEGOTIATED")
     SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE")
@@ -307,6 +311,8 @@
 
 if HAVE_OPENSSL_RAND:
     ssl_external('RAND_add', [rffi.CCHARP, rffi.INT, rffi.DOUBLE], lltype.Void)
+    ssl_external('RAND_bytes', [rffi.UCHARP, rffi.INT], rffi.INT)
+    ssl_external('RAND_pseudo_bytes', [rffi.UCHARP, rffi.INT], rffi.INT)
     ssl_external('RAND_status', [], rffi.INT)
     if HAVE_OPENSSL_RAND_EGD:
         ssl_external('RAND_egd', [rffi.CCHARP], rffi.INT)
@@ -465,6 +471,7 @@
 ssl_external('GENERAL_NAME_print', [BIO, GENERAL_NAME], rffi.INT)
 ssl_external('pypy_GENERAL_NAME_dirn', [GENERAL_NAME], X509_NAME,
              macro=True)
+
 ssl_external('pypy_GENERAL_NAME_uri', [GENERAL_NAME], ASN1_IA5STRING,
              macro=True)
 
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -223,7 +223,7 @@
             pass
 
 if _WIN32:
-    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h']
+    includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h']
     libraries = []
 else:
     if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')):
@@ -254,10 +254,11 @@
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
+    CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
     if not _WIN32:
         UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT)
         GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT)
-        CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT)
+        TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ')
 
         TMS = rffi_platform.Struct(
             'struct tms', [('tms_utime', rffi.INT),
@@ -265,6 +266,12 @@
                            ('tms_cutime', rffi.INT),
                            ('tms_cstime', rffi.INT)])
 
+        WINSIZE = rffi_platform.Struct(
+            'struct winsize', [('ws_row', rffi.USHORT),
+                           ('ws_col', rffi.USHORT),
+                           ('ws_xpixel', rffi.USHORT),
+                           ('ws_ypixel', rffi.USHORT)])
+
     GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)")
     SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)")
 
@@ -365,15 +372,27 @@
         raise OSError(get_saved_errno(), '%s failed' % name)
     return result
 
+def _dup(fd, inheritable=True):
+    validate_fd(fd)
+    if inheritable:
+        res = c_dup(fd)
+    else:
+        res = c_dup_noninheritable(fd)
+    return res
+
 @replace_os_function('dup')
-def dup(fd):
-    validate_fd(fd)
-    return handle_posix_error('dup', c_dup(fd))
+def dup(fd, inheritable=True):
+    res = _dup(fd, inheritable)
+    return handle_posix_error('dup', res)
 
 @replace_os_function('dup2')
-def dup2(fd, newfd):
+def dup2(fd, newfd, inheritable=True):
     validate_fd(fd)
-    handle_posix_error('dup2', c_dup2(fd, newfd))
+    if inheritable:
+        res = c_dup2(fd, newfd)
+    else:
+        res = c_dup2_noninheritable(fd, newfd)
+    handle_posix_error('dup2', res)
 
 #___________________________________________________________________
 
@@ -604,7 +623,8 @@
     class CConfig:
         _compilation_info_ = eci
         DIRENT = rffi_platform.Struct('struct dirent',
-            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))]
+            [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1)),
+             ('d_ino', lltype.Signed)]
             + [('d_type', rffi.INT)] if HAVE_D_TYPE else [])
         if HAVE_D_TYPE:
             DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN')
@@ -628,6 +648,8 @@
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
     c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
 else:
     dirent_config = {}
 
@@ -1113,37 +1135,69 @@
     c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T,
                                                     rffi.INT],
                                 rffi.INT)
+    HAVE_PIPE2 = False
+    HAVE_DUP3 = False
+    O_CLOEXEC = None
 else:
     INT_ARRAY_P = rffi.CArrayPtr(rffi.INT)
     c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT,
                       save_err=rffi.RFFI_SAVE_ERRNO)
+    class CConfig:
+        _compilation_info_ = eci
+        HAVE_PIPE2 = rffi_platform.Has('pipe2')
+        HAVE_DUP3 = rffi_platform.Has('dup3')
+        O_CLOEXEC = rffi_platform.DefinedConstantInteger('O_CLOEXEC')
+    config = rffi_platform.configure(CConfig)
+    HAVE_PIPE2 = config['HAVE_PIPE2']
+    HAVE_DUP3 = config['HAVE_DUP3']
+    O_CLOEXEC = config['O_CLOEXEC']
+    if HAVE_PIPE2:
+        c_pipe2 = external('pipe2', [INT_ARRAY_P, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
 @replace_os_function('pipe')
-def pipe():
+def pipe(flags=0):
+    # 'flags' might be ignored.  Check the result.
     if _WIN32:
-        pread  = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw')
-        try:
-            if not CreatePipe(
-                    pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
-                raise WindowsError(rwin32.GetLastError_saved(),
-                                   "CreatePipe failed")
-            hread = rffi.cast(rffi.INTPTR_T, pread[0])
-            hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0])
-        finally:
-            lltype.free(pwrite, flavor='raw')
-            lltype.free(pread, flavor='raw')
-        fdread = c_open_osfhandle(hread, 0)
-        fdwrite = c_open_osfhandle(hwrite, 1)
-        return (fdread, fdwrite)
+        # 'flags' ignored
+        ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
+        walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
+        with ralloc as pread, walloc as pwrite:
+            if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0):
+                hread = pread[0]
+                hwrite = pwrite[0]
+                fdread = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hread), 0)
+                fdwrite = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hwrite), 1)
+                if not (fdread == -1 or fdwrite == -1):
+                    return (fdread, fdwrite)
+                rwin32.CloseHandle(hread)
+                rwin32.CloseHandle(hwrite)
+        raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed")
     else:
         filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
         try:
-            handle_posix_error('pipe', c_pipe(filedes))
+            if HAVE_PIPE2 and _pipe2_syscall.attempt_syscall():
+                res = c_pipe2(filedes, flags)
+                if _pipe2_syscall.fallback(res):
+                    res = c_pipe(filedes)
+            else:
+                res = c_pipe(filedes)      # 'flags' ignored
+            handle_posix_error('pipe', res)
             return (widen(filedes[0]), widen(filedes[1]))
         finally:
             lltype.free(filedes, flavor='raw')
 
+def pipe2(flags):
+    # Only available if there is really a c_pipe2 function.
+    # No fallback to pipe() if we get ENOSYS.
+    filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw')
+    try:
+        res = c_pipe2(filedes, flags)
+        handle_posix_error('pipe2', res)
+        return (widen(filedes[0]), widen(filedes[1]))
+    finally:
+        lltype.free(filedes, flavor='raw')
+
 c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
                   save_err=rffi.RFFI_SAVE_ERRNO,)
 c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT,
@@ -2079,14 +2133,47 @@
 
 
 eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=["""
+    separate_module_sources=[r"""
+#include <errno.h>
+#include <sys/ioctl.h>
+
 RPY_EXTERN
 int rpy_set_inheritable(int fd, int inheritable)
 {
-    /* XXX minimal impl. XXX */
-    int request = inheritable ? FIONCLEX : FIOCLEX;
-    return ioctl(fd, request, NULL);
+    static int ioctl_works = -1;
+    int flags;
+
+    if (ioctl_works != 0) {
+        int request = inheritable ? FIONCLEX : FIOCLEX;
+        int err = ioctl(fd, request, NULL);
+        if (!err) {
+            ioctl_works = 1;
+            return 0;
+        }
+
+        if (errno != ENOTTY && errno != EACCES) {
+            return -1;
+        }
+        else {
+            /* ENOTTY: The ioctl is declared but not supported by the
+               kernel.  EACCES: SELinux policy, this can be the case on
+               Android. */
+            ioctl_works = 0;
+        }
+        /* fallback to fcntl() if ioctl() does not work */
+    }
+
+    flags = fcntl(fd, F_GETFD);
+    if (flags < 0)
+        return -1;
+
+    if (inheritable)
+        flags &= ~FD_CLOEXEC;
+    else
+        flags |= FD_CLOEXEC;
+    return fcntl(fd, F_SETFD, flags);
 }
+
 RPY_EXTERN
 int rpy_get_inheritable(int fd)
 {
@@ -2095,8 +2182,64 @@
         return -1;
     return !(flags & FD_CLOEXEC);
 }
-    """],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);']))
+
+RPY_EXTERN
+int rpy_dup_noninheritable(int fd)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUPFD_CLOEXEC
+    return fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    fd = dup(fd);
+    if (fd >= 0) {
+        if (rpy_set_inheritable(fd, 0) != 0) {
+            close(fd);
+            return -1;
+        }
+    }
+    return fd;
+#endif
+}
+
+RPY_EXTERN
+int rpy_dup2_noninheritable(int fd, int fd2)
+{
+#ifdef _WIN32
+#error NotImplementedError
+#endif
+
+#ifdef F_DUP2FD_CLOEXEC
+    return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
+
+#else
+# if %(HAVE_DUP3)d   /* HAVE_DUP3 */
+    static int dup3_works = -1;
+    if (dup3_works != 0) {
+        if (dup3(fd, fd2, O_CLOEXEC) >= 0)
+            return 0;
+        if (dup3_works == -1)
+            dup3_works = (errno != ENOSYS);
+        if (dup3_works)
+            return -1;
+    }
+# endif
+    if (dup2(fd, fd2) < 0)
+        return -1;
+    if (rpy_set_inheritable(fd2, 0) != 0) {
+        close(fd2);
+        return -1;
+    }
+    return 0;
+#endif
+}
+    """ % {'HAVE_DUP3': HAVE_DUP3}],
+    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
 
 c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
@@ -2104,12 +2247,56 @@
 c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
                              rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
                              compilation_info=eci_inheritable)
+c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                compilation_info=eci_inheritable)
+c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                 compilation_info=eci_inheritable)
 
 def set_inheritable(fd, inheritable):
-    error = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', error)
+    result = c_set_inheritable(fd, inheritable)
+    handle_posix_error('set_inheritable', result)
 
 def get_inheritable(fd):
     res = c_get_inheritable(fd)
     res = handle_posix_error('get_inheritable', res)
     return res != 0
+
+class SetNonInheritableCache(object):
+    """Make one prebuilt instance of this for each path that creates
+    file descriptors, where you don't necessarily know if that function
+    returns inheritable or non-inheritable file descriptors.
+    """
+    _immutable_fields_ = ['cached_inheritable?']
+    cached_inheritable = -1    # -1 = don't know yet; 0 = off; 1 = on
+
+    def set_non_inheritable(self, fd):
+        if self.cached_inheritable == -1:
+            self.cached_inheritable = get_inheritable(fd)
+        if self.cached_inheritable == 1:
+            # 'fd' is inheritable; we must manually turn it off
+            set_inheritable(fd, False)
+
+    def _cleanup_(self):
+        self.cached_inheritable = -1
+
+class ENoSysCache(object):
+    """Cache whether a system call returns ENOSYS or not."""
+    _immutable_fields_ = ['cached_nosys?']
+    cached_nosys = -1      # -1 = don't know; 0 = no; 1 = yes, getting ENOSYS
+
+    def attempt_syscall(self):
+        return self.cached_nosys != 1
+
+    def fallback(self, res):
+        nosys = self.cached_nosys
+        if nosys == -1:
+            nosys = (res < 0 and get_saved_errno() == errno.ENOSYS)
+            self.cached_nosys = nosys
+        return nosys
+
+    def _cleanup_(self):
+        self.cached_nosys = -1
+
+_pipe2_syscall = ENoSysCache()
diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -50,3 +50,6 @@
     if rposix.HAVE_D_TYPE:
         return rffi.getintfield(direntp, 'c_d_type')
     return DT_UNKNOWN
+
+def get_inode(direntp):
+    return rffi.getintfield(direntp, 'c_d_ino')
diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py
--- a/rpython/rlib/rsignal.py
+++ b/rpython/rlib/rsignal.py
@@ -31,7 +31,7 @@
     signal_names.append('CTRL_BREAK_EVENT')
     CTRL_C_EVENT = 0
     CTRL_BREAK_EVENT = 1
-includes = ['stdlib.h', 'src/signals.h']
+includes = ['stdlib.h', 'src/signals.h', 'signal.h']
 if sys.platform != 'win32':
     includes.append('sys/time.h')
 
@@ -47,7 +47,9 @@
     _compilation_info_ = eci
 
 if sys.platform != 'win32':
-    for name in """ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF""".split():
+    for name in """
+        ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF
+        SIG_BLOCK SIG_UNBLOCK SIG_SETMASK""".split():
         setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name))
 
     CConfig.timeval = rffi_platform.Struct(
@@ -71,7 +73,8 @@
 pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void)
 pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void)
 pypysig_reinstall = external('pypysig_reinstall', [rffi.INT], lltype.Void)
-pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', [rffi.INT], rffi.INT)
+pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd',
+                                 [rffi.INT, rffi.INT], rffi.INT)
 pypysig_poll = external('pypysig_poll', [], rffi.INT, releasegil=False)
 # don't bother releasing the GIL around a call to pypysig_poll: it's
 # pointless and a performance issue
@@ -98,3 +101,20 @@
                            [rffi.INT, itimervalP, itimervalP], rffi.INT,
                            save_err=rffi.RFFI_SAVE_ERRNO)
     c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT)
+
+c_pthread_kill = external('pthread_kill', [lltype.Signed, rffi.INT], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+
+if sys.platform != 'win32':
+    c_sigset_t = rffi.COpaquePtr('sigset_t', compilation_info=eci)
+    c_sigemptyset = external('sigemptyset', [c_sigset_t], rffi.INT)
+    c_sigaddset = external('sigaddset', [c_sigset_t, rffi.INT], rffi.INT)
+    c_sigismember = external('sigismember', [c_sigset_t, rffi.INT], rffi.INT)
+    c_sigwait = external('sigwait', [c_sigset_t, rffi.INTP], rffi.INT,
+                         releasegil=True,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    c_sigpending = external('sigpending', [c_sigset_t], rffi.INT,
+                            save_err=rffi.RFFI_SAVE_ERRNO)
+    c_pthread_sigmask = external('pthread_sigmask',
+                                 [rffi.INT, c_sigset_t, c_sigset_t], rffi.INT,
+                                 save_err=rffi.RFFI_SAVE_ERRNO)
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -8,10 +8,11 @@
 # XXX this does not support yet the least common AF_xxx address families
 # supported by CPython.  See http://bugs.pypy.org/issue1942
 
+from errno import EINVAL
 from rpython.rlib import _rsocket_rffi as _c, jit, rgc
 from rpython.rlib.objectmodel import instantiate, keepalive_until_here
 from rpython.rlib.rarithmetic import intmask, r_uint
-from rpython.rlib import rthread
+from rpython.rlib import rthread, rposix
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rffi import sizeof, offsetof
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -522,12 +523,28 @@
     timeout = -1.0
 
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0,
-                 fd=_c.INVALID_SOCKET):
+                 fd=_c.INVALID_SOCKET, inheritable=True):
         """Create a new socket."""
         if _c.invalid_socket(fd):
-            fd = _c.socket(family, type, proto)
-        if _c.invalid_socket(fd):
-            raise self.error_handler()
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
+                # Non-inheritable: we try to call socket() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                fd = _c.socket(family, type | SOCK_CLOEXEC, proto)
+                if fd < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise self.error_handler()
+            if _c.invalid_socket(fd):
+                fd = _c.socket(family, type, proto)
+                if _c.invalid_socket(fd):
+                    raise self.error_handler()
+                if not inheritable:
+                    sock_set_inheritable(fd, False)
         # PLAT RISCOS
         self.fd = fd
         self.family = family
@@ -630,20 +647,33 @@
         return addr, addr.addr_p, addrlen_p
 
     @jit.dont_look_inside
-    def accept(self):
+    def accept(self, inheritable=True):
         """Wait for an incoming connection.
         Return (new socket fd, client address)."""
         if self._select(False) == 1:
             raise SocketTimeout
         address, addr_p, addrlen_p = self._addrbuf()
         try:
-            newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+            remove_inheritable = not inheritable
+            if (not inheritable and 'SOCK_CLOEXEC' in constants
+                    and _c.HAVE_ACCEPT4
+                    and _accept4_syscall.attempt_syscall()):
+                newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p,
+                                         SOCK_CLOEXEC)
+                if _accept4_syscall.fallback(newfd):
+                    newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
+                else:
+                    remove_inheritable = False
+            else:
+                newfd = _c.socketaccept(self.fd, addr_p, addrlen_p)
             addrlen = addrlen_p[0]
         finally:
             lltype.free(addrlen_p, flavor='raw')
             address.unlock()
         if _c.invalid_socket(newfd):
             raise self.error_handler()
+        if remove_inheritable:
+            sock_set_inheritable(newfd, False)
         address.addrlen = rffi.cast(lltype.Signed, addrlen)
         return (newfd, address)
 
@@ -1032,6 +1062,12 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
+def sock_set_inheritable(fd, inheritable):
+    try:
+        rposix.set_inheritable(fd, inheritable)
+    except OSError as e:
+        raise CSocketError(e.errno)
+
 class SocketError(Exception):
     applevelerrcls = 'error'
     def __init__(self):
@@ -1090,7 +1126,7 @@
 
 if hasattr(_c, 'socketpair'):
     def socketpair(family=socketpair_default_family, type=SOCK_STREAM, proto=0,
-                   SocketClass=RSocket):
+                   SocketClass=RSocket, inheritable=True):
         """socketpair([family[, type[, proto]]]) -> (socket object, socket object)
 
         Create a pair of socket objects from the sockets returned by the platform
@@ -1100,18 +1136,41 @@
         """
         result = lltype.malloc(_c.socketpair_t, 2, flavor='raw')
         try:
-            res = _c.socketpair(family, type, proto, result)
+            res = -1
+            remove_inheritable = not inheritable
+            if not inheritable and 'SOCK_CLOEXEC' in constants:
+                # Non-inheritable: we try to call socketpair() with
+                # SOCK_CLOEXEC, which may fail.  If we get EINVAL,
+                # then we fall back to the SOCK_CLOEXEC-less case.
+                res = _c.socketpair(family, type | SOCK_CLOEXEC,
+                                    proto, result)
+                if res < 0:
+                    if _c.geterrno() == EINVAL:
+                        # Linux older than 2.6.27 does not support
+                        # SOCK_CLOEXEC.  An EINVAL might be caused by
+                        # random other things, though.  Don't cache.
+                        pass
+                    else:
+                        raise last_error()
+                else:
+                    remove_inheritable = False
+            #
             if res < 0:
-                raise last_error()
+                res = _c.socketpair(family, type, proto, result)
+                if res < 0:
+                    raise last_error()
             fd0 = rffi.cast(lltype.Signed, result[0])
             fd1 = rffi.cast(lltype.Signed, result[1])
         finally:
             lltype.free(result, flavor='raw')
+        if remove_inheritable:
+            sock_set_inheritable(fd0, False)
+            sock_set_inheritable(fd1, False)
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
 if _c.WIN32:
-    def dup(fd):
+    def dup(fd, inheritable=True):
         with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info:
             if _c.WSADuplicateSocket(fd, rwin32.GetCurrentProcessId(), info):
                 raise last_error()
@@ -1122,15 +1181,16 @@
                 raise last_error()
             return result
 else:
-    def dup(fd):
-        return _c.dup(fd)
-

From pypy.commits at gmail.com  Fri Sep  2 02:49:15 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Sep 2016 23:49:15 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: fix micronumpy test_zjit by
 adding _attrs_ to the class
Message-ID: <57c920eb.05d71c0a.db689.9858@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86835:6cebc47b780b
Date: 2016-09-02 09:47 +0300
http://bitbucket.org/pypy/pypy/changeset/6cebc47b780b/

Log:	fix micronumpy test_zjit by adding _attrs_ to the class

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -14,6 +14,7 @@
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
+    _attrs_ = ['buf']
 
     def __init__(self, buf):
         assert isinstance(buf, Buffer)

From pypy.commits at gmail.com  Fri Sep  2 03:11:51 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 02 Sep 2016 00:11:51 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Remove object layout hack,
 update qcgc codebase
Message-ID: <57c92637.44ce1c0a.c2e76.9f88@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86837:d98166b96e01
Date: 2016-09-02 09:11 +0200
http://bitbucket.org/pypy/pypy/changeset/d98166b96e01/

Log:	Remove object layout hack, update qcgc codebase

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -1,5 +1,5 @@
 from rpython.memory.gc.base import GCBase
-#from rpython.memory.support import mangle_hash
+from rpython.memory.support import mangle_hash
 from rpython.rtyper.lltypesystem import rffi, lltype, llgroup, llmemory, llarena
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
@@ -19,22 +19,19 @@
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
 
     typeid_is_in_field = 'tid'
-    withhash_flag_is_in_field = 'flags', QCGC_HAS_HASH
 
     TRANSLATION_PARAMS = {}
     HDR = lltype.Struct(
             'header',
-            #('hdr', rffi.COpaque('object_t', hints={"is_qcgc_header": True})),
-            ('flags', lltype.Signed),   # XXX: exploits knowledge about object_t
+            ('hdr', rffi.COpaque('object_t', hints={"is_qcgc_header": True})),
             ('tid', lltype.Signed),
             ('hash', lltype.Signed))
     #HDR = rffi.COpaque('object_t')
 
-    def init_gc_object(self, addr, typeid, flags=0):
+    def init_gc_object(self, addr, typeid):
         hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.flags = rffi.cast(lltype.Signed, flags)
         hdr.tid = rffi.cast(lltype.Signed, typeid)
-        hdr.hash = rffi.cast(lltype.Signed, addr)
+        hdr.hash = rffi.cast(lltype.Signed, 0)
 
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False,
@@ -63,15 +60,10 @@
         (obj + offset_to_length).signed[0] = length
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
-    def init_gc_object_immortal(self, addr, typeid, flags=0): # XXX: Prebuilt Objects?
+    def init_gc_object_immortal(self, addr, typeid, flags=0):
         assert flags == 0
-        ptr = self.gcheaderbuilder.object_from_header(addr.ptr)
-        prebuilt_hash = lltype.identityhash_nocache(ptr)
-        assert prebuilt_hash != 0
-        flags |= QCGC_PREBUILT_OBJECT
         #
-        self.init_gc_object(addr, typeid.index, flags)
-        llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)).hash = prebuilt_hash
+        self.init_gc_object(addr, typeid.index)
 
     def collect(self, gen=1):
         """Do a minor (gen=0) or major (gen>0) collection."""
@@ -84,11 +76,7 @@
         # this. Unfortunately I don't fully understand what this is supposed to
         # do, so I can't optimize it ATM.
         return False
-        # Possible implementation?
-        #llop.gc_writebarrier(dest_addr)
-        #return True
 
-    # XXX: WRITE BARRIER
     def write_barrier(self, addr_struct):
         llop.qcgc_write_barrier(lltype.Void, addr_struct)
 
@@ -97,14 +85,14 @@
         pass
 
     def id_or_identityhash(self, gcobj, is_hash):
-        hdr = self.header(llmemory.cast_ptr_to_adr(gcobj))
-        has_hash = (hdr.flags & QCGC_HAS_HASH)
+        obj = llmemory.cast_ptr_to_adr(gcobj)
+        hdr = self.header(obj)
         i = hdr.hash
         #
-        if is_hash:
-            if has_hash:
-                return i # Do not mangle for objects with built in hash
-            i = i ^ (i >> 5)
+        if i == 0:
+            i = llmemory.cast_adr_to_int(obj)
+            if is_hash:
+                i = mangle_hash(i)
         return i
 
     def id(self, gcobje):
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -36,16 +36,10 @@
 
     def gc_header_for(self, obj, needs_hash=False):
         hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
-        withhash, flag = self.gcdata.gc.withhash_flag_is_in_field
-        x = getattr(hdr, withhash)
-        TYPE = lltype.typeOf(x)
-        x = lltype.cast_primitive(lltype.Signed, x)
         if needs_hash:
-            x |= flag       # set the flag in the header
+            hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
         else:
-            x &= ~flag      # clear the flag in the header
-        x = lltype.cast_primitive(TYPE, x)
-        setattr(hdr, withhash, x)
+            assert hdr.hash == 0
         return hdr
 
     def push_roots(self, hop, keep_current_args=False):
diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
--- a/rpython/translator/c/src/qcgc/allocator.c
+++ b/rpython/translator/c/src/qcgc/allocator.c
@@ -11,6 +11,7 @@
 
 QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells);
 QCGC_STATIC void bump_allocator_advance(size_t cells);
+QCGC_STATIC void bump_allocator_renew_block(void);
 
 QCGC_STATIC bool is_small(size_t cells);
 QCGC_STATIC size_t small_index(size_t cells);
@@ -26,6 +27,7 @@
 void qcgc_allocator_initialize(void) {
 	qcgc_allocator_state.arenas =
 		qcgc_arena_bag_create(QCGC_ARENA_BAG_INIT_SIZE);
+	qcgc_allocator_state.free_arenas = qcgc_arena_bag_create(4); // XXX
 
 	// Bump Allocator
 	qcgc_allocator_state.bump_state.bump_ptr = NULL;
@@ -59,10 +61,24 @@
 		qcgc_arena_destroy(qcgc_allocator_state.arenas->items[i]);
 	}
 
+	arena_count = qcgc_allocator_state.free_arenas->count;
+	for (size_t i = 0; i < arena_count; i++) {
+		qcgc_arena_destroy(qcgc_allocator_state.free_arenas->items[i]);
+	}
+
 	free(qcgc_allocator_state.arenas);
+	free(qcgc_allocator_state.free_arenas);
 }
 
 void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
+#if CHECKED
+	if (cells > 0) {
+		assert((((object_t *)ptr)->flags & QCGC_PREBUILT_OBJECT) == 0);
+		assert((cell_t *) qcgc_arena_addr(ptr) != ptr);
+		assert(qcgc_arena_get_blocktype(ptr) == BLOCK_FREE ||
+				qcgc_arena_get_blocktype(ptr) == BLOCK_EXTENT);
+	}
+#endif
 	if (cells > 0) {
 		if (is_small(cells)) {
 			size_t index = small_index(cells);
@@ -80,30 +96,17 @@
 	}
 }
 
-QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells) {
-	qcgc_allocator_state.bump_state.bump_ptr = ptr;
-	qcgc_allocator_state.bump_state.remaining_cells = cells;
-}
-
-QCGC_STATIC void bump_allocator_advance(size_t cells) {
-	qcgc_allocator_state.bump_state.bump_ptr += cells;
-	qcgc_allocator_state.bump_state.remaining_cells -= cells;
-}
-
 /*******************************************************************************
- * Allocators                                                                  *
+ * Bump Allocator                                                              *
  ******************************************************************************/
 
 object_t *qcgc_bump_allocate(size_t bytes) {
+#if CHECKED
+	assert(bytes <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP);
+#endif
 	size_t cells = bytes_to_cells(bytes);
 	if (cells > qcgc_allocator_state.bump_state.remaining_cells) {
-		// Grab a new arena
-		// FIXME: Add remaining memory to fit allocator
-		arena_t *arena = qcgc_arena_create();
-		bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
-				QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
-		qcgc_allocator_state.arenas =
-			qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
+		bump_allocator_renew_block();
 	}
 	cell_t *mem = qcgc_allocator_state.bump_state.bump_ptr;
 	bump_allocator_advance(cells);
@@ -116,9 +119,92 @@
 #endif
 
 	result->flags |= QCGC_GRAY_FLAG;
+#if CHECKED
+	assert(qcgc_arena_is_coalesced(qcgc_arena_addr((cell_t *)result)));
+	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
+		assert(qcgc_arena_get_blocktype(
+					qcgc_allocator_state.bump_state.bump_ptr) == BLOCK_FREE);
+		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
+				i++) {
+			assert(qcgc_arena_get_blocktype(
+						qcgc_allocator_state.bump_state.bump_ptr + i)
+					== BLOCK_EXTENT);
+		}
+	}
+#endif
 	return result;
 }
 
+QCGC_STATIC void bump_allocator_renew_block(void) {
+#if CHECKED
+	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
+		assert(qcgc_arena_get_blocktype(
+					qcgc_allocator_state.bump_state.bump_ptr) == BLOCK_FREE);
+		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
+				i++) {
+			assert(qcgc_arena_get_blocktype(
+						qcgc_allocator_state.bump_state.bump_ptr + i)
+					== BLOCK_EXTENT);
+		}
+	}
+#endif
+	// Add remaining memory to fit allocator
+	qcgc_fit_allocator_add(qcgc_allocator_state.bump_state.bump_ptr,
+			qcgc_allocator_state.bump_state.remaining_cells);
+
+	// Try finding some huge block from fit allocator
+	exp_free_list_t *free_list = qcgc_allocator_state.fit_state.
+		large_free_list[QCGC_LARGE_FREE_LISTS - 1];
+	while (free_list->count > 0 && !valid_block(free_list->items[0].ptr,
+				free_list->items[0].size)) {
+		free_list = qcgc_exp_free_list_remove_index(free_list, 0);
+	}
+
+	if (free_list->count > 0) {
+		// Assign huge block to bump allocator
+		bump_allocator_assign(free_list->items[0].ptr,
+				free_list->items[0].size);
+		free_list = qcgc_exp_free_list_remove_index(free_list, 0);
+	} else {
+		// Grab a new arena
+		arena_t *arena = qcgc_arena_create();
+		bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
+				QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
+		qcgc_allocator_state.arenas =
+			qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
+	}
+
+	qcgc_allocator_state.fit_state.
+		large_free_list[QCGC_LARGE_FREE_LISTS - 1] = free_list;
+#if CHECKED
+	assert(qcgc_allocator_state.bump_state.bump_ptr != NULL);
+	assert(qcgc_arena_get_blocktype(qcgc_allocator_state.bump_state.bump_ptr) ==
+			BLOCK_FREE);
+	for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
+			i++) {
+		assert(qcgc_arena_get_blocktype(
+					qcgc_allocator_state.bump_state.bump_ptr + i)
+				== BLOCK_EXTENT);
+	}
+#endif
+}
+
+QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells) {
+#if CHECKED
+	assert(qcgc_arena_get_blocktype(ptr) == BLOCK_FREE);
+	for (size_t i = 1; i < cells; i++) {
+		assert(qcgc_arena_get_blocktype(ptr + i) == BLOCK_EXTENT);
+	}
+#endif
+	qcgc_allocator_state.bump_state.bump_ptr = ptr;
+	qcgc_allocator_state.bump_state.remaining_cells = cells;
+}
+
+QCGC_STATIC void bump_allocator_advance(size_t cells) {
+	qcgc_allocator_state.bump_state.bump_ptr += cells;
+	qcgc_allocator_state.bump_state.remaining_cells -= cells;
+}
+
 object_t *qcgc_fit_allocate(size_t bytes) {
 	size_t cells = bytes_to_cells(bytes);
 	cell_t *mem;
@@ -291,7 +377,7 @@
 	cells = cells >> QCGC_LARGE_FREE_LIST_FIRST_EXP;
 
 	// calculates floor(log(cells))
-	return (8 * sizeof(unsigned long)) - __builtin_clzl(cells) - 1;
+	return MIN((8 * sizeof(unsigned long)) - __builtin_clzl(cells) - 1, QCGC_LARGE_FREE_LISTS - 1);
 }
 
 QCGC_STATIC size_t small_index_to_cells(size_t index) {
@@ -306,6 +392,7 @@
 	assert(ptr != NULL);
 	assert(cells > 0);
 #endif
-	return (qcgc_arena_get_blocktype(ptr) == BLOCK_FREE &&
-			qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT);
+	return (qcgc_arena_get_blocktype(ptr) == BLOCK_FREE && (
+				((qcgc_arena_addr(ptr + cells)) == (arena_t *) (ptr + cells)) ||
+				qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT));
 }
diff --git a/rpython/translator/c/src/qcgc/allocator.h b/rpython/translator/c/src/qcgc/allocator.h
--- a/rpython/translator/c/src/qcgc/allocator.h
+++ b/rpython/translator/c/src/qcgc/allocator.h
@@ -17,6 +17,7 @@
  *                +---+---+-----+----+
  * size (cells):  | 1 | 2 | ... | 31 |
  *                +---+---+-----+----+
+ * (31 is 2^QCGC_LARGE_FREE_LIST_FIRST_EXP - 1)
  *
  * Large free lists:
  *                        +-----+-----+-----+---------+
@@ -24,18 +25,21 @@
  *                        +-----+-----+-----+---------+
  * minimal size (cells):  | 2^5 | 2^6 | ... | 2^(x+5) |
  *                        +-----+-----+-----+---------+
+ * (5 is QCGC_LARGE_FREE_LIST_FIRST_EXP)
  *
- * where x is chosen such that x + 5 + 1 = QCGC_ARENA_SIZE_EXP - 4 (i.e. the
- * next bin would hold chunks that have the size of at least one arena size,
- * which is impossible as an arena contains overhead)
+ * where x is chosen such that 2^(x + 5) = 2^QCGC_LARGE_ALLOC_THRESHOLD_EXP
+ * (i.e. such that the last bin contains all blocks that are larger or equal
+ * than the threshold for huge blocks. These blocks can be returned to the
+ * bump allocator)
  */
-
-#define QCGC_LARGE_FREE_LISTS (QCGC_ARENA_SIZE_EXP - 4 - QCGC_LARGE_FREE_LIST_FIRST_EXP)
+#define QCGC_LARGE_FREE_LISTS (QCGC_LARGE_ALLOC_THRESHOLD_EXP - QCGC_LARGE_FREE_LIST_FIRST_EXP - 4 + 1)
+// -4 because of turning bytes into cells, +1 because we start to count at 0
 
 #define QCGC_SMALL_FREE_LISTS ((1<<QCGC_LARGE_FREE_LIST_FIRST_EXP) - 1)
 
 struct qcgc_allocator_state {
 	arena_bag_t *arenas;
+	arena_bag_t *free_arenas;
 	struct bump_state {
 		cell_t *bump_ptr;
 		size_t remaining_cells;
diff --git a/rpython/translator/c/src/qcgc/arena.c b/rpython/translator/c/src/qcgc/arena.c
--- a/rpython/translator/c/src/qcgc/arena.c
+++ b/rpython/translator/c/src/qcgc/arena.c
@@ -5,6 +5,10 @@
 #include <sys/mman.h>
 #include <unistd.h>
 
+#if DEBUG_ZERO_ON_SWEEP
+#include <string.h>
+#endif
+
 #include "allocator.h"
 #include "event_logger.h"
 
@@ -146,12 +150,27 @@
 void qcgc_arena_mark_allocated(cell_t *ptr, size_t cells) {
 	size_t index = qcgc_arena_cell_index(ptr);
 	arena_t *arena = qcgc_arena_addr(ptr);
+#if CHECKED
+	assert(get_blocktype(arena, index) == BLOCK_FREE);
+	for (size_t i = 1; i < cells; i++) {
+		assert(get_blocktype(arena, index + i) == BLOCK_EXTENT);
+	}
+#endif
 	set_blocktype(arena, index, BLOCK_WHITE);
 	size_t index_of_next_block = index + cells;
 	if (index_of_next_block < QCGC_ARENA_CELLS_COUNT &&
 			get_blocktype(arena, index_of_next_block) == BLOCK_EXTENT) {
 		set_blocktype(arena, index_of_next_block, BLOCK_FREE);
 	}
+#if CHECKED
+	assert(get_blocktype(arena, index) == BLOCK_WHITE);
+	for (size_t i = 1; i < cells; i++) {
+		assert(get_blocktype(arena, index + i) == BLOCK_EXTENT);
+	}
+	if (index_of_next_block < QCGC_ARENA_CELLS_COUNT) {
+		assert(get_blocktype(arena, index + cells) != BLOCK_EXTENT);
+	}
+#endif
 }
 
 void qcgc_arena_mark_free(cell_t *ptr) {
@@ -164,15 +183,35 @@
 	assert(arena != NULL);
 	assert(qcgc_arena_is_coalesced(arena));
 #endif
+#if DEBUG_ZERO_ON_SWEEP
+	bool zero = true;
+#endif
 	bool free = true;
 	bool coalesce = false;
 	bool add_to_free_list = false;
 	size_t last_free_cell = QCGC_ARENA_FIRST_CELL_INDEX;
+
+	if (qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) == arena) {
+		for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
+				cell < QCGC_ARENA_CELLS_COUNT;
+				cell++) {
+			if (get_blocktype(arena, cell) == BLOCK_BLACK) {
+				set_blocktype(arena, cell, BLOCK_WHITE);
+			}
+		}
+		return false;
+	}
+
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
-		switch (qcgc_arena_get_blocktype(arena->cells + cell)) {
+		switch (get_blocktype(arena, cell)) {
 			case BLOCK_EXTENT:
+#if DEBUG_ZERO_ON_SWEEP
+				if (zero) {
+					memset(&arena->cells[cell], 0, sizeof(cell_t));
+				}
+#endif
 				break;
 			case BLOCK_FREE:
 				if (coalesce) {
@@ -181,6 +220,10 @@
 					last_free_cell = cell;
 				}
 				coalesce = true;
+#if DEBUG_ZERO_ON_SWEEP
+				zero = true;
+				memset(&arena->cells[cell], 0, sizeof(cell_t));
+#endif
 				break;
 			case BLOCK_WHITE:
 				if (coalesce) {
@@ -191,25 +234,33 @@
 				}
 				coalesce = true;
 				add_to_free_list = true;
+#if DEBUG_ZERO_ON_SWEEP
+				zero = true;
+				memset(&arena->cells[cell], 0, sizeof(cell_t));
+#endif
 				break;
 			case BLOCK_BLACK:
 				set_blocktype(arena, cell, BLOCK_WHITE);
 				if (add_to_free_list) {
-					qcgc_fit_allocator_add(&(arena->cells[last_free_cell]),
+					qcgc_fit_allocator_add(arena->cells + last_free_cell,
 							cell - last_free_cell);
 				}
 				free = false;
 				coalesce = false;
 				add_to_free_list = false;
+#if DEBUG_ZERO_ON_SWEEP
+				zero = false;
+#endif
 				break;
 		}
 	}
 	if (add_to_free_list && !free) {
-		qcgc_fit_allocator_add(&(arena->cells[last_free_cell]),
+		qcgc_fit_allocator_add(arena->cells + last_free_cell,
 							QCGC_ARENA_CELLS_COUNT - last_free_cell);
 	}
 #if CHECKED
 	assert(qcgc_arena_is_coalesced(arena));
+	assert(free == qcgc_arena_is_empty(arena));
 #endif
 	return free;
 }
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -1,13 +1,15 @@
 #pragma once
 
-#define CHECKED 1							// Enable runtime sanity checks
+#define CHECKED 0							// Enable runtime sanity checks
+											// warning: huge performance impact
+#define DEBUG_ZERO_ON_SWEEP 0				// Zero memory on sweep (debug only)
 
 #define QCGC_INIT_ZERO 1					// Init new objects with zero bytes
 
 /**
  * Event logger
  */
-#define EVENT_LOG 1							// Enable event log
+#define EVENT_LOG 0							// Enable event log
 #define LOGFILE "./qcgc_events.log"			// Default logfile
 #define LOG_ALLOCATION 0					// Enable allocation log (warning:
 											// significant performance impact)
@@ -16,7 +18,7 @@
 											// shadow stack
 #define QCGC_ARENA_BAG_INIT_SIZE 16			// Initial size of the arena bag
 #define QCGC_ARENA_SIZE_EXP 20				// Between 16 (64kB) and 20 (1MB)
-#define QCGC_LARGE_ALLOC_THRESHOLD 1<<14
+#define QCGC_LARGE_ALLOC_THRESHOLD_EXP 14	// Less than QCGC_ARENA_SIZE_EXP
 #define QCGC_MARK_LIST_SEGMENT_SIZE 64		// TODO: Tune for performance
 #define QCGC_GRAY_STACK_INIT_SIZE 128		// TODO: Tune for performance
 #define QCGC_INC_MARK_MIN 64				// TODO: Tune for performance
@@ -32,8 +34,16 @@
  * DO NOT MODIFY BELOW HERE
  */
 
+#if QCGC_LARGE_ALLOC_THRESHOLD_EXP >= QCGC_ARENA_SIZE_EXP
+#error	"Inconsistent configuration. Huge block threshold must be smaller " \
+		"than the arena size."
+#endif
+
 #ifdef TESTING
 #define QCGC_STATIC
 #else
 #define QCGC_STATIC static
 #endif
+
+#define MAX(a,b) (((a)>(b))?(a):(b))
+#define MIN(a,b) (((a)<(b))?(a):(b))
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -10,10 +10,6 @@
 #include "hugeblocktable.h"
 #include "event_logger.h"
 
-// TODO: Eventually move to own header?
-#define MAX(a,b) (((a)>(b))?(a):(b))
-#define MIN(a,b) (((a)<(b))?(a):(b))
-
 void qcgc_mark(bool incremental);
 void qcgc_pop_object(object_t *object);
 void qcgc_push_object(object_t *object);
@@ -117,7 +113,7 @@
 			(uint8_t *) &size);
 #endif
 	object_t *result;
-	if (size <= QCGC_LARGE_ALLOC_THRESHOLD) {
+	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
 		// Use bump / fit allocator
 		if (true) { // FIXME: Implement reasonable switch
 			result = qcgc_bump_allocate(size);
@@ -138,9 +134,6 @@
 	qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
 			(uint8_t *) &result);
 #endif
-#if CHECKED
-	assert(qcgc_state.phase != GC_COLLECT);
-#endif
 	return result;
 }
 
@@ -167,9 +160,7 @@
 			return MARK_COLOR_BLACK;
 		}
 	} else {
-#if CHECKED
-		assert(false);
-#endif
+		return MARK_COLOR_INVALID;
 	}
 }
 
@@ -210,8 +201,8 @@
 
 		while (to_process > 0) {
 			object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
-			qcgc_state.gp_gray_stack =
-				qcgc_gray_stack_pop(qcgc_state.gp_gray_stack);
+			qcgc_state.gp_gray_stack = qcgc_gray_stack_pop(
+					qcgc_state.gp_gray_stack);
 			qcgc_pop_object(top);
 			to_process--;
 		}
@@ -225,10 +216,8 @@
 					(arena->gray_stack->index));
 
 			while (to_process > 0) {
-				object_t *top =
-					qcgc_gray_stack_top(arena->gray_stack);
-				arena->gray_stack =
-					qcgc_gray_stack_pop(arena->gray_stack);
+				object_t *top = qcgc_gray_stack_top(arena->gray_stack);
+				arena->gray_stack = qcgc_gray_stack_pop(arena->gray_stack);
 				qcgc_pop_object(top);
 				to_process--;
 			}
@@ -247,6 +236,7 @@
 	qcgc_event_logger_log(EVENT_MARK_DONE, 0, NULL);
 #if CHECKED
 	assert(incremental || (qcgc_state.phase = GC_COLLECT));
+	assert(qcgc_state.phase != GC_PAUSE);
 #endif
 }
 
@@ -319,8 +309,22 @@
 			(uint8_t *) &arena_count);
 
 	qcgc_hbtable_sweep();
-	for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
-		qcgc_arena_sweep(qcgc_allocator_state.arenas->items[i]);
+	size_t i = 0;
+	while (i < qcgc_allocator_state.arenas->count) {
+		arena_t *arena = qcgc_allocator_state.arenas->items[i];
+		// The arena that contains the bump pointer is autmatically skipped
+		if (qcgc_arena_sweep(arena)) {
+			// Free
+			qcgc_allocator_state.arenas = qcgc_arena_bag_remove_index(
+					qcgc_allocator_state.arenas, i);
+			qcgc_allocator_state.free_arenas = qcgc_arena_bag_add(
+					qcgc_allocator_state.free_arenas, arena);
+
+			// NO i++
+		} else {
+			// Not free
+			i++;
+		}
 	}
 	qcgc_state.phase = GC_PAUSE;
 
diff --git a/rpython/translator/c/src/qcgc/qcgc.h b/rpython/translator/c/src/qcgc/qcgc.h
--- a/rpython/translator/c/src/qcgc/qcgc.h
+++ b/rpython/translator/c/src/qcgc/qcgc.h
@@ -27,6 +27,7 @@
 	MARK_COLOR_LIGHT_GRAY,
 	MARK_COLOR_DARK_GRAY,
 	MARK_COLOR_BLACK,
+	MARK_COLOR_INVALID,
 } mark_color_t;
 
 /**

From pypy.commits at gmail.com  Fri Sep  2 09:57:45 2016
From: pypy.commits at gmail.com (sbauman)
Date: Fri, 02 Sep 2016 06:57:45 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Add test for edge case when
 forcing virtual args (currently failing)
Message-ID: <57c98559.12331c0a.4f134.4002@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86838:fde1ff5108d5
Date: 2016-09-01 22:12 -0400
http://bitbucket.org/pypy/pypy/changeset/fde1ff5108d5/

Log:	Add test for edge case when forcing virtual args (currently failing)

diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1006,6 +1006,41 @@
         self.check_target_token_count(2)
         self.check_trace_count(3)
 
+    def test_conflated_virtual_states(self):
+        # All cases are covered when forcing one component of the virtual state
+        # also forces an as yet unseen component.
+        # i.e. expect [NotVirtual, Virtual] and given a pair of aliasing virtual
+        # objects
+        driver = JitDriver(greens=[], reds=['i', 'v1', 'v2'])
+        class Box(object):
+            def __init__(self, v):
+                self.v = v
+
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        const = Box(X(0))
+        def f():
+            set_param(None, 'retrace_limit', -1)
+            set_param(None, 'threshold', 1)
+            i = 0
+            v1 = X(0)
+            v2 = X(0)
+            const.v = X(0)
+            while i < 17:
+                driver.jit_merge_point(i=i, v1=v1, v2=v2)
+                driver.can_enter_jit(i=i, v1=v1, v2=v2)
+                if i & 1 == 0:
+                    v1 = const.v
+                    v2 = X(i)
+                else:
+                    v1 = v2 = X(i)
+                i += 1
+            return None
+        self.meta_interp(f, [])
+        # assert did not crash
+
 
 class VirtualMiscTests:
     def test_multiple_equal_virtuals(self):

From pypy.commits at gmail.com  Fri Sep  2 11:14:35 2016
From: pypy.commits at gmail.com (vext01)
Date: Fri, 02 Sep 2016 08:14:35 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Another W^X site.
Message-ID: <57c9975b.8628c20a.f6b0b.c572@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86839:5e719144f315
Date: 2016-09-02 16:13 +0100
http://bitbucket.org/pypy/pypy/changeset/5e719144f315/

Log:	Another W^X site.

diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -8,7 +8,7 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.rarithmetic import intmask, is_emulated_long
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rmmap import alloc
+from rpython.rlib.rmmap import alloc, set_pages_executable, set_pages_writable
 from rpython.rlib.rdynload import dlopen, dlclose, dlsym, dlsym_byordinal
 from rpython.rlib.rdynload import DLOpenError, DLLHANDLE
 from rpython.rlib import jit, rposix
@@ -446,10 +446,13 @@
     def _more(self):
         chunk = rffi.cast(CLOSURES, alloc(CHUNK))
         count = CHUNK//rffi.sizeof(FFI_CLOSUREP.TO)
+        chunk_p = rffi.cast(rffi.CCHARP, chunk)
+        set_pages_writable(chunk_p, CHUNK)
         for i in range(count):
             rffi.cast(rffi.VOIDPP, chunk)[0] = self.free_list
             self.free_list = rffi.cast(rffi.VOIDP, chunk)
             chunk = rffi.ptradd(chunk, 1)
+        set_pages_executable(chunk_p, CHUNK)
 
     def alloc(self):
         if not self.free_list:

From pypy.commits at gmail.com  Fri Sep  2 11:14:37 2016
From: pypy.commits at gmail.com (vext01)
Date: Fri, 02 Sep 2016 08:14:37 -0700 (PDT)
Subject: [pypy-commit] pypy asmmemmgr-for-code-only: Mark an argument we
 send to a C-callback as writable.
Message-ID: <57c9975d.041f1c0a.fbb56.6686@mx.google.com>

Author: Edd Barrett <vext01 at gmail.com>
Branch: asmmemmgr-for-code-only
Changeset: r86840:42b309a4db14
Date: 2016-09-02 16:14 +0100
http://bitbucket.org/pypy/pypy/changeset/42b309a4db14/

Log:	Mark an argument we send to a C-callback as writable.

diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -540,9 +540,14 @@
                                          track_allocation=False)
         self.ll_userdata.callback = rffi.llhelper(CALLBACK_TP, func)
         self.ll_userdata.addarg = additional_arg
+
+        ll_closure_p = rffi.cast(rffi.CCHARP, self.ll_closure)
+        set_pages_writable(ll_closure_p, CHUNK)
         res = c_ffi_prep_closure(self.ll_closure, self.ll_cif,
                                  ll_callback, rffi.cast(rffi.VOIDP,
                                                         self.ll_userdata))
+        set_pages_executable(ll_closure_p, CHUNK)
+
         if not res == FFI_OK:
             raise LibFFIError
 

From pypy.commits at gmail.com  Fri Sep  2 11:27:54 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 02 Sep 2016 08:27:54 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: add a flags test from numpy to
 test_memoryobject.py, implement enough to run it
Message-ID: <57c99a7a.482cc20a.5879c.c8f3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86841:0c283916dcf6
Date: 2016-09-02 14:20 +0300
http://bitbucket.org/pypy/pypy/changeset/0c283916dcf6/

Log:	add a flags test from numpy to test_memoryobject.py, implement
	enough to run it

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,8 +1,8 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
-from pypy.module.cpyext.pyobject import PyObject
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, as_pyobj, incref
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -33,13 +33,80 @@
     raise an error if the object can't support a simpler view of its memory.
 
     0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
+    buf = space.call_method(w_obj, "__buffer__", space.newint(flags))
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    view.c_len = buf.getlength()
+    view.c_obj = as_pyobj(space, w_obj)
+    incref(space, view.c_obj)
+    ndim = buf.getndim()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_readonly', int(buf.readonly))
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.str2charp(buf.getformat())
+    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    if view.ndim == 0:
+        return 1
+    if not view.strides:
+        return view.ndim == 1
+    sd = view.itemsize
+    if view.ndim == 1:
+        return view.shape[0] == 1 or sd == view.strides[0]
+    for i in range(view.ndim):
+        dim = view.shape[i]
+        if dim == 0:
+            return 1
+        if view.strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    if view.ndim == 0:
+        return 1
+    if not view.strides:
+        return view.ndim == 1
+    sd = view.itemsize
+    if view.ndim == 1:
+        return view.shape[0] == 1 or sd == view.strides[0]
+    for i in range(view.ndim-1, -1, -1):
+        dim = view.shape[i]
+        if dim == 0:
+            return 1
+        if view.strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+        
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
+def PyBuffer_IsContiguous(space, view, fort):
     """Return 1 if the memory defined by the view is C-style (fortran is
     'C') or Fortran-style (fortran is 'F') contiguous or either one
     (fortran is 'A').  Return 0 otherwise."""
-    # PyPy only supports contiguous Py_buffers for now.
-    return 1
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
+    
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -13,6 +13,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -198,10 +198,118 @@
     return PyInt_FromLong(view->len);
 }
 
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG)                                        \
+    buf_flag = PyUnicode_FromString(#FLAG);                         \
+    flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ);  \
+    Py_DECREF(buf_flag);                                            \
+    if (flag_matches == 1) {                                        \
+        Py_DECREF(tmp);                                             \
+        flags |= PyBUF_##FLAG;                                      \
+        continue;                                                   \
+    }                                                               \
+    else if (flag_matches == -1) {                                  \
+        Py_DECREF(tmp);                                             \
+        return NULL;                                                \
+    }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+    PyObject *buffer_obj, *pyflags;
+    PyObject *tmp, *buf_flag;
+    Py_buffer buffer;
+    PyObject *shape, *strides;
+    Py_ssize_t i, n;
+    int flag_matches;
+    int flags = 0;
+
+    if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+        return NULL;
+    }
+
+    n = PySequence_Length(pyflags);
+    if (n < 0) {
+        return NULL;
+    }
+
+    for (i=0; i < n; i++) {
+        tmp = PySequence_GetItem(pyflags, i);
+        if (tmp == NULL) {
+            return NULL;
+        }
+
+        GET_PYBUF_FLAG(SIMPLE);
+        GET_PYBUF_FLAG(WRITABLE);
+        GET_PYBUF_FLAG(STRIDES);
+        GET_PYBUF_FLAG(ND);
+        GET_PYBUF_FLAG(C_CONTIGUOUS);
+        GET_PYBUF_FLAG(F_CONTIGUOUS);
+        GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+        GET_PYBUF_FLAG(INDIRECT);
+        GET_PYBUF_FLAG(FORMAT);
+        GET_PYBUF_FLAG(STRIDED);
+        GET_PYBUF_FLAG(STRIDED_RO);
+        GET_PYBUF_FLAG(RECORDS);
+        GET_PYBUF_FLAG(RECORDS_RO);
+        GET_PYBUF_FLAG(FULL);
+        GET_PYBUF_FLAG(FULL_RO);
+        GET_PYBUF_FLAG(CONTIG);
+        GET_PYBUF_FLAG(CONTIG_RO);
+
+        Py_DECREF(tmp);
+
+        /* One of the flags must match */
+        PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+        return NULL;
+    }
+
+    if (buffer.shape == NULL) {
+        Py_INCREF(Py_None);
+        shape = Py_None;
+    }
+    else {
+        shape = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+        }
+    }
+
+    if (buffer.strides == NULL) {
+        Py_INCREF(Py_None);
+        strides = Py_None;
+    }
+    else {
+        strides = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+        }
+    }
+
+    PyBuffer_Release(&buffer);
+    return Py_BuildValue("(NN)", shape, strides);
+}
+
 
 
 static PyMethodDef buffer_functions[] = {
     {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
+    {"get_buffer_info",   (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -27,3 +27,19 @@
         assert s == struct.pack('i', 3)
         viewlen = module.test_buffer(arr)
         assert viewlen == y.itemsize * len(y)
+
+    def test_buffer_info(self):
+        from _numpypy import multiarray as np
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        # test_export_flags from numpy test_multiarray
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        # test_relaxed_strides from numpy test_multiarray
+        arr = np.ones((1, 10))
+        if arr.flags.f_contiguous:
+            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+            assert strides[0] == 8
+            arr = np.ones((10, 1), order='F')
+            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+            assert strides[-1] == 8
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
                     STRUCT_TYPE = PySequenceMethods
+                elif slot_names[0] == 'c_tp_as_buffer':
+                    STRUCT_TYPE = PyBufferProcs
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -377,7 +377,8 @@
     def __exit__(self, typ, value, traceback):
         keepalive_until_here(self)
 
-    def get_buffer(self, space, readonly):
+    def get_buffer(self, space, flags):
+        readonly = not bool(flags & space.BUF_WRITABLE)
         return ArrayBuffer(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
@@ -695,6 +696,8 @@
                  index + self.impl.start)
 
     def setitem(self, index, v):
+        if self.readonly:
+            raise oefmt(space.w_BufferError, "cannot write to a readonly buffer")
         raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -804,20 +804,20 @@
         """)
         return w_result
 
-    def buffer_w(self, space, flags):
-        return self.implementation.get_buffer(space, True)
+    def buffer_w(self, space, w_flags):
+        return self.implementation.get_buffer(space, space.int_w(w_flags))
 
     def readbuf_w(self, space):
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO)
 
     def writebuf_w(self, space):
-        return self.implementation.get_buffer(space, False)
+        return self.implementation.get_buffer(space, space.BUF_FULL)
 
     def charbuf_w(self, space):
-        return self.implementation.get_buffer(space, True).as_str()
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
 
     def descr_get_data(self, space):
-        return space.newbuffer(self.implementation.get_buffer(space, False))
+        return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
@@ -1697,6 +1697,7 @@
     __array_wrap__ = interp2app(W_NDimArray.descr___array_wrap__),
     __array_priority__ = GetSetProperty(W_NDimArray.descr___array_priority__),
     __array__         = interp2app(W_NDimArray.descr___array__),
+    __buffer__ = interp2app(W_NDimArray.buffer_w),
 )
 
 

From pypy.commits at gmail.com  Fri Sep  2 11:27:56 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 02 Sep 2016 08:27:56 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: add all the logic needed to
 make the tests pass, fixes for translation
Message-ID: <57c99a7c.c62f1c0a.6b352.6041@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86842:04da0bbd16eb
Date: 2016-09-02 18:26 +0300
http://bitbucket.org/pypy/pypy/changeset/04da0bbd16eb/

Log:	add all the logic needed to make the tests pass, fixes for
	translation

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1428,6 +1428,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,5 +1,6 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, as_pyobj, incref
@@ -33,7 +34,8 @@
     raise an error if the object can't support a simpler view of its memory.
 
     0 is returned on success and -1 on error."""
-    buf = space.call_method(w_obj, "__buffer__", space.newint(flags))
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
     try:
         view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     except ValueError:
@@ -58,35 +60,37 @@
     return 0
 
 def _IsFortranContiguous(view):
-    if view.ndim == 0:
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
         return 1
-    if not view.strides:
-        return view.ndim == 1
-    sd = view.itemsize
-    if view.ndim == 1:
-        return view.shape[0] == 1 or sd == view.strides[0]
-    for i in range(view.ndim):
-        dim = view.shape[i]
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
         if dim == 0:
             return 1
-        if view.strides[i] != sd:
+        if view.c_strides[i] != sd:
             return 0
         sd *= dim
     return 1
 
 def _IsCContiguous(view):
-    if view.ndim == 0:
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
         return 1
-    if not view.strides:
-        return view.ndim == 1
-    sd = view.itemsize
-    if view.ndim == 1:
-        return view.shape[0] == 1 or sd == view.strides[0]
-    for i in range(view.ndim-1, -1, -1):
-        dim = view.shape[i]
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
         if dim == 0:
             return 1
-        if view.strides[i] != sd:
+        if view.c_strides[i] != sd:
             return 0
         sd *= dim
     return 1
@@ -99,7 +103,7 @@
     (fortran is 'A').  Return 0 otherwise."""
     # traverse the strides, checking for consistent stride increases from
     # right-to-left (c) or left-to-right (fortran). Copied from cpython
-    if not view.suboffsets:
+    if not view.c_suboffsets:
         return 0
     if (fort == 'C'):
         return _IsCContiguous(view)
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -35,7 +35,7 @@
         # test_export_flags from numpy test_multiarray
         raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
         # test_relaxed_strides from numpy test_multiarray
-        arr = np.ones((1, 10))
+        arr = np.zeros((1, 10))
         if arr.flags.f_contiguous:
             shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
             assert strides[0] == 8
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -378,7 +378,24 @@
         keepalive_until_here(self)
 
     def get_buffer(self, space, flags):
-        readonly = not bool(flags & space.BUF_WRITABLE)
+        errtype = space.w_ValueError # should be BufferError, numpy does this instead
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_F_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not Fortran contiguous")
+        if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                     self.flags & NPY.ARRAY_C_CONTIGUOUS)):
+           raise oefmt(errtype, "ndarray is not contiguous")
+        if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and
+            not self.flags & NPY.ARRAY_WRITEABLE):
+           raise oefmt(errtype, "buffer source array is read-only")
+        readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
         return ArrayBuffer(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
@@ -696,8 +713,7 @@
                  index + self.impl.start)
 
     def setitem(self, index, v):
-        if self.readonly:
-            raise oefmt(space.w_BufferError, "cannot write to a readonly buffer")
+        # XXX what if self.readonly?
         raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -804,8 +804,8 @@
         """)
         return w_result
 
-    def buffer_w(self, space, w_flags):
-        return self.implementation.get_buffer(space, space.int_w(w_flags))
+    def buffer_w(self, space, flags):
+        return self.implementation.get_buffer(space, flags)
 
     def readbuf_w(self, space):
         return self.implementation.get_buffer(space, space.BUF_FULL_RO)
@@ -1697,7 +1697,6 @@
     __array_wrap__ = interp2app(W_NDimArray.descr___array_wrap__),
     __array_priority__ = GetSetProperty(W_NDimArray.descr___array_priority__),
     __array__         = interp2app(W_NDimArray.descr___array__),
-    __buffer__ = interp2app(W_NDimArray.buffer_w),
 )
 
 

From pypy.commits at gmail.com  Fri Sep  2 11:38:53 2016
From: pypy.commits at gmail.com (sbauman)
Date: Fri, 02 Sep 2016 08:38:53 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Tentative solution to
 having aliased objects in virtual state when forcing
Message-ID: <57c99d0d.09afc20a.e370d.cbad@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86843:8855c1e2a325
Date: 2016-09-02 11:38 -0400
http://bitbucket.org/pypy/pypy/changeset/8855c1e2a325/

Log:	Tentative solution to having aliased objects in virtual state when
	forcing

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -325,10 +325,20 @@
                         guard.rd_resume_position = patchguardop.rd_resume_position
                         guard.setdescr(compile.ResumeAtPositionDescr())
                     self.send_extra_operation(guard)
-            except VirtualStatesCantMatch as e:
+            except VirtualStatesCantMatch:
                 continue
-            args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
-                args, self.optimizer, force_boxes=force_boxes)
+
+            # When force_boxes == True, creating the virtual args can fail when
+            # components of the virtual state alias. If this occurs, we must
+            # recompute the virtual state as boxes will have been forced.
+            try:
+                args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
+                    args, self.optimizer, force_boxes=force_boxes)
+            except VirtualStatesCantMatch:
+                assert force_boxes
+                virtual_state = self.get_virtual_state(args)
+                continue
+
             short_preamble = target_token.short_preamble
             try:
                 extra = self.inline_short_preamble(args + virtuals, args,

From pypy.commits at gmail.com  Fri Sep  2 12:09:30 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 02 Sep 2016 09:09:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: Specialize str_decode_utf_8_impl on
 allow_surrogates, to resolve a translation failure in pycket
Message-ID: <57c9a43a.898b1c0a.2cfd1.7d5d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86844:d61e115b2d73
Date: 2016-09-02 17:08 +0100
http://bitbucket.org/pypy/pypy/changeset/d61e115b2d73/

Log:	Specialize str_decode_utf_8_impl on allow_surrogates, to resolve a
	translation failure in pycket

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -145,6 +145,7 @@
 _invalid_byte_3_of_4 = _invalid_cont_byte
 _invalid_byte_4_of_4 = _invalid_cont_byte
 
+ at specialize.arg(2)
 def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xe0 and ordch2 < 0xa0)
@@ -156,7 +157,7 @@
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
- at specialize.argtype(6)
+ at specialize.arg(5)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:

From pypy.commits at gmail.com  Fri Sep  2 13:23:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Sep 2016 10:23:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix: this is needed to JIT code that
 uses ll_ullong_py_mod_zer
Message-ID: <57c9b598.05d71c0a.db689.8f5b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86845:01245dcd8f95
Date: 2016-09-02 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/01245dcd8f95/

Log:	Fix: this is needed to JIT code that uses ll_ullong_py_mod_zer

diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -540,7 +540,7 @@
     def ll_ullong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
-        return llop.ullong_mod(UnsignedLongLong, x, y)
+        return ll_ullong_py_mod(x, y)
 
 @jit.dont_look_inside
 def ll_lllong_py_mod(x, y):

From pypy.commits at gmail.com  Fri Sep  2 13:46:35 2016
From: pypy.commits at gmail.com (sbauman)
Date: Fri, 02 Sep 2016 10:46:35 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Merge with default
Message-ID: <57c9bafb.94071c0a.8c4bb.961c@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86846:a5fbf228be6d
Date: 2016-09-02 13:42 -0400
http://bitbucket.org/pypy/pypy/changeset/a5fbf228be6d/

Log:	Merge with default

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -369,3 +369,109 @@
   Roman Podoliaka
   Dan Loewenherz
   werat
+
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
+
+The PyPy Logo as used by http://speed.pypy.org and others was created
+by Samuel Reis and is distributed on terms of Creative Commons Share Alike
+License.
+ 
+License for 'lib-python/2.7'
+============================
+
+Except when otherwise stated (look for LICENSE files or copyright/license
+information at the beginning of each file) the files in the 'lib-python/2.7'
+directory are all copyrighted by the Python Software Foundation and licensed
+under the terms that you can find here: https://docs.python.org/2/license.html
+
+License for 'pypy/module/unicodedata/'
+======================================
+
+The following files are from the website of The Unicode Consortium
+at http://www.unicode.org/.  For the terms of use of these files, see
+http://www.unicode.org/terms_of_use.html .  Or they are derived from
+files from the above website, and the same terms of use apply.
+
+    CompositionExclusions-*.txt
+    EastAsianWidth-*.txt
+    LineBreak-*.txt
+    UnicodeData-*.txt
+    UnihanNumeric-*.txt
+
+License for 'dotviewer/font/'
+=============================
+
+Copyright (C) 2008 The Android Open Source Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+  
+     http://www.apache.org/licenses/LICENSE-2.0
+  
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Detailed license information is contained in the NOTICE file in the
+directory.
+
+
+Licenses and Acknowledgements for Incorporated Software
+=======================================================
+
+This section is an incomplete, but growing list of licenses and
+acknowledgements for third-party software incorporated in the PyPy
+distribution.
+
+License for 'Tcl/Tk'
+--------------------
+
+This copy of PyPy contains library code that may, when used, result in
+the Tcl/Tk library to be loaded.  PyPy also includes code that may be
+regarded as being a copy of some parts of the Tcl/Tk header files.
+You may see a copy of the License for Tcl/Tk in the file
+`lib_pypy/_tkinter/license.terms` included here.
+
+License for 'bzip2'
+-------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
+
+    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
+
+License for 'openssl'
+---------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+openssl library.  You may see a copy of the License for OpenSSL at
+
+    https://www.openssl.org/source/license.html
+
+License for 'gdbm'
+------------------
+
+The gdbm module includes code from gdbm.h, which is distributed under
+the terms of the GPL license version 2 or any later version.  Thus the
+gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
+under the terms of the GPL license as well.
+
+License for 'rpython/rlib/rvmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://github.com/gperftools/gperftools/blob/master/COPYING
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -164,8 +164,15 @@
             # annotations that are passed in, and don't annotate the old
             # graph -- it's already low-level operations!
             for a, s_newarg in zip(block.inputargs, cells):
-                s_oldarg = self.binding(a)
-                assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
+                s_oldarg = a.annotation
+                # XXX: Should use s_oldarg.contains(s_newarg) but that breaks
+                # PyPy translation
+                if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg:
+                    raise annmodel.AnnotatorError(
+                        "Late-stage annotation is not allowed to modify the "
+                        "existing annotation for variable %s: %s" %
+                            (a, s_oldarg))
+
         else:
             assert not self.frozen
             if block not in self.annotated:
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -750,6 +750,7 @@
                 s1 = pair(s1, s2).union()
     else:
         # this is just a performance shortcut
+        # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
         if s1 != s2:
             s1 = pair(s1, s2).union()
     return s1
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -145,6 +145,7 @@
 _invalid_byte_3_of_4 = _invalid_cont_byte
 _invalid_byte_4_of_4 = _invalid_cont_byte
 
+ at specialize.arg(2)
 def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xe0 and ordch2 < 0xa0)
@@ -156,7 +157,7 @@
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
- at specialize.argtype(6)
+ at specialize.arg(5)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:

From pypy.commits at gmail.com  Fri Sep  2 14:36:31 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 02 Sep 2016 11:36:31 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Create union() as a
 side-effect-free binary function to replace unionof() wherever possible
Message-ID: <57c9c6af.d42f1c0a.665d8.aa80@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86847:033077287c06
Date: 2016-09-02 19:35 +0100
http://bitbucket.org/pypy/pypy/changeset/033077287c06/

Log:	Create union() as a side-effect-free binary function to replace
	unionof() wherever possible

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -94,16 +94,9 @@
         if self == other:
             return True
         try:
-            TLS.no_side_effects_in_union += 1
-        except AttributeError:
-            TLS.no_side_effects_in_union = 1
-        try:
-            try:
-                return pair(self, other).union() == self
-            except UnionError:
-                return False
-        finally:
-            TLS.no_side_effects_in_union -= 1
+            return union(self, other) == self
+        except UnionError:
+            return False
 
     def is_constant(self):
         d = self.__dict__
@@ -739,6 +732,23 @@
     def __repr__(self):
         return str(self)
 
+def union(s1, s2):
+    """The join operation in the lattice of annotations.
+
+    It is the most precise SomeObject instance that contains both arguments.
+
+    union() is (supposed to be) idempotent, commutative, associative and has
+    no side-effects.
+    """
+    try:
+        TLS.no_side_effects_in_union += 1
+    except AttributeError:
+        TLS.no_side_effects_in_union = 1
+    try:
+        return pair(s1, s2).union()
+    finally:
+        TLS.no_side_effects_in_union -= 1
+
 def unionof(*somevalues):
     "The most precise SomeValue instance that contains all the values."
     try:

From pypy.commits at gmail.com  Fri Sep  2 15:22:11 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 02 Sep 2016 12:22:11 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Fix some broken testcases due to
 wrong typeids for immortal objects
Message-ID: <57c9d163.04141c0a.b87ba.b674@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86848:a6ec7b4fb726
Date: 2016-09-02 21:21 +0200
http://bitbucket.org/pypy/pypy/changeset/a6ec7b4fb726/

Log:	Fix some broken testcases due to wrong typeids for immortal objects

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -26,11 +26,10 @@
             ('hdr', rffi.COpaque('object_t', hints={"is_qcgc_header": True})),
             ('tid', lltype.Signed),
             ('hash', lltype.Signed))
-    #HDR = rffi.COpaque('object_t')
 
     def init_gc_object(self, addr, typeid):
         hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.tid = rffi.cast(lltype.Signed, typeid)
+        hdr.tid = llop.combine_ushort(lltype.Signed, typeid, 0)
         hdr.hash = rffi.cast(lltype.Signed, 0)
 
     def malloc_fixedsize_clear(self, typeid, size,
@@ -63,7 +62,7 @@
     def init_gc_object_immortal(self, addr, typeid, flags=0):
         assert flags == 0
         #
-        self.init_gc_object(addr, typeid.index)
+        self.init_gc_object(addr, typeid)
 
     def collect(self, gen=1):
         """Do a minor (gen=0) or major (gen>0) collection."""

From pypy.commits at gmail.com  Fri Sep  2 16:44:02 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 02 Sep 2016 13:44:02 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Use union() instead of
 unionof() in a few places
Message-ID: <57c9e492.05371c0a.1ada0.d1e2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86849:53937853a5d2
Date: 2016-09-02 21:40 +0100
http://bitbucket.org/pypy/pypy/changeset/53937853a5d2/

Log:	Use union() instead of unionof() in a few places

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -246,7 +246,7 @@
         if s_old is not None:
             if not s_value.contains(s_old):
                 log.WARNING("%s does not contain %s" % (s_value, s_old))
-                log.WARNING("%s" % annmodel.unionof(s_value, s_old))
+                log.WARNING("%s" % annmodel.union(s_value, s_old))
                 assert False
         arg.annotation = s_value
 
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -11,7 +11,7 @@
     SomeBuiltinMethod, SomeIterator, SomePBC, SomeNone, SomeFloat, s_None,
     SomeByteArray, SomeWeakRef, SomeSingleFloat,
     SomeLongFloat, SomeType, SomeTypeOf, SomeConstantType, unionof, UnionError,
-    read_can_only_throw, add_knowntypedata,
+    union, read_can_only_throw, add_knowntypedata,
     merge_knowntypedata,)
 from rpython.annotator.bookkeeper import immutablevalue, getbookkeeper
 from rpython.flowspace.model import Variable, Constant, const
@@ -703,13 +703,13 @@
         pairtype(SomeException, SomeInstance),
         pairtype(SomeException, SomeNone)):
     def union((s_exc, s_inst)):
-        return unionof(s_exc.as_SomeInstance(), s_inst)
+        return union(s_exc.as_SomeInstance(), s_inst)
 
 class __extend__(
         pairtype(SomeInstance, SomeException),
         pairtype(SomeNone, SomeException)):
     def union((s_inst, s_exc)):
-        return unionof(s_exc.as_SomeInstance(), s_inst)
+        return union(s_exc.as_SomeInstance(), s_inst)
 
 class __extend__(pairtype(SomeException, SomeException)):
     def union((s_exc1, s_exc2)):
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -6,7 +6,7 @@
 
 from rpython.annotator.model import (
     SomeInteger, SomeChar, SomeBool, SomeString, SomeTuple,
-    SomeUnicodeCodePoint, SomeFloat, unionof, SomeUnicodeString,
+    SomeUnicodeCodePoint, SomeFloat, union, SomeUnicodeString,
     SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator,
     SomeOrderedDict, SomeByteArray, add_knowntypedata, s_ImpossibleValue,)
 from rpython.annotator.bookkeeper import (
@@ -166,14 +166,14 @@
         s_iter = s_values[0].iter()
         return s_iter.next()
     else:
-        return unionof(*s_values)
+        return union(*s_values)
 
 def builtin_max(*s_values):
     if len(s_values) == 1: # xxx do we support this?
         s_iter = s_values[0].iter()
         return s_iter.next()
     else:
-        s = unionof(*s_values)
+        s = union(*s_values)
         if type(s) is SomeInteger and not s.nonneg:
             nonneg = False
             for s1 in s_values:
diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -1,5 +1,5 @@
-from rpython.annotator.model import s_ImpossibleValue
-from rpython.annotator.model import SomeInteger, s_Bool, unionof
+from rpython.annotator.model import (
+    s_ImpossibleValue, SomeInteger, s_Bool, union)
 from rpython.annotator.listdef import ListItem
 from rpython.rlib.objectmodel import compute_hash
 
@@ -34,8 +34,8 @@
 
     def update_rdict_annotations(self, s_eqfn, s_hashfn, other=None):
         assert self.custom_eq_hash
-        s_eqfn = unionof(s_eqfn, self.s_rdict_eqfn)
-        s_hashfn = unionof(s_hashfn, self.s_rdict_hashfn)
+        s_eqfn = union(s_eqfn, self.s_rdict_eqfn)
+        s_hashfn = union(s_hashfn, self.s_rdict_hashfn)
         self.s_rdict_eqfn = s_eqfn
         self.s_rdict_hashfn = s_hashfn
         self.emulate_rdict_calls(other=other)
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -146,14 +146,14 @@
     someinst = lambda cls, **kw: SomeInstance(bk.getuniqueclassdef(cls), **kw)
     s_inst = someinst(Exception)
     s_exc = bk.new_exception([ValueError, IndexError])
-    assert unionof(s_exc, s_inst) == s_inst
-    assert unionof(s_inst, s_exc) == s_inst
-    s_nullable = unionof(s_None, bk.new_exception([ValueError]))
+    assert union(s_exc, s_inst) == s_inst
+    assert union(s_inst, s_exc) == s_inst
+    s_nullable = union(s_None, bk.new_exception([ValueError]))
     assert isinstance(s_nullable, SomeInstance)
     assert s_nullable.can_be_None
     s_exc1 = bk.new_exception([ValueError])
     s_exc2 = bk.new_exception([IndexError])
-    unionof(s_exc1, s_exc2) == unionof(s_exc2, s_exc1)
+    union(s_exc1, s_exc2) == union(s_exc2, s_exc1)
 
 def contains_s(s_a, s_b):
     if s_b is None:
diff --git a/rpython/rtyper/test/test_llannotation.py b/rpython/rtyper/test/test_llannotation.py
--- a/rpython/rtyper/test/test_llannotation.py
+++ b/rpython/rtyper/test/test_llannotation.py
@@ -1,6 +1,6 @@
 import py.test
 from rpython.annotator.model import (
-    SomeInteger, SomeBool, SomeChar, unionof, SomeImpossibleValue,
+    SomeInteger, SomeBool, SomeChar, union, SomeImpossibleValue,
     UnionError, SomeInstance, SomeSingleFloat)
 from rpython.rlib.rarithmetic import r_uint, r_singlefloat
 from rpython.rtyper.llannotation import (
@@ -69,22 +69,22 @@
     PA1 = lltype.Ptr(lltype.GcArray())
     PA2 = lltype.Ptr(lltype.GcArray())
 
-    assert unionof(SomePtr(PS1), SomePtr(PS1)) == SomePtr(PS1)
-    assert unionof(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS2)
-    assert unionof(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomePtr(PS1)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS2)
+    assert union(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS1)
 
-    assert unionof(SomePtr(PA1), SomePtr(PA1)) == SomePtr(PA1)
-    assert unionof(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA2)
-    assert unionof(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA1)
+    assert union(SomePtr(PA1), SomePtr(PA1)) == SomePtr(PA1)
+    assert union(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA2)
+    assert union(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA1)
 
-    assert unionof(SomePtr(PS1), SomeImpossibleValue()) == SomePtr(PS1)
-    assert unionof(SomeImpossibleValue(), SomePtr(PS1)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomeImpossibleValue()) == SomePtr(PS1)
+    assert union(SomeImpossibleValue(), SomePtr(PS1)) == SomePtr(PS1)
 
     with py.test.raises(UnionError):
-        unionof(SomePtr(PA1), SomePtr(PS1))
+        union(SomePtr(PA1), SomePtr(PS1))
     with py.test.raises(UnionError):
-        unionof(SomePtr(PS1), SomePtr(PS3))
+        union(SomePtr(PS1), SomePtr(PS3))
     with py.test.raises(UnionError):
-        unionof(SomePtr(PS1), SomeInteger())
+        union(SomePtr(PS1), SomeInteger())
     with py.test.raises(UnionError):
-        unionof(SomeInteger(), SomePtr(PS1))
+        union(SomeInteger(), SomePtr(PS1))

From pypy.commits at gmail.com  Fri Sep  2 22:42:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 02 Sep 2016 19:42:45 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Fix union() and give a
 correct explanation for the s1==s2 case
Message-ID: <57ca38a5.eeb8c20a.69f14.86e7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86851:81eb595adb63
Date: 2016-09-03 01:44 +0100
http://bitbucket.org/pypy/pypy/changeset/81eb595adb63/

Log:	Fix union() and give a correct explanation for the s1==s2 case

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -745,6 +745,10 @@
     except AttributeError:
         TLS.no_side_effects_in_union = 1
     try:
+        if s1 == s2:
+            # Most pair(...).union() methods deal incorrectly with that case
+            # when constants are involved.
+            return s1
         return pair(s1, s2).union()
     finally:
         TLS.no_side_effects_in_union -= 1
@@ -759,8 +763,7 @@
             if s1 != s2:
                 s1 = pair(s1, s2).union()
     else:
-        # this is just a performance shortcut
-        # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
+        # See comment in union() above
         if s1 != s2:
             s1 = pair(s1, s2).union()
     return s1

From pypy.commits at gmail.com  Fri Sep  2 22:42:47 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 02 Sep 2016 19:42:47 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Expand hypothesis testing of
 union() until it fails
Message-ID: <57ca38a7.4676c20a.2e4e.b0e6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86852:fc2b86d220a0
Date: 2016-09-03 03:41 +0100
http://bitbucket.org/pypy/pypy/changeset/fc2b86d220a0/

Log:	Expand hypothesis testing of union() until it fails

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -673,6 +673,10 @@
 
 s_None = SomeNone()
 s_Bool = SomeBool()
+s_True = SomeBool()
+s_True.const = True
+s_False = SomeBool()
+s_False.const = False
 s_Int = SomeInteger()
 s_ImpossibleValue = SomeImpossibleValue()
 s_Str0 = SomeString(no_nul=True)
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -70,11 +70,11 @@
     except TypeError:    # if A0 is also a new-style class, e.g. in PyPy
         class B3(A0, object):
             pass
-    assert commonbase(A1,A2) is A0
-    assert commonbase(A1,A0) is A0
-    assert commonbase(A1,A1) is A1
-    assert commonbase(A2,B2) is object
-    assert commonbase(A2,B3) is A0
+    assert commonbase(A1, A2) is A0
+    assert commonbase(A1, A0) is A0
+    assert commonbase(A1, A1) is A1
+    assert commonbase(A2, B2) is object
+    assert commonbase(A2, B3) is A0
 
 def test_list_union():
     listdef1 = ListDef('dummy', SomeInteger(nonneg=True))
@@ -105,20 +105,89 @@
     assert f2.contains(f1)
     assert f1.contains(f2)
 
+def const_float(x):
+    s = SomeFloat()
+    s.const = x
+    return s
+
 def const_int(n):
-    s = SomeInteger(nonneg=(n>=0))
+    s = SomeInteger(nonneg=(n >= 0))
     s.const = n
     return s
 
+def const_str(x):
+    no_nul = not '\x00' in x
+    if len(x) == 1:
+        result = SomeChar(no_nul=no_nul)
+    else:
+        result = SomeString(no_nul=no_nul)
+    result.const = x
+    return result
+
+def const_unicode(x):
+    no_nul = not u'\x00' in x
+    if len(x) == 1:
+        result = SomeUnicodeCodePoint(no_nul=no_nul)
+    else:
+        result = SomeUnicodeString(no_nul=no_nul)
+    result.const = x
+    return result
+
+def compatible(s1, s2):
+    try:
+        union(s1, s2)
+    except UnionError:
+        return False
+    return True
+
+def compatible_pair(pair_s):
+    return compatible(*pair_s)
+
+st_float = st.just(SomeFloat()) | st.builds(const_float, st.floats())
 st_int = st.one_of(st.builds(SomeInteger, st.booleans(), st.booleans()),
                    st.builds(const_int, st.integers()))
-st_annotation = st_int
+st_bool = st.sampled_from([s_Bool, s_True, s_False])
+st_numeric = st.one_of(st_float, st_int, st_bool)
+st_str = (st.builds(SomeString, st.booleans(), st.booleans())
+          | st.builds(const_str, st.binary()))
+st_unicode = (st.builds(SomeUnicodeString, st.booleans(), st.booleans())
+              | st.builds(const_unicode, st.text()))
+st_simple = st.one_of(st_numeric, st_str, st_unicode, st.just(s_ImpossibleValue), st.just(s_None))
 
- at given(s=st_annotation)
+def valid_unions(st_ann):
+    """From a strategy generating annotations, create a strategy returning
+    unions of these annotations."""
+    pairs = st.tuples(st_ann, st_ann)
+    return pairs.filter(compatible_pair).map(lambda t: union(*t))
+
+
+st_annotation = st.recursive(st_simple,
+    lambda st_ann: valid_unions(st_ann) | st.builds(SomeTuple, st.lists(st_ann)),
+    max_leaves=3)
+
+ at given(s=st_numeric)
 def test_union_unary(s):
     assert union(s, s) == s
     assert union(s_ImpossibleValue, s) == s
 
+ at given(s1=st_numeric, s2=st_numeric)
+def test_commutativity_of_union_compatibility(s1, s2):
+    assert compatible(s1, s2) == compatible(s2, s1)
+
+ at given(st.tuples(st_annotation, st_annotation).filter(lambda t: compatible(*t)))
+def test_union_commutative(t):
+    s1, s2 = t
+    s_union = union(s1, s2)
+    assert union(s2, s1) == s_union
+    assert s_union.contains(s1)
+    assert s_union.contains(s2)
+
+ at given(st.tuples(st_annotation, st_annotation, st_annotation).filter(
+    lambda t: compatible(t[0], t[1]) and compatible(t[1], t[2]) and compatible(t[0], t[2])))
+def test_union_associative(t):
+    s1, s2, s3 = t
+    assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
+
 
 def compile_function(function, annotation=[]):
     t = TranslationContext()

From pypy.commits at gmail.com  Fri Sep  2 22:42:44 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 02 Sep 2016 19:42:44 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Add an elementary (but
 failing) hypothesis test for union()
Message-ID: <57ca38a4.c3d41c0a.2923b.249f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86850:34fcf0746426
Date: 2016-09-03 01:24 +0100
http://bitbucket.org/pypy/pypy/changeset/34fcf0746426/

Log:	Add an elementary (but failing) hypothesis test for union()

diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -1,5 +1,8 @@
 import pytest
 
+from hypothesis import given
+from hypothesis import strategies as st
+
 from rpython.flowspace.model import Variable
 from rpython.flowspace.operation import op
 from rpython.translator.translator import TranslationContext
@@ -102,6 +105,21 @@
     assert f2.contains(f1)
     assert f1.contains(f2)
 
+def const_int(n):
+    s = SomeInteger(nonneg=(n>=0))
+    s.const = n
+    return s
+
+st_int = st.one_of(st.builds(SomeInteger, st.booleans(), st.booleans()),
+                   st.builds(const_int, st.integers()))
+st_annotation = st_int
+
+ at given(s=st_annotation)
+def test_union_unary(s):
+    assert union(s, s) == s
+    assert union(s_ImpossibleValue, s) == s
+
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)

From pypy.commits at gmail.com  Sat Sep  3 05:11:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 02:11:55 -0700 (PDT)
Subject: [pypy-commit] cffi default: Kill the ctypes backend.
Message-ID: <57ca93db.8aacc20a.9c360.e831@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2749:0087e2aec9ef
Date: 2016-09-03 11:11 +0200
http://bitbucket.org/cffi/cffi/changeset/0087e2aec9ef/

Log:	Kill the ctypes backend.

diff too long, truncating to 2000 out of 2925 lines

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -46,20 +46,21 @@
     '''
 
     def __init__(self, backend=None):
-        """Create an FFI instance.  The 'backend' argument is used to
-        select a non-default backend, mostly for tests.
+        """Create an FFI instance.
+
+        The 'backend' argument is not used any more and must be set to None.
+        It is still present only so that 'FFI(None)' still works, and
+        for a few tests.
         """
         from . import cparser, model
+
         if backend is None:
-            # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with
-            # _cffi_backend.so compiled.
+            # You need the corresponding version of PyPy, or CPython
+            # with the '_cffi_backend' C extension module compiled.
             import _cffi_backend as backend
             from . import __version__
             assert backend.__version__ == __version__, \
                "version mismatch, %s != %s" % (backend.__version__, __version__)
-            # (If you insist you can also try to pass the option
-            # 'backend=backend_ctypes.CTypesBackend()', but don't
-            # rely on it!  It's probably not going to work well.)
 
         self._backend = backend
         self._lock = allocate_lock()
@@ -75,8 +76,6 @@
         self._init_once_cache = {}
         self._cdef_version = None
         self._embedding = None
-        if hasattr(backend, 'set_ffi'):
-            backend.set_ffi(self)
         for name in backend.__dict__:
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
@@ -84,15 +83,10 @@
         with self._lock:
             self.BVoidP = self._get_cached_btype(model.voidp_type)
             self.BCharA = self._get_cached_btype(model.char_array_type)
-        if isinstance(backend, types.ModuleType):
-            # _cffi_backend: attach these constants to the class
-            if not hasattr(FFI, 'NULL'):
-                FFI.NULL = self.cast(self.BVoidP, 0)
-                FFI.CData, FFI.CType = backend._get_types()
-        else:
-            # ctypes backend: attach these constants to the instance
-            self.NULL = self.cast(self.BVoidP, 0)
-            self.CData, self.CType = backend._get_types()
+        # attach these constants to the class
+        if not hasattr(FFI, 'NULL'):
+            FFI.NULL = self.cast(self.BVoidP, 0)
+            FFI.CData, FFI.CType = backend._get_types()
 
     def cdef(self, csource, override=False, packed=False):
         """Parse the given C source.  This registers all declared functions,
diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
deleted file mode 100644
--- a/cffi/backend_ctypes.py
+++ /dev/null
@@ -1,1097 +0,0 @@
-import ctypes, ctypes.util, operator, sys
-from . import model
-
-if sys.version_info < (3,):
-    bytechr = chr
-else:
-    unicode = str
-    long = int
-    xrange = range
-    bytechr = lambda num: bytes([num])
-
-class CTypesType(type):
-    pass
-
-class CTypesData(object):
-    __metaclass__ = CTypesType
-    __slots__ = ['__weakref__']
-    __name__ = '<cdata>'
-
-    def __init__(self, *args):
-        raise TypeError("cannot instantiate %r" % (self.__class__,))
-
-    @classmethod
-    def _newp(cls, init):
-        raise TypeError("expected a pointer or array ctype, got '%s'"
-                        % (cls._get_c_name(),))
-
-    @staticmethod
-    def _to_ctypes(value):
-        raise TypeError
-
-    @classmethod
-    def _arg_to_ctypes(cls, *value):
-        try:
-            ctype = cls._ctype
-        except AttributeError:
-            raise TypeError("cannot create an instance of %r" % (cls,))
-        if value:
-            res = cls._to_ctypes(*value)
-            if not isinstance(res, ctype):
-                res = cls._ctype(res)
-        else:
-            res = cls._ctype()
-        return res
-
-    @classmethod
-    def _create_ctype_obj(cls, init):
-        if init is None:
-            return cls._arg_to_ctypes()
-        else:
-            return cls._arg_to_ctypes(init)
-
-    @staticmethod
-    def _from_ctypes(ctypes_value):
-        raise TypeError
-
-    @classmethod
-    def _get_c_name(cls, replace_with=''):
-        return cls._reftypename.replace(' &', replace_with)
-
-    @classmethod
-    def _fix_class(cls):
-        cls.__name__ = 'CData<%s>' % (cls._get_c_name(),)
-        cls.__qualname__ = 'CData<%s>' % (cls._get_c_name(),)
-        cls.__module__ = 'ffi'
-
-    def _get_own_repr(self):
-        raise NotImplementedError
-
-    def _addr_repr(self, address):
-        if address == 0:
-            return 'NULL'
-        else:
-            if address < 0:
-                address += 1 << (8*ctypes.sizeof(ctypes.c_void_p))
-            return '0x%x' % address
-
-    def __repr__(self, c_name=None):
-        own = self._get_own_repr()
-        return '<cdata %r %s>' % (c_name or self._get_c_name(), own)
-
-    def _convert_to_address(self, BClass):
-        if BClass is None:
-            raise TypeError("cannot convert %r to an address" % (
-                self._get_c_name(),))
-        else:
-            raise TypeError("cannot convert %r to %r" % (
-                self._get_c_name(), BClass._get_c_name()))
-
-    @classmethod
-    def _get_size(cls):
-        return ctypes.sizeof(cls._ctype)
-
-    def _get_size_of_instance(self):
-        return ctypes.sizeof(self._ctype)
-
-    @classmethod
-    def _cast_from(cls, source):
-        raise TypeError("cannot cast to %r" % (cls._get_c_name(),))
-
-    def _cast_to_integer(self):
-        return self._convert_to_address(None)
-
-    @classmethod
-    def _alignment(cls):
-        return ctypes.alignment(cls._ctype)
-
-    def __iter__(self):
-        raise TypeError("cdata %r does not support iteration" % (
-            self._get_c_name()),)
-
-    def _make_cmp(name):
-        cmpfunc = getattr(operator, name)
-        def cmp(self, other):
-            if isinstance(other, CTypesData):
-                return cmpfunc(self._convert_to_address(None),
-                               other._convert_to_address(None))
-            else:
-                return NotImplemented
-        cmp.func_name = name
-        return cmp
-
-    __eq__ = _make_cmp('__eq__')
-    __ne__ = _make_cmp('__ne__')
-    __lt__ = _make_cmp('__lt__')
-    __le__ = _make_cmp('__le__')
-    __gt__ = _make_cmp('__gt__')
-    __ge__ = _make_cmp('__ge__')
-
-    def __hash__(self):
-        return hash(type(self)) ^ hash(self._convert_to_address(None))
-
-    def _to_string(self, maxlen):
-        raise TypeError("string(): %r" % (self,))
-
-
-class CTypesGenericPrimitive(CTypesData):
-    __slots__ = []
-
-    def __eq__(self, other):
-        return self is other
-
-    def __ne__(self, other):
-        return self is not other
-
-    def __hash__(self):
-        return object.__hash__(self)
-
-    def _get_own_repr(self):
-        return repr(self._from_ctypes(self._value))
-
-
-class CTypesGenericArray(CTypesData):
-    __slots__ = []
-
-    @classmethod
-    def _newp(cls, init):
-        return cls(init)
-
-    def __iter__(self):
-        for i in xrange(len(self)):
-            yield self[i]
-
-    def _get_own_repr(self):
-        return self._addr_repr(ctypes.addressof(self._blob))
-
-
-class CTypesGenericPtr(CTypesData):
-    __slots__ = ['_address', '_as_ctype_ptr']
-    _automatic_casts = False
-    kind = "pointer"
-
-    @classmethod
-    def _newp(cls, init):
-        return cls(init)
-
-    @classmethod
-    def _cast_from(cls, source):
-        if source is None:
-            address = 0
-        elif isinstance(source, CTypesData):
-            address = source._cast_to_integer()
-        elif isinstance(source, (int, long)):
-            address = source
-        else:
-            raise TypeError("bad type for cast to %r: %r" %
-                            (cls, type(source).__name__))
-        return cls._new_pointer_at(address)
-
-    @classmethod
-    def _new_pointer_at(cls, address):
-        self = cls.__new__(cls)
-        self._address = address
-        self._as_ctype_ptr = ctypes.cast(address, cls._ctype)
-        return self
-
-    def _get_own_repr(self):
-        try:
-            return self._addr_repr(self._address)
-        except AttributeError:
-            return '???'
-
-    def _cast_to_integer(self):
-        return self._address
-
-    def __nonzero__(self):
-        return bool(self._address)
-    __bool__ = __nonzero__
-
-    @classmethod
-    def _to_ctypes(cls, value):
-        if not isinstance(value, CTypesData):
-            raise TypeError("unexpected %s object" % type(value).__name__)
-        address = value._convert_to_address(cls)
-        return ctypes.cast(address, cls._ctype)
-
-    @classmethod
-    def _from_ctypes(cls, ctypes_ptr):
-        address = ctypes.cast(ctypes_ptr, ctypes.c_void_p).value or 0
-        return cls._new_pointer_at(address)
-
-    @classmethod
-    def _initialize(cls, ctypes_ptr, value):
-        if value:
-            ctypes_ptr.contents = cls._to_ctypes(value).contents
-
-    def _convert_to_address(self, BClass):
-        if (BClass in (self.__class__, None) or BClass._automatic_casts
-            or self._automatic_casts):
-            return self._address
-        else:
-            return CTypesData._convert_to_address(self, BClass)
-
-
-class CTypesBaseStructOrUnion(CTypesData):
-    __slots__ = ['_blob']
-
-    @classmethod
-    def _create_ctype_obj(cls, init):
-        # may be overridden
-        raise TypeError("cannot instantiate opaque type %s" % (cls,))
-
-    def _get_own_repr(self):
-        return self._addr_repr(ctypes.addressof(self._blob))
-
-    @classmethod
-    def _offsetof(cls, fieldname):
-        return getattr(cls._ctype, fieldname).offset
-
-    def _convert_to_address(self, BClass):
-        if getattr(BClass, '_BItem', None) is self.__class__:
-            return ctypes.addressof(self._blob)
-        else:
-            return CTypesData._convert_to_address(self, BClass)
-
-    @classmethod
-    def _from_ctypes(cls, ctypes_struct_or_union):
-        self = cls.__new__(cls)
-        self._blob = ctypes_struct_or_union
-        return self
-
-    @classmethod
-    def _to_ctypes(cls, value):
-        return value._blob
-
-    def __repr__(self, c_name=None):
-        return CTypesData.__repr__(self, c_name or self._get_c_name(' &'))
-
-
-class CTypesBackend(object):
-
-    PRIMITIVE_TYPES = {
-        'char': ctypes.c_char,
-        'short': ctypes.c_short,
-        'int': ctypes.c_int,
-        'long': ctypes.c_long,
-        'long long': ctypes.c_longlong,
-        'signed char': ctypes.c_byte,
-        'unsigned char': ctypes.c_ubyte,
-        'unsigned short': ctypes.c_ushort,
-        'unsigned int': ctypes.c_uint,
-        'unsigned long': ctypes.c_ulong,
-        'unsigned long long': ctypes.c_ulonglong,
-        'float': ctypes.c_float,
-        'double': ctypes.c_double,
-        '_Bool': ctypes.c_bool,
-        }
-
-    for _name in ['unsigned long long', 'unsigned long',
-                  'unsigned int', 'unsigned short', 'unsigned char']:
-        _size = ctypes.sizeof(PRIMITIVE_TYPES[_name])
-        PRIMITIVE_TYPES['uint%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name]
-        if _size == ctypes.sizeof(ctypes.c_void_p):
-            PRIMITIVE_TYPES['uintptr_t'] = PRIMITIVE_TYPES[_name]
-        if _size == ctypes.sizeof(ctypes.c_size_t):
-            PRIMITIVE_TYPES['size_t'] = PRIMITIVE_TYPES[_name]
-
-    for _name in ['long long', 'long', 'int', 'short', 'signed char']:
-        _size = ctypes.sizeof(PRIMITIVE_TYPES[_name])
-        PRIMITIVE_TYPES['int%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name]
-        if _size == ctypes.sizeof(ctypes.c_void_p):
-            PRIMITIVE_TYPES['intptr_t'] = PRIMITIVE_TYPES[_name]
-            PRIMITIVE_TYPES['ptrdiff_t'] = PRIMITIVE_TYPES[_name]
-        if _size == ctypes.sizeof(ctypes.c_size_t):
-            PRIMITIVE_TYPES['ssize_t'] = PRIMITIVE_TYPES[_name]
-
-
-    def __init__(self):
-        self.RTLD_LAZY = 0   # not supported anyway by ctypes
-        self.RTLD_NOW  = 0
-        self.RTLD_GLOBAL = ctypes.RTLD_GLOBAL
-        self.RTLD_LOCAL = ctypes.RTLD_LOCAL
-
-    def set_ffi(self, ffi):
-        self.ffi = ffi
-
-    def _get_types(self):
-        return CTypesData, CTypesType
-
-    def load_library(self, path, flags=0):
-        cdll = ctypes.CDLL(path, flags)
-        return CTypesLibrary(self, cdll)
-
-    def new_void_type(self):
-        class CTypesVoid(CTypesData):
-            __slots__ = []
-            _reftypename = 'void &'
-            @staticmethod
-            def _from_ctypes(novalue):
-                return None
-            @staticmethod
-            def _to_ctypes(novalue):
-                if novalue is not None:
-                    raise TypeError("None expected, got %s object" %
-                                    (type(novalue).__name__,))
-                return None
-        CTypesVoid._fix_class()
-        return CTypesVoid
-
-    def new_primitive_type(self, name):
-        if name == 'wchar_t':
-            raise NotImplementedError(name)
-        ctype = self.PRIMITIVE_TYPES[name]
-        if name == 'char':
-            kind = 'char'
-        elif name in ('float', 'double'):
-            kind = 'float'
-        else:
-            if name in ('signed char', 'unsigned char'):
-                kind = 'byte'
-            elif name == '_Bool':
-                kind = 'bool'
-            else:
-                kind = 'int'
-            is_signed = (ctype(-1).value == -1)
-        #
-        def _cast_source_to_int(source):
-            if isinstance(source, (int, long, float)):
-                source = int(source)
-            elif isinstance(source, CTypesData):
-                source = source._cast_to_integer()
-            elif isinstance(source, bytes):
-                source = ord(source)
-            elif source is None:
-                source = 0
-            else:
-                raise TypeError("bad type for cast to %r: %r" %
-                                (CTypesPrimitive, type(source).__name__))
-            return source
-        #
-        kind1 = kind
-        class CTypesPrimitive(CTypesGenericPrimitive):
-            __slots__ = ['_value']
-            _ctype = ctype
-            _reftypename = '%s &' % name
-            kind = kind1
-
-            def __init__(self, value):
-                self._value = value
-
-            @staticmethod
-            def _create_ctype_obj(init):
-                if init is None:
-                    return ctype()
-                return ctype(CTypesPrimitive._to_ctypes(init))
-
-            if kind == 'int' or kind == 'byte':
-                @classmethod
-                def _cast_from(cls, source):
-                    source = _cast_source_to_int(source)
-                    source = ctype(source).value     # cast within range
-                    return cls(source)
-                def __int__(self):
-                    return self._value
-
-            if kind == 'bool':
-                @classmethod
-                def _cast_from(cls, source):
-                    if not isinstance(source, (int, long, float)):
-                        source = _cast_source_to_int(source)
-                    return cls(bool(source))
-                def __int__(self):
-                    return self._value
-
-            if kind == 'char':
-                @classmethod
-                def _cast_from(cls, source):
-                    source = _cast_source_to_int(source)
-                    source = bytechr(source & 0xFF)
-                    return cls(source)
-                def __int__(self):
-                    return ord(self._value)
-
-            if kind == 'float':
-                @classmethod
-                def _cast_from(cls, source):
-                    if isinstance(source, float):
-                        pass
-                    elif isinstance(source, CTypesGenericPrimitive):
-                        if hasattr(source, '__float__'):
-                            source = float(source)
-                        else:
-                            source = int(source)
-                    else:
-                        source = _cast_source_to_int(source)
-                    source = ctype(source).value     # fix precision
-                    return cls(source)
-                def __int__(self):
-                    return int(self._value)
-                def __float__(self):
-                    return self._value
-
-            _cast_to_integer = __int__
-
-            if kind == 'int' or kind == 'byte' or kind == 'bool':
-                @staticmethod
-                def _to_ctypes(x):
-                    if not isinstance(x, (int, long)):
-                        if isinstance(x, CTypesData):
-                            x = int(x)
-                        else:
-                            raise TypeError("integer expected, got %s" %
-                                            type(x).__name__)
-                    if ctype(x).value != x:
-                        if not is_signed and x < 0:
-                            raise OverflowError("%s: negative integer" % name)
-                        else:
-                            raise OverflowError("%s: integer out of bounds"
-                                                % name)
-                    return x
-
-            if kind == 'char':
-                @staticmethod
-                def _to_ctypes(x):
-                    if isinstance(x, bytes) and len(x) == 1:
-                        return x
-                    if isinstance(x, CTypesPrimitive):    # <CData <char>>
-                        return x._value
-                    raise TypeError("character expected, got %s" %
-                                    type(x).__name__)
-                def __nonzero__(self):
-                    return ord(self._value) != 0
-            else:
-                def __nonzero__(self):
-                    return self._value != 0
-            __bool__ = __nonzero__
-
-            if kind == 'float':
-                @staticmethod
-                def _to_ctypes(x):
-                    if not isinstance(x, (int, long, float, CTypesData)):
-                        raise TypeError("float expected, got %s" %
-                                        type(x).__name__)
-                    return ctype(x).value
-
-            @staticmethod
-            def _from_ctypes(value):
-                return getattr(value, 'value', value)
-
-            @staticmethod
-            def _initialize(blob, init):
-                blob.value = CTypesPrimitive._to_ctypes(init)
-
-            if kind == 'char':
-                def _to_string(self, maxlen):
-                    return self._value
-            if kind == 'byte':
-                def _to_string(self, maxlen):
-                    return chr(self._value & 0xff)
-        #
-        CTypesPrimitive._fix_class()
-        return CTypesPrimitive
-
-    def new_pointer_type(self, BItem):
-        getbtype = self.ffi._get_cached_btype
-        if BItem is getbtype(model.PrimitiveType('char')):
-            kind = 'charp'
-        elif BItem in (getbtype(model.PrimitiveType('signed char')),
-                       getbtype(model.PrimitiveType('unsigned char'))):
-            kind = 'bytep'
-        elif BItem is getbtype(model.void_type):
-            kind = 'voidp'
-        else:
-            kind = 'generic'
-        #
-        class CTypesPtr(CTypesGenericPtr):
-            __slots__ = ['_own']
-            if kind == 'charp':
-                __slots__ += ['__as_strbuf']
-            _BItem = BItem
-            if hasattr(BItem, '_ctype'):
-                _ctype = ctypes.POINTER(BItem._ctype)
-                _bitem_size = ctypes.sizeof(BItem._ctype)
-            else:
-                _ctype = ctypes.c_void_p
-            if issubclass(BItem, CTypesGenericArray):
-                _reftypename = BItem._get_c_name('(* &)')
-            else:
-                _reftypename = BItem._get_c_name(' * &')
-
-            def __init__(self, init):
-                ctypeobj = BItem._create_ctype_obj(init)
-                if kind == 'charp':
-                    self.__as_strbuf = ctypes.create_string_buffer(
-                        ctypeobj.value + b'\x00')
-                    self._as_ctype_ptr = ctypes.cast(
-                        self.__as_strbuf, self._ctype)
-                else:
-                    self._as_ctype_ptr = ctypes.pointer(ctypeobj)
-                self._address = ctypes.cast(self._as_ctype_ptr,
-                                            ctypes.c_void_p).value
-                self._own = True
-
-            def __add__(self, other):
-                if isinstance(other, (int, long)):
-                    return self._new_pointer_at(self._address +
-                                                other * self._bitem_size)
-                else:
-                    return NotImplemented
-
-            def __sub__(self, other):
-                if isinstance(other, (int, long)):
-                    return self._new_pointer_at(self._address -
-                                                other * self._bitem_size)
-                elif type(self) is type(other):
-                    return (self._address - other._address) // self._bitem_size
-                else:
-                    return NotImplemented
-
-            def __getitem__(self, index):
-                if getattr(self, '_own', False) and index != 0:
-                    raise IndexError
-                return BItem._from_ctypes(self._as_ctype_ptr[index])
-
-            def __setitem__(self, index, value):
-                self._as_ctype_ptr[index] = BItem._to_ctypes(value)
-
-            if kind == 'charp' or kind == 'voidp':
-                @classmethod
-                def _arg_to_ctypes(cls, *value):
-                    if value and isinstance(value[0], bytes):
-                        return ctypes.c_char_p(value[0])
-                    else:
-                        return super(CTypesPtr, cls)._arg_to_ctypes(*value)
-
-            if kind == 'charp' or kind == 'bytep':
-                def _to_string(self, maxlen):
-                    if maxlen < 0:
-                        maxlen = sys.maxsize
-                    p = ctypes.cast(self._as_ctype_ptr,
-                                    ctypes.POINTER(ctypes.c_char))
-                    n = 0
-                    while n < maxlen and p[n] != b'\x00':
-                        n += 1
-                    return b''.join([p[i] for i in range(n)])
-
-            def _get_own_repr(self):
-                if getattr(self, '_own', False):
-                    return 'owning %d bytes' % (
-                        ctypes.sizeof(self._as_ctype_ptr.contents),)
-                return super(CTypesPtr, self)._get_own_repr()
-        #
-        if (BItem is self.ffi._get_cached_btype(model.void_type) or
-            BItem is self.ffi._get_cached_btype(model.PrimitiveType('char'))):
-            CTypesPtr._automatic_casts = True
-        #
-        CTypesPtr._fix_class()
-        return CTypesPtr
-
-    def new_array_type(self, CTypesPtr, length):
-        if length is None:
-            brackets = ' &[]'
-        else:
-            brackets = ' &[%d]' % length
-        BItem = CTypesPtr._BItem
-        getbtype = self.ffi._get_cached_btype
-        if BItem is getbtype(model.PrimitiveType('char')):
-            kind = 'char'
-        elif BItem in (getbtype(model.PrimitiveType('signed char')),
-                       getbtype(model.PrimitiveType('unsigned char'))):
-            kind = 'byte'
-        else:
-            kind = 'generic'
-        #
-        class CTypesArray(CTypesGenericArray):
-            __slots__ = ['_blob', '_own']
-            if length is not None:
-                _ctype = BItem._ctype * length
-            else:
-                __slots__.append('_ctype')
-            _reftypename = BItem._get_c_name(brackets)
-            _declared_length = length
-            _CTPtr = CTypesPtr
-
-            def __init__(self, init):
-                if length is None:
-                    if isinstance(init, (int, long)):
-                        len1 = init
-                        init = None
-                    elif kind == 'char' and isinstance(init, bytes):
-                        len1 = len(init) + 1    # extra null
-                    else:
-                        init = tuple(init)
-                        len1 = len(init)
-                    self._ctype = BItem._ctype * len1
-                self._blob = self._ctype()
-                self._own = True
-                if init is not None:
-                    self._initialize(self._blob, init)
-
-            @staticmethod
-            def _initialize(blob, init):
-                if isinstance(init, bytes):
-                    init = [init[i:i+1] for i in range(len(init))]
-                else:
-                    init = tuple(init)
-                if len(init) > len(blob):
-                    raise IndexError("too many initializers")
-                addr = ctypes.cast(blob, ctypes.c_void_p).value
-                PTR = ctypes.POINTER(BItem._ctype)
-                itemsize = ctypes.sizeof(BItem._ctype)
-                for i, value in enumerate(init):
-                    p = ctypes.cast(addr + i * itemsize, PTR)
-                    BItem._initialize(p.contents, value)
-
-            def __len__(self):
-                return len(self._blob)
-
-            def __getitem__(self, index):
-                if not (0 <= index < len(self._blob)):
-                    raise IndexError
-                return BItem._from_ctypes(self._blob[index])
-
-            def __setitem__(self, index, value):
-                if not (0 <= index < len(self._blob)):
-                    raise IndexError
-                self._blob[index] = BItem._to_ctypes(value)
-
-            if kind == 'char' or kind == 'byte':
-                def _to_string(self, maxlen):
-                    if maxlen < 0:
-                        maxlen = len(self._blob)
-                    p = ctypes.cast(self._blob,
-                                    ctypes.POINTER(ctypes.c_char))
-                    n = 0
-                    while n < maxlen and p[n] != b'\x00':
-                        n += 1
-                    return b''.join([p[i] for i in range(n)])
-
-            def _get_own_repr(self):
-                if getattr(self, '_own', False):
-                    return 'owning %d bytes' % (ctypes.sizeof(self._blob),)
-                return super(CTypesArray, self)._get_own_repr()
-
-            def _convert_to_address(self, BClass):
-                if BClass in (CTypesPtr, None) or BClass._automatic_casts:
-                    return ctypes.addressof(self._blob)
-                else:
-                    return CTypesData._convert_to_address(self, BClass)
-
-            @staticmethod
-            def _from_ctypes(ctypes_array):
-                self = CTypesArray.__new__(CTypesArray)
-                self._blob = ctypes_array
-                return self
-
-            @staticmethod
-            def _arg_to_ctypes(value):
-                return CTypesPtr._arg_to_ctypes(value)
-
-            def __add__(self, other):
-                if isinstance(other, (int, long)):
-                    return CTypesPtr._new_pointer_at(
-                        ctypes.addressof(self._blob) +
-                        other * ctypes.sizeof(BItem._ctype))
-                else:
-                    return NotImplemented
-
-            @classmethod
-            def _cast_from(cls, source):
-                raise NotImplementedError("casting to %r" % (
-                    cls._get_c_name(),))
-        #
-        CTypesArray._fix_class()
-        return CTypesArray
-
-    def _new_struct_or_union(self, kind, name, base_ctypes_class):
-        #
-        class struct_or_union(base_ctypes_class):
-            pass
-        struct_or_union.__name__ = '%s_%s' % (kind, name)
-        kind1 = kind
-        #
-        class CTypesStructOrUnion(CTypesBaseStructOrUnion):
-            __slots__ = ['_blob']
-            _ctype = struct_or_union
-            _reftypename = '%s &' % (name,)
-            _kind = kind = kind1
-        #
-        CTypesStructOrUnion._fix_class()
-        return CTypesStructOrUnion
-
-    def new_struct_type(self, name):
-        return self._new_struct_or_union('struct', name, ctypes.Structure)
-
-    def new_union_type(self, name):
-        return self._new_struct_or_union('union', name, ctypes.Union)
-
-    def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
-                                 totalsize=-1, totalalignment=-1, sflags=0):
-        if totalsize >= 0 or totalalignment >= 0:
-            raise NotImplementedError("the ctypes backend of CFFI does not support "
-                                      "structures completed by verify(); please "
-                                      "compile and install the _cffi_backend module.")
-        struct_or_union = CTypesStructOrUnion._ctype
-        fnames = [fname for (fname, BField, bitsize) in fields]
-        btypes = [BField for (fname, BField, bitsize) in fields]
-        bitfields = [bitsize for (fname, BField, bitsize) in fields]
-        #
-        bfield_types = {}
-        cfields = []
-        for (fname, BField, bitsize) in fields:
-            if bitsize < 0:
-                cfields.append((fname, BField._ctype))
-                bfield_types[fname] = BField
-            else:
-                cfields.append((fname, BField._ctype, bitsize))
-                bfield_types[fname] = Ellipsis
-        if sflags & 8:
-            struct_or_union._pack_ = 1
-        struct_or_union._fields_ = cfields
-        CTypesStructOrUnion._bfield_types = bfield_types
-        #
-        @staticmethod
-        def _create_ctype_obj(init):
-            result = struct_or_union()
-            if init is not None:
-                initialize(result, init)
-            return result
-        CTypesStructOrUnion._create_ctype_obj = _create_ctype_obj
-        #
-        def initialize(blob, init):
-            if is_union:
-                if len(init) > 1:
-                    raise ValueError("union initializer: %d items given, but "
-                                    "only one supported (use a dict if needed)"
-                                     % (len(init),))
-            if not isinstance(init, dict):
-                if isinstance(init, (bytes, unicode)):
-                    raise TypeError("union initializer: got a str")
-                init = tuple(init)
-                if len(init) > len(fnames):
-                    raise ValueError("too many values for %s initializer" %
-                                     CTypesStructOrUnion._get_c_name())
-                init = dict(zip(fnames, init))
-            addr = ctypes.addressof(blob)
-            for fname, value in init.items():
-                BField, bitsize = name2fieldtype[fname]
-                assert bitsize < 0, \
-                       "not implemented: initializer with bit fields"
-                offset = CTypesStructOrUnion._offsetof(fname)
-                PTR = ctypes.POINTER(BField._ctype)
-                p = ctypes.cast(addr + offset, PTR)
-                BField._initialize(p.contents, value)
-        is_union = CTypesStructOrUnion._kind == 'union'
-        name2fieldtype = dict(zip(fnames, zip(btypes, bitfields)))
-        #
-        for fname, BField, bitsize in fields:
-            if fname == '':
-                raise NotImplementedError("nested anonymous structs/unions")
-            if hasattr(CTypesStructOrUnion, fname):
-                raise ValueError("the field name %r conflicts in "
-                                 "the ctypes backend" % fname)
-            if bitsize < 0:
-                def getter(self, fname=fname, BField=BField,
-                           offset=CTypesStructOrUnion._offsetof(fname),
-                           PTR=ctypes.POINTER(BField._ctype)):
-                    addr = ctypes.addressof(self._blob)
-                    p = ctypes.cast(addr + offset, PTR)
-                    return BField._from_ctypes(p.contents)
-                def setter(self, value, fname=fname, BField=BField):
-                    setattr(self._blob, fname, BField._to_ctypes(value))
-                #
-                if issubclass(BField, CTypesGenericArray):
-                    setter = None
-                    if BField._declared_length == 0:
-                        def getter(self, fname=fname, BFieldPtr=BField._CTPtr,
-                                   offset=CTypesStructOrUnion._offsetof(fname),
-                                   PTR=ctypes.POINTER(BField._ctype)):
-                            addr = ctypes.addressof(self._blob)
-                            p = ctypes.cast(addr + offset, PTR)
-                            return BFieldPtr._from_ctypes(p)
-                #
-            else:
-                def getter(self, fname=fname, BField=BField):
-                    return BField._from_ctypes(getattr(self._blob, fname))
-                def setter(self, value, fname=fname, BField=BField):
-                    # xxx obscure workaround
-                    value = BField._to_ctypes(value)
-                    oldvalue = getattr(self._blob, fname)
-                    setattr(self._blob, fname, value)
-                    if value != getattr(self._blob, fname):
-                        setattr(self._blob, fname, oldvalue)
-                        raise OverflowError("value too large for bitfield")
-            setattr(CTypesStructOrUnion, fname, property(getter, setter))
-        #
-        CTypesPtr = self.ffi._get_cached_btype(model.PointerType(tp))
-        for fname in fnames:
-            if hasattr(CTypesPtr, fname):
-                raise ValueError("the field name %r conflicts in "
-                                 "the ctypes backend" % fname)
-            def getter(self, fname=fname):
-                return getattr(self[0], fname)
-            def setter(self, value, fname=fname):
-                setattr(self[0], fname, value)
-            setattr(CTypesPtr, fname, property(getter, setter))
-
-    def new_function_type(self, BArgs, BResult, has_varargs):
-        nameargs = [BArg._get_c_name() for BArg in BArgs]
-        if has_varargs:
-            nameargs.append('...')
-        nameargs = ', '.join(nameargs)
-        #
-        class CTypesFunctionPtr(CTypesGenericPtr):
-            __slots__ = ['_own_callback', '_name']
-            _ctype = ctypes.CFUNCTYPE(getattr(BResult, '_ctype', None),
-                                      *[BArg._ctype for BArg in BArgs],
-                                      use_errno=True)
-            _reftypename = BResult._get_c_name('(* &)(%s)' % (nameargs,))
-
-            def __init__(self, init, error=None):
-                # create a callback to the Python callable init()
-                import traceback
-                assert not has_varargs, "varargs not supported for callbacks"
-                if getattr(BResult, '_ctype', None) is not None:
-                    error = BResult._from_ctypes(
-                        BResult._create_ctype_obj(error))
-                else:
-                    error = None
-                def callback(*args):
-                    args2 = []
-                    for arg, BArg in zip(args, BArgs):
-                        args2.append(BArg._from_ctypes(arg))
-                    try:
-                        res2 = init(*args2)
-                        res2 = BResult._to_ctypes(res2)
-                    except:
-                        traceback.print_exc()
-                        res2 = error
-                    if issubclass(BResult, CTypesGenericPtr):
-                        if res2:
-                            res2 = ctypes.cast(res2, ctypes.c_void_p).value
-                                # .value: http://bugs.python.org/issue1574593
-                        else:
-                            res2 = None
-                    #print repr(res2)
-                    return res2
-                if issubclass(BResult, CTypesGenericPtr):
-                    # The only pointers callbacks can return are void*s:
-                    # http://bugs.python.org/issue5710
-                    callback_ctype = ctypes.CFUNCTYPE(
-                        ctypes.c_void_p,
-                        *[BArg._ctype for BArg in BArgs],
-                        use_errno=True)
-                else:
-                    callback_ctype = CTypesFunctionPtr._ctype
-                self._as_ctype_ptr = callback_ctype(callback)
-                self._address = ctypes.cast(self._as_ctype_ptr,
-                                            ctypes.c_void_p).value
-                self._own_callback = init
-
-            @staticmethod
-            def _initialize(ctypes_ptr, value):
-                if value:
-                    raise NotImplementedError("ctypes backend: not supported: "
-                                          "initializers for function pointers")
-
-            def __repr__(self):
-                c_name = getattr(self, '_name', None)
-                if c_name:
-                    i = self._reftypename.index('(* &)')
-                    if self._reftypename[i-1] not in ' )*':
-                        c_name = ' ' + c_name
-                    c_name = self._reftypename.replace('(* &)', c_name)
-                return CTypesData.__repr__(self, c_name)
-
-            def _get_own_repr(self):
-                if getattr(self, '_own_callback', None) is not None:
-                    return 'calling %r' % (self._own_callback,)
-                return super(CTypesFunctionPtr, self)._get_own_repr()
-
-            def __call__(self, *args):
-                if has_varargs:
-                    assert len(args) >= len(BArgs)
-                    extraargs = args[len(BArgs):]
-                    args = args[:len(BArgs)]
-                else:
-                    assert len(args) == len(BArgs)
-                ctypes_args = []
-                for arg, BArg in zip(args, BArgs):
-                    ctypes_args.append(BArg._arg_to_ctypes(arg))
-                if has_varargs:
-                    for i, arg in enumerate(extraargs):
-                        if arg is None:
-                            ctypes_args.append(ctypes.c_void_p(0))  # NULL
-                            continue
-                        if not isinstance(arg, CTypesData):
-                            raise TypeError(
-                                "argument %d passed in the variadic part "
-                                "needs to be a cdata object (got %s)" %
-                                (1 + len(BArgs) + i, type(arg).__name__))
-                        ctypes_args.append(arg._arg_to_ctypes(arg))
-                result = self._as_ctype_ptr(*ctypes_args)
-                return BResult._from_ctypes(result)
-        #
-        CTypesFunctionPtr._fix_class()
-        return CTypesFunctionPtr
-
-    def new_enum_type(self, name, enumerators, enumvalues, CTypesInt):
-        assert isinstance(name, str)
-        reverse_mapping = dict(zip(reversed(enumvalues),
-                                   reversed(enumerators)))
-        #
-        class CTypesEnum(CTypesInt):
-            __slots__ = []
-            _reftypename = '%s &' % name
-
-            def _get_own_repr(self):
-                value = self._value
-                try:
-                    return '%d: %s' % (value, reverse_mapping[value])
-                except KeyError:
-                    return str(value)
-
-            def _to_string(self, maxlen):
-                value = self._value
-                try:
-                    return reverse_mapping[value]
-                except KeyError:
-                    return str(value)
-        #
-        CTypesEnum._fix_class()
-        return CTypesEnum
-
-    def get_errno(self):
-        return ctypes.get_errno()
-
-    def set_errno(self, value):
-        ctypes.set_errno(value)
-
-    def string(self, b, maxlen=-1):
-        return b._to_string(maxlen)
-
-    def buffer(self, bptr, size=-1):
-        raise NotImplementedError("buffer() with ctypes backend")
-
-    def sizeof(self, cdata_or_BType):
-        if isinstance(cdata_or_BType, CTypesData):
-            return cdata_or_BType._get_size_of_instance()
-        else:
-            assert issubclass(cdata_or_BType, CTypesData)
-            return cdata_or_BType._get_size()
-
-    def alignof(self, BType):
-        assert issubclass(BType, CTypesData)
-        return BType._alignment()
-
-    def newp(self, BType, source):
-        if not issubclass(BType, CTypesData):
-            raise TypeError
-        return BType._newp(source)
-
-    def cast(self, BType, source):
-        return BType._cast_from(source)
-
-    def callback(self, BType, source, error, onerror):
-        assert onerror is None   # XXX not implemented
-        return BType(source, error)
-
-    def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
-
-        if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
-                raise TypeError("Can remove destructor only on a object "
-                                "previously returned by ffi.gc()")
-            cdata._destructor = None
-            return None
-
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
-        return new_cdata
-
-    typeof = type
-
-    def getcname(self, BType, replace_with):
-        return BType._get_c_name(replace_with)
-
-    def typeoffsetof(self, BType, fieldname, num=0):
-        if isinstance(fieldname, str):
-            if num == 0 and issubclass(BType, CTypesGenericPtr):
-                BType = BType._BItem
-            if not issubclass(BType, CTypesBaseStructOrUnion):
-                raise TypeError("expected a struct or union ctype")
-            BField = BType._bfield_types[fieldname]
-            if BField is Ellipsis:
-                raise TypeError("not supported for bitfields")
-            return (BField, BType._offsetof(fieldname))
-        elif isinstance(fieldname, (int, long)):
-            if issubclass(BType, CTypesGenericArray):
-                BType = BType._CTPtr
-            if not issubclass(BType, CTypesGenericPtr):
-                raise TypeError("expected an array or ptr ctype")
-            BItem = BType._BItem
-            offset = BItem._get_size() * fieldname
-            if offset > sys.maxsize:
-                raise OverflowError
-            return (BItem, offset)
-        else:
-            raise TypeError(type(fieldname))
-
-    def rawaddressof(self, BTypePtr, cdata, offset=None):
-        if isinstance(cdata, CTypesBaseStructOrUnion):
-            ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata))
-        elif isinstance(cdata, CTypesGenericPtr):
-            if offset is None or not issubclass(type(cdata)._BItem,
-                                                CTypesBaseStructOrUnion):
-                raise TypeError("unexpected cdata type")
-            ptr = type(cdata)._to_ctypes(cdata)
-        elif isinstance(cdata, CTypesGenericArray):
-            ptr = type(cdata)._to_ctypes(cdata)
-        else:
-            raise TypeError("expected a <cdata 'struct-or-union'>")
-        if offset:
-            ptr = ctypes.cast(
-                ctypes.c_void_p(
-                    ctypes.cast(ptr, ctypes.c_void_p).value + offset),
-                type(ptr))
-        return BTypePtr._from_ctypes(ptr)
-
-
-class CTypesLibrary(object):
-
-    def __init__(self, backend, cdll):
-        self.backend = backend
-        self.cdll = cdll
-
-    def load_function(self, BType, name):
-        c_func = getattr(self.cdll, name)
-        funcobj = BType._from_ctypes(c_func)
-        funcobj._name = name
-        return funcobj
-
-    def read_variable(self, BType, name):
-        try:
-            ctypes_obj = BType._ctype.in_dll(self.cdll, name)
-        except AttributeError as e:
-            raise NotImplementedError(e)
-        return BType._from_ctypes(ctypes_obj)
-
-    def write_variable(self, BType, name, value):
-        new_ctypes_obj = BType._to_ctypes(value)
-        ctypes_obj = BType._ctype.in_dll(self.cdll, name)
-        ctypes.memmove(ctypes.addressof(ctypes_obj),
-                       ctypes.addressof(new_ctypes_obj),
-                       ctypes.sizeof(BType._ctype))
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -23,6 +23,10 @@
   argument (in older versions, a copy would be made).  This used to be
   a CPython-only optimization.
 
+* Removed the ctypes backend.  If ``_cffi_backend`` was not compiled,
+  you could ask (using an undocumented interface) for ``backend_ctypes``
+  instead.  That was never fully functional and long deprecated.
+
 
 v1.7
 ====
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/test_backend.py
rename from testing/cffi0/backend_tests.py
rename to testing/cffi0/test_backend.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/test_backend.py
@@ -11,10 +11,10 @@
 SIZE_OF_WCHAR = ctypes.sizeof(ctypes.c_wchar)
 
 
-class BackendTests:
+class TestBackend(object):
 
     def test_integer_ranges(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         for (c_type, size) in [('char', 1),
                                ('short', 2),
                                ('short int', 2),
@@ -34,7 +34,7 @@
                 self._test_int_type(ffi, c_decl, size, unsigned)
 
     def test_fixedsize_int(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         for size in [1, 2, 4, 8]:
             self._test_int_type(ffi, 'int%d_t' % (8*size), size, False)
             self._test_int_type(ffi, 'uint%d_t' % (8*size), size, True)
@@ -79,12 +79,12 @@
         assert ffi.new(c_decl_ptr, long(max))[0] == max
 
     def test_new_unsupported_type(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         e = py.test.raises(TypeError, ffi.new, "int")
         assert str(e.value) == "expected a pointer or array ctype, got 'int'"
 
     def test_new_single_integer(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int *")     # similar to ffi.new("int[1]")
         assert p[0] == 0
         p[0] = -123
@@ -94,14 +94,14 @@
         assert repr(p) == "<cdata 'int *' owning %d bytes>" % SIZE_OF_INT
 
     def test_new_array_no_arg(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int[10]")
         # the object was zero-initialized:
         for i in range(10):
             assert p[i] == 0
 
     def test_array_indexing(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int[10]")
         p[0] = 42
         p[9] = 43
@@ -113,7 +113,7 @@
         py.test.raises(IndexError, "p[-1] = 44")
 
     def test_new_array_args(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         # this tries to be closer to C: where we say "int x[5] = {10, 20, ..}"
         # then here we must enclose the items in a list
         p = ffi.new("int[5]", [10, 20, 30, 40, 50])
@@ -132,7 +132,7 @@
         assert repr(p) == "<cdata 'int[4]' owning %d bytes>" % (4*SIZE_OF_INT)
 
     def test_new_array_varsize(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int[]", 10)     # a single integer is the length
         assert p[9] == 0
         py.test.raises(IndexError, "p[10]")
@@ -151,7 +151,7 @@
         assert repr(p) == "<cdata 'int[]' owning 0 bytes>"
 
     def test_pointer_init(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         n = ffi.new("int *", 24)
         a = ffi.new("int *[10]", [ffi.NULL, ffi.NULL, n, n, ffi.NULL])
         for i in range(10):
@@ -160,14 +160,14 @@
         assert a[2] == a[3] == n
 
     def test_cannot_cast(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("short int[10]")
         e = py.test.raises(TypeError, ffi.new, "long int **", a)
         msg = str(e.value)
         assert "'short[10]'" in msg and "'long *'" in msg
 
     def test_new_pointer_to_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("int[4]", [100, 102, 104, 106])
         p = ffi.new("int **", a)
         assert p[0] == ffi.cast("int *", a)
@@ -180,7 +180,7 @@
         # keepalive: a
 
     def test_pointer_direct(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.cast("int*", 0)
         assert p is not None
         assert bool(p) is False
@@ -195,9 +195,11 @@
         assert p[0] == 123
         assert p[1] == 456
 
+    TypeRepr = "<ctype '%s'>"
+
     def test_repr(self):
         typerepr = self.TypeRepr
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { short a, b, c; };")
         p = ffi.cast("short unsigned int", 0)
         assert repr(p) == "<cdata 'unsigned short' 0>"
@@ -248,7 +250,7 @@
         assert repr(ffi.typeof(q)) == typerepr % "struct foo"
 
     def test_new_array_of_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int[3][4]")
         p[0][0] = 10
         p[2][3] = 33
@@ -257,12 +259,12 @@
         py.test.raises(IndexError, "p[1][-1]")
 
     def test_constructor_array_of_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int[3][2]", [[10, 11], [12, 13], [14, 15]])
         assert p[2][1] == 15
 
     def test_new_array_of_pointer_1(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         n = ffi.new("int*", 99)
         p = ffi.new("int*[4]")
         p[3] = n
@@ -271,7 +273,7 @@
         assert a[0] == 99
 
     def test_new_array_of_pointer_2(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         n = ffi.new("int[1]", [99])
         p = ffi.new("int*[4]")
         p[3] = n
@@ -280,7 +282,7 @@
         assert a[0] == 99
 
     def test_char(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         assert ffi.new("char*", b"\xff")[0] == b'\xff'
         assert ffi.new("char*")[0] == b'\x00'
         assert int(ffi.cast("char", 300)) == 300 - 256
@@ -317,7 +319,7 @@
             py.test.skip("NotImplementedError: wchar_t")
 
     def test_wchar_t(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         self.check_wchar_t(ffi)
         assert ffi.new("wchar_t*", u+'x')[0] == u+'x'
         assert ffi.new("wchar_t*", u+'\u1234')[0] == u+'\u1234'
@@ -372,7 +374,7 @@
         py.test.raises(IndexError, ffi.new, "wchar_t[2]", u+"abc")
 
     def test_none_as_null_doesnt_work(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int*[1]")
         assert p[0] is not None
         assert p[0] != None
@@ -387,7 +389,7 @@
         assert p[0] == ffi.NULL
 
     def test_float(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("float[]", [-2, -2.5])
         assert p[0] == -2.0
         assert p[1] == -2.5
@@ -412,7 +414,7 @@
         assert p[0] == INF     # infinite, not enough precision
 
     def test_struct_simple(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a; short b, c; };")
         s = ffi.new("struct foo*")
         assert s.a == s.b == s.c == 0
@@ -431,7 +433,7 @@
         py.test.raises(ValueError, ffi.new, "struct foo*", [1, 2, 3, 4])
 
     def test_constructor_struct_from_dict(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a; short b, c; };")
         s = ffi.new("struct foo*", {'b': 123, 'c': 456})
         assert s.a == 0
@@ -440,7 +442,7 @@
         py.test.raises(KeyError, ffi.new, "struct foo*", {'d': 456})
 
     def test_struct_pointer(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a; short b, c; };")
         s = ffi.new("struct foo*")
         assert s[0].a == s[0].b == s[0].c == 0
@@ -450,13 +452,13 @@
         py.test.raises(IndexError, "s[1]")
 
     def test_struct_opaque(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         py.test.raises(TypeError, ffi.new, "struct baz*")
         p = ffi.new("struct baz **")    # this works
         assert p[0] == ffi.NULL
 
     def test_pointer_to_struct(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a; short b, c; };")
         s = ffi.new("struct foo *")
         s.a = -42
@@ -478,7 +480,7 @@
         assert p[0][0].a == -46
 
     def test_constructor_struct_of_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a[2]; char b[3]; };")
         s = ffi.new("struct foo *", [[10, 11], [b'a', b'b', b'c']])
         assert s.a[1] == 11
@@ -489,7 +491,7 @@
         assert s.b[2] == b'c'
 
     def test_recursive_struct(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int value; struct foo *next; };")
         s = ffi.new("struct foo*")
         t = ffi.new("struct foo*")
@@ -500,7 +502,7 @@
         assert s.next.value == 456
 
     def test_union_simple(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("union foo { int a; short b, c; };")
         u = ffi.new("union foo*")
         assert u.a == u.b == u.c == 0
@@ -515,13 +517,13 @@
         assert repr(u) == "<cdata 'union foo *' owning %d bytes>" % SIZE_OF_INT
 
     def test_union_opaque(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         py.test.raises(TypeError, ffi.new, "union baz *")
         u = ffi.new("union baz **")   # this works
         assert u[0] == ffi.NULL
 
     def test_union_initializer(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("union foo { char a; int b; };")
         py.test.raises(TypeError, ffi.new, "union foo*", b'A')
         py.test.raises(TypeError, ffi.new, "union foo*", 5)
@@ -536,7 +538,7 @@
         assert u.b == 0
 
     def test_sizeof_type(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("""
             struct foo { int a; short b, c, d; };
             union foo { int a; short b, c, d; };
@@ -553,7 +555,7 @@
             assert size == expected_size, (size, expected_size, ctype)
 
     def test_sizeof_cdata(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         assert ffi.sizeof(ffi.new("short*")) == SIZE_OF_PTR
         assert ffi.sizeof(ffi.cast("short", 123)) == SIZE_OF_SHORT
         #
@@ -562,7 +564,7 @@
         assert ffi.sizeof(a) == 5 * SIZE_OF_INT
 
     def test_string_from_char_pointer(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         x = ffi.new("char*", b"x")
         assert str(x) == repr(x)
         assert ffi.string(x) == b"x"
@@ -570,7 +572,7 @@
         py.test.raises(TypeError, ffi.new, "char*", unicode("foo"))
 
     def test_unicode_from_wchar_pointer(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         self.check_wchar_t(ffi)
         x = ffi.new("wchar_t*", u+"x")
         assert unicode(x) == unicode(repr(x))
@@ -578,7 +580,7 @@
         assert ffi.string(ffi.new("wchar_t*", u+"\x00")) == u+""
 
     def test_string_from_char_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("char[]", b"hello.")
         p[5] = b'!'
         assert ffi.string(p) == b"hello!"
@@ -595,7 +597,7 @@
         assert ffi.string(p) == b'hello'
 
     def test_string_from_wchar_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         self.check_wchar_t(ffi)
         assert ffi.string(ffi.cast("wchar_t", "x")) == u+"x"
         assert ffi.string(ffi.cast("wchar_t", u+"x")) == u+"x"
@@ -623,7 +625,7 @@
 
     def test_fetch_const_char_p_field(self):
         # 'const' is ignored so far
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { const char *name; };")
         t = ffi.new("const char[]", b"testing")
         s = ffi.new("struct foo*", [t])
@@ -635,7 +637,7 @@
 
     def test_fetch_const_wchar_p_field(self):
         # 'const' is ignored so far
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         self.check_wchar_t(ffi)
         ffi.cdef("struct foo { const wchar_t *name; };")
         t = ffi.new("const wchar_t[]", u+"testing")
@@ -646,7 +648,7 @@
         assert s.name == ffi.NULL
 
     def test_voidp(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         py.test.raises(TypeError, ffi.new, "void*")
         p = ffi.new("void **")
         assert p[0] == ffi.NULL
@@ -667,7 +669,7 @@
         py.test.raises(TypeError, "s.r = b")    # fails
 
     def test_functionptr_simple(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         py.test.raises(TypeError, ffi.callback, "int(*)(int)", 0)
         def cb(n):
             return n + 1
@@ -692,12 +694,12 @@
         assert res == 46
 
     def test_functionptr_advanced(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         t = ffi.typeof("int(*(*)(int))(int)")
         assert repr(t) == self.TypeRepr % "int(*(*)(int))(int)"
 
     def test_functionptr_voidptr_return(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         def cb():
             return ffi.NULL
         p = ffi.callback("void*(*)()", cb)
@@ -713,7 +715,7 @@
         assert res == void_ptr
 
     def test_functionptr_intptr_return(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         def cb():
             return ffi.NULL
         p = ffi.callback("int*(*)()", cb)
@@ -735,7 +737,7 @@
         assert res == int_array_ptr
 
     def test_functionptr_void_return(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         def foo():
             pass
         foo_cb = ffi.callback("void foo()", foo)
@@ -743,7 +745,7 @@
         assert result is None
 
     def test_char_cast(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.cast("int", b'\x01')
         assert ffi.typeof(p) is ffi.typeof("int")
         assert int(p) == 1
@@ -755,7 +757,7 @@
         assert int(p) == 0x81
 
     def test_wchar_cast(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         self.check_wchar_t(ffi)
         p = ffi.cast("int", ffi.cast("wchar_t", u+'\u1234'))
         assert int(p) == 0x1234
@@ -771,7 +773,7 @@
         assert int(p) == 0x1234
 
     def test_cast_array_to_charp(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("short int[]", [0x1234, 0x5678])
         p = ffi.cast("char*", a)
         data = b''.join([p[i] for i in range(4)])
@@ -781,7 +783,7 @@
             assert data == b'\x12\x34\x56\x78'
 
     def test_cast_between_pointers(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("short int[]", [0x1234, 0x5678])
         p = ffi.cast("short*", a)
         p2 = ffi.cast("int*", p)
@@ -793,7 +795,7 @@
             assert data == b'\x12\x34\x56\x78'
 
     def test_cast_pointer_and_int(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("short int[]", [0x1234, 0x5678])
         l1 = ffi.cast("intptr_t", a)
         p = ffi.cast("short*", a)
@@ -805,7 +807,7 @@
         assert int(ffi.cast("intptr_t", ffi.NULL)) == 0
 
     def test_cast_functionptr_and_int(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         def cb(n):
             return n + 1
         a = ffi.callback("int(*)(int)", cb)
@@ -817,7 +819,7 @@
         assert hash(a) == hash(b)
 
     def test_callback_crash(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         def cb(n):
             raise Exception
         a = ffi.callback("int(*)(int)", cb, error=42)
@@ -825,7 +827,7 @@
         assert res == 42
 
     def test_structptr_argument(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo_s { int a, b; };")
         def cb(p):
             return p[0].a * 1000 + p[0].b * 100 + p[1].a * 10 + p[1].b
@@ -836,7 +838,7 @@
         assert res == 5008
 
     def test_array_argument_as_list(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo_s { int a, b; };")
         seen = []
         def cb(argv):
@@ -847,7 +849,7 @@
         assert seen == [b"foobar", b"baz"]
 
     def test_cast_float(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.cast("float", 12)
         assert float(a) == 12.0
         a = ffi.cast("float", 12.5)
@@ -871,7 +873,7 @@
         assert ffi.string(a) == b"B"
 
     def test_enum(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("enum foo { A0, B0, CC0, D0 };")
         assert ffi.string(ffi.cast("enum foo", 0)) == "A0"
         assert ffi.string(ffi.cast("enum foo", 2)) == "CC0"
@@ -893,7 +895,7 @@
         assert ffi.string(ffi.cast("enum baz", 0x2000)) == "B2"
 
     def test_enum_in_struct(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("enum foo { A, B, C, D }; struct bar { enum foo e; };")
         s = ffi.new("struct bar *")
         s.e = 0
@@ -914,7 +916,7 @@
         py.test.raises(TypeError, "s.e = '#7'")
 
     def test_enum_non_contiguous(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("enum foo { A, B=42, C };")
         assert ffi.string(ffi.cast("enum foo", 0)) == "A"
         assert ffi.string(ffi.cast("enum foo", 42)) == "B"
@@ -924,7 +926,7 @@
         assert ffi.string(invalid_value) == "2"
 
     def test_enum_char_hex_oct(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef(r"enum foo{A='!', B='\'', C=0x10, D=010, E=- 0x10, F=-010};")
         assert ffi.string(ffi.cast("enum foo", ord('!'))) == "A"
         assert ffi.string(ffi.cast("enum foo", ord("'"))) == "B"
@@ -934,7 +936,7 @@
         assert ffi.string(ffi.cast("enum foo", -8)) == "F"
 
     def test_enum_partial(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef(r"enum foo {A, ...}; enum bar { B, C };")
         lib = ffi.dlopen(None)
         assert lib.B == 0
@@ -942,7 +944,7 @@
         assert lib.C == 1
 
     def test_array_of_struct(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a, b; };")
         s = ffi.new("struct foo[1]")
         py.test.raises(AttributeError, 's.b')
@@ -952,12 +954,12 @@
         py.test.raises(IndexError, 's[1]')
 
     def test_pointer_to_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int(**)[5]")
         assert repr(p) == "<cdata 'int(* *)[5]' owning %d bytes>" % SIZE_OF_PTR
 
     def test_iterate_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("char[]", b"hello")
         assert list(a) == [b"h", b"e", b"l", b"l", b"o", b"\0"]
         assert list(iter(a)) == [b"h", b"e", b"l", b"l", b"o", b"\0"]
@@ -968,14 +970,14 @@
         py.test.raises(TypeError, list, ffi.new("int *"))
 
     def test_offsetof(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a, b, c; };")
         assert ffi.offsetof("struct foo", "a") == 0
         assert ffi.offsetof("struct foo", "b") == 4
         assert ffi.offsetof("struct foo", "c") == 8
 
     def test_offsetof_nested(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a, b, c; };"
                  "struct bar { struct foo d, e; };")
         assert ffi.offsetof("struct bar", "e") == 12
@@ -985,7 +987,7 @@
         assert ffi.offsetof("struct bar", "e", "c") == 20
 
     def test_offsetof_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         assert ffi.offsetof("int[]", 51) == 51 * ffi.sizeof("int")
         assert ffi.offsetof("int *", 51) == 51 * ffi.sizeof("int")
         ffi.cdef("struct bar { int a, b; int c[99]; };")
@@ -994,14 +996,14 @@
         assert ffi.offsetof("struct bar", "c", 51) == 53 * ffi.sizeof("int")
 
     def test_alignof(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { char a; short b; char c; };")
         assert ffi.alignof("int") == 4
         assert ffi.alignof("double") in (4, 8)
         assert ffi.alignof("struct foo") == 2
 
     def test_bitfield(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo { int a:10, b:20, c:3; };")
         assert ffi.sizeof("struct foo") == 8
         s = ffi.new("struct foo *")
@@ -1021,7 +1023,7 @@
         assert s.c == -4
 
     def test_bitfield_enum(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("""
             typedef enum { AA, BB, CC } foo_e;
             typedef struct { foo_e f:2; } foo_s;
@@ -1031,7 +1033,7 @@
         assert s.f == 2
 
     def test_anonymous_struct(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("typedef struct { int a; } foo_t;")
         ffi.cdef("typedef struct { char b, c; } bar_t;")
         f = ffi.new("foo_t *", [12345])
@@ -1043,13 +1045,13 @@
 
     def test_struct_with_two_usages(self):
         for name in ['foo_s', '']:    # anonymous or not
-            ffi = FFI(backend=self.Backend())
+            ffi = FFI()
             ffi.cdef("typedef struct %s { int a; } foo_t, *foo_p;" % name)
             f = ffi.new("foo_t *", [12345])
             ps = ffi.new("foo_p[]", [f])
 
     def test_pointer_arithmetic(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         s = ffi.new("short[]", list(range(100, 110)))
         p = ffi.cast("short *", s)
         assert p[2] == 102
@@ -1063,7 +1065,7 @@
         assert p+1 == s+1
 
     def test_pointer_comparison(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         s = ffi.new("short[]", list(range(100)))
         p = ffi.cast("short *", s)
         assert (p <  s) is False
@@ -1114,7 +1116,7 @@
         assert (q != None) is True
 
     def test_no_integer_comparison(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         x = ffi.cast("int", 123)
         y = ffi.cast("int", 456)
         py.test.raises(TypeError, "x < y")
@@ -1124,7 +1126,7 @@
         py.test.raises(TypeError, "z < y")
 
     def test_ffi_buffer_ptr(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("short *", 100)
         try:
             b = ffi.buffer(a)
@@ -1143,7 +1145,7 @@
         assert a[0] == 101
 
     def test_ffi_buffer_array(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("int[]", list(range(100, 110)))
         try:
             b = ffi.buffer(a)
@@ -1160,7 +1162,7 @@
         assert a[1] == 0x45
 
     def test_ffi_buffer_ptr_size(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a = ffi.new("short *", 0x4243)
         try:
             b = ffi.buffer(a, 1)
@@ -1178,7 +1180,7 @@
             assert a[0] == 0x6343
 
     def test_ffi_buffer_array_size(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         a1 = ffi.new("int[]", list(range(100, 110)))
         a2 = ffi.new("int[]", list(range(100, 115)))
         try:
@@ -1188,7 +1190,7 @@
         assert ffi.buffer(a1)[:] == ffi.buffer(a2, 4*10)[:]
 
     def test_ffi_buffer_with_file(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         import tempfile, os, array
         fd, filename = tempfile.mkstemp()
         f = os.fdopen(fd, 'r+b')
@@ -1208,7 +1210,7 @@
         os.unlink(filename)
 
     def test_ffi_buffer_with_io(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         import io, array
         f = io.BytesIO()
         a = ffi.new("int[]", list(range(1005)))
@@ -1226,7 +1228,7 @@
         f.close()
 
     def test_array_in_struct(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo_s { int len; short data[5]; };")
         p = ffi.new("struct foo_s *")
         p.data[3] = 5
@@ -1234,7 +1236,7 @@
         assert repr(p.data).startswith("<cdata 'short[5]' 0x")
 
     def test_struct_containing_array_varsize_workaround(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo_s { int len; short data[0]; };")
         p = ffi.new("char[]", ffi.sizeof("struct foo_s") + 7 * SIZE_OF_SHORT)
         q = ffi.cast("struct foo_s *", p)
@@ -1247,7 +1249,7 @@
 
     def test_new_struct_containing_array_varsize(self):
         py.test.skip("later?")
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("struct foo_s { int len; short data[]; };")
         p = ffi.new("struct foo_s *", 10)     # a single integer is the length
         assert p.len == 0
@@ -1255,7 +1257,7 @@
         py.test.raises(IndexError, "p.data[10]")
 
     def test_ffi_typeof_getcname(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         assert ffi.getctype("int") == "int"
         assert ffi.getctype("int", 'x') == "int x"
         assert ffi.getctype("int*") == "int *"
@@ -1273,7 +1275,7 @@
         assert ffi.getctype("int[5]", ' ** foo ') == "int(** foo)[5]"
 
     def test_array_of_func_ptr(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         f = ffi.cast("int(*)(int)", 42)
         assert f != ffi.NULL
         py.test.raises(CDefError, ffi.cast, "int(int)", 42)
@@ -1294,7 +1296,7 @@
     def test_callback_as_function_argument(self):
         # In C, function arguments can be declared with a function type,
         # which is automatically replaced with the ptr-to-function type.
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         def cb(a, b):
             return chr(ord(a) + ord(b)).encode()
         f = ffi.callback("char cb(char, char)", cb)
@@ -1306,7 +1308,7 @@
 
     def test_vararg_callback(self):
         py.test.skip("callback with '...'")
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         def cb(i, va_list):
             j = ffi.va_arg(va_list, "int")
             k = ffi.va_arg(va_list, "long long")
@@ -1316,7 +1318,7 @@
         assert res == 20 + 300 + 5000
 
     def test_callback_decorator(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         #
         @ffi.callback("long(long, long)", error=42)
         def cb(a, b):
@@ -1327,8 +1329,8 @@
         assert cb((1 << (sz*8-1)) - 1, -10) == 42
 
     def test_unique_types(self):
-        ffi1 = FFI(backend=self.Backend())
-        ffi2 = FFI(backend=self.Backend())
+        ffi1 = FFI()
+        ffi2 = FFI()
         assert ffi1.typeof("char") is ffi2.typeof("char ")
         assert ffi1.typeof("long") is ffi2.typeof("signed long int")
         assert ffi1.typeof("double *") is ffi2.typeof("double*")
@@ -1347,7 +1349,7 @@
         assert ffi1.typeof("struct foo*") is ffi1.typeof("struct foo *")
 
     def test_anonymous_enum(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("typedef enum { Value0 = 0 } e, *pe;\n"
                  "typedef enum { Value1 = 1 } e1;")
         assert ffi.getctype("e*") == 'e *'
@@ -1355,7 +1357,7 @@
         assert ffi.getctype("e1*") == 'e1 *'
 
     def test_opaque_enum(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         ffi.cdef("enum foo;")
         from cffi import __version_info__
         if __version_info__ < (1, 8):
@@ -1366,20 +1368,20 @@
             "which integer type it is meant to be (unsigned/signed, int/long)")
 
     def test_new_ctype(self):
-        ffi = FFI(backend=self.Backend())
+        ffi = FFI()
         p = ffi.new("int *")
         py.test.raises(TypeError, ffi.new, p)
         p = ffi.new(ffi.typeof("int *"), 42)
         assert p[0] == 42
 

From pypy.commits at gmail.com  Sat Sep  3 05:02:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 02:02:21 -0700 (PDT)
Subject: [pypy-commit] cffi default: Fix test
Message-ID: <57ca919d.03121c0a.ff81.7b33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2748:849a40dd7925
Date: 2016-09-03 11:02 +0200
http://bitbucket.org/cffi/cffi/changeset/849a40dd7925/

Log:	Fix test

diff --git a/testing/cffi0/test_zintegration.py b/testing/cffi0/test_zintegration.py
--- a/testing/cffi0/test_zintegration.py
+++ b/testing/cffi0/test_zintegration.py
@@ -166,7 +166,7 @@
 
             setuptools.__version__ = 'development'
             kwds = _set_py_limited_api(Extension, {})
-            assert not kwds
+            assert kwds['py_limited_api'] == True
 
         finally:
             setuptools.__version__ = orig_version

From pypy.commits at gmail.com  Sat Sep  3 05:36:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 02:36:29 -0700 (PDT)
Subject: [pypy-commit] cffi default: Windows fun
Message-ID: <57ca999d.c3f0c20a.54624.e9bf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2750:d83bdf06b04f
Date: 2016-09-03 11:36 +0200
http://bitbucket.org/cffi/cffi/changeset/d83bdf06b04f/

Log:	Windows fun

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1974,9 +1974,9 @@
 
 def test_function_returns_partial_struct():
     ffi = FFI()
-    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    ffi.cdef("struct aaa { int a; ...; }; struct aaa f1(int);")
     lib = verify(ffi, "test_function_returns_partial_struct", """
-        struct a { int b, a, c; };
-        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+        struct aaa { int b, a, c; };
+        static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52

From pypy.commits at gmail.com  Sat Sep  3 04:32:11 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sat, 03 Sep 2016 01:32:11 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Claim QCGC supports rweakref to
 make pypy executable useable
Message-ID: <57ca8a8b.81a2c20a.d8de6.de70@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86853:9c2ceea892ed
Date: 2016-09-03 10:31 +0200
http://bitbucket.org/pypy/pypy/changeset/9c2ceea892ed/

Log:	Claim QCGC supports rweakref to make pypy executable useable

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -79,7 +79,7 @@
                               ("translation.gcrootfinder", "qcgc"),
                               ("translation.gcremovetypeptr", True),
                               ("translation.thread", False),
-                              ("translation.rweakref", False)],
+                              ("translation.rweakref", True)], # XXX
                      "minimark": [("translation.gctransformer", "framework")],
                      "incminimark": [("translation.gctransformer", "framework")],
                      },

From pypy.commits at gmail.com  Sat Sep  3 06:36:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 03:36:25 -0700 (PDT)
Subject: [pypy-commit] cffi default: If we say Py_LIMITED_API and we're
 compiling with a debug version of
Message-ID: <57caa7a9.11051c0a.8851f.9c7e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2751:0ff89f851d2c
Date: 2016-09-03 12:36 +0200
http://bitbucket.org/cffi/cffi/changeset/0ff89f851d2c/

Log:	If we say Py_LIMITED_API and we're compiling with a debug version of
	CPython, "#include <Python.h>" crashes. Work around it the hard way.

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')

From pypy.commits at gmail.com  Sat Sep  3 06:39:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 03:39:21 -0700 (PDT)
Subject: [pypy-commit] cffi default: Document 0ff89f851d2c
Message-ID: <57caa859.09afc20a.e370d.0052@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2752:02ca935a5c3f
Date: 2016-09-03 12:39 +0200
http://bitbucket.org/cffi/cffi/changeset/02ca935a5c3f/

Log:	Document 0ff89f851d2c

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -529,7 +529,9 @@
 the same version of CPython x.y).  However, the standard ``distutils``
 package will still produce a file called e.g.
 ``NAME.cpython-35m-x86_64-linux-gnu.so``.  You can manually rename it to
-``NAME.abi3.so``, or use setuptools version 26 or later.
+``NAME.abi3.so``, or use setuptools version 26 or later.  Also, note
+that compiling with a debug version of Python will not actually define
+``Py_LIMITED_API``, as doing so makes ``Python.h`` unhappy.
 
 **ffibuilder.compile(tmpdir='.', verbose=False):**
 explicitly generate the .py or .c file,

From pypy.commits at gmail.com  Sat Sep  3 13:22:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 10:22:05 -0700 (PDT)
Subject: [pypy-commit] cffi default: - add ffi.compile(debug=flag)
Message-ID: <57cb06bd.81091c0a.d9c91.1d6e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2753:ecd17895c8df
Date: 2016-09-03 19:21 +0200
http://bitbucket.org/cffi/cffi/changeset/ecd17895c8df/

Log:	- add ffi.compile(debug=flag)

	- this small addition, and a few others not in pypy 5.4, will make
	the next cffi release v1.8.1

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -646,7 +646,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -663,7 +663,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py
--- a/cffi/ffiplatform.py
+++ b/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -533,7 +533,7 @@
 that compiling with a debug version of Python will not actually define
 ``Py_LIMITED_API``, as doing so makes ``Python.h`` unhappy.
 
-**ffibuilder.compile(tmpdir='.', verbose=False):**
+**ffibuilder.compile(tmpdir='.', verbose=False, debug=None):**
 explicitly generate the .py or .c file,
 and (if .c) compile it.  The output file is (or are) put in the
 directory given by ``tmpdir``.  In the examples given here, we use
@@ -548,6 +548,13 @@
 compiler.  (This parameter might be changed to True by default in a
 future release.)
 
+*New in version 1.8.1:* ``debug`` argument.  If set to a bool, it
+controls whether the C code is compiled in debug mode or not.  The
+default None means to use the host Python's ``sys.flags.debug``.
+Starting with version 1.8.1, if you are running a debug-mode Python, the
+C code is thus compiled in debug mode by default (note that it is anyway
+necessary to do so on Windows).
+
 **ffibuilder.emit_python_code(filename):** generate the given .py file (same
 as ``ffibuilder.compile()`` for ABI mode, with an explicitly-named file to
 write).  If you choose, you can include this .py file pre-packaged in
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,8 +3,8 @@
 ======================
 
 
-v1.8
-====
+v1.8.1
+======
 
 * CPython 3.x: experimental: the generated C extension modules now use
   the "limited API", which means that, as a compiled .so/.dll, it should
@@ -13,6 +13,18 @@
   name, you can rename it manually to ``NAME.abi3.so``, or use the very
   recent setuptools 26.
 
+* Removed the ctypes backend.  If ``_cffi_backend`` was not compiled,
+  you could ask (using an undocumented interface) for ``backend_ctypes``
+  instead.  That was never fully functional and long deprecated.
+
+* Added ``ffi.compile(debug=...)``, similar to ``python setup.py build
+  --debug`` but defaulting to True if we are running a debugging
+  version of Python itself.
+
+
+v1.8
+====
+
 * Removed the restriction that ``ffi.from_buffer()`` cannot be used on
   byte strings.  Now you can get a ``char *`` out of a byte string,
   which is valid as long as the string object is kept alive.  (But
@@ -23,10 +35,6 @@
   argument (in older versions, a copy would be made).  This used to be
   a CPython-only optimization.
 
-* Removed the ctypes backend.  If ``_cffi_backend`` was not compiled,
-  you could ask (using an undocumented interface) for ``backend_ctypes``
-  instead.  That was never fully functional and long deprecated.
-
 
 v1.7
 ====

From pypy.commits at gmail.com  Sat Sep  3 13:26:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 10:26:09 -0700 (PDT)
Subject: [pypy-commit] cffi default: Bump version number to 1.8.1
Message-ID: <57cb07b1.8bd51c0a.5659f.03d6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2754:90e2e3ee4411
Date: 2016-09-03 19:25 +0200
http://bitbucket.org/cffi/cffi/changeset/90e2e3ee4411/

Log:	Bump version number to 1.8.1

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.8.0"
+#define CFFI_VERSION  "1.8.1"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.1"
+__version_info__ = (1, 8, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.8'
 # The full version, including alpha/beta/rc tags.
-release = '1.8.0'
+release = '1.8.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,7 +51,7 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.1.tar.gz
 
    - MD5: ...
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.8.0',
+        version='1.8.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Sat Sep  3 14:04:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 11:04:39 -0700 (PDT)
Subject: [pypy-commit] cffi default: Backed out changeset 0087e2aec9ef
Message-ID: <57cb10b7.88cb1c0a.267a7.25f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2755:e8e4775048a1
Date: 2016-09-03 19:29 +0200
http://bitbucket.org/cffi/cffi/changeset/e8e4775048a1/

Log:	Backed out changeset 0087e2aec9ef

	Un-kill the ctypes backend. Issue #282 for a justification.

diff too long, truncating to 2000 out of 5541 lines

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -46,21 +46,20 @@
     '''
 
     def __init__(self, backend=None):
-        """Create an FFI instance.
-
-        The 'backend' argument is not used any more and must be set to None.
-        It is still present only so that 'FFI(None)' still works, and
-        for a few tests.
+        """Create an FFI instance.  The 'backend' argument is used to
+        select a non-default backend, mostly for tests.
         """
         from . import cparser, model
-
         if backend is None:
-            # You need the corresponding version of PyPy, or CPython
-            # with the '_cffi_backend' C extension module compiled.
+            # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with
+            # _cffi_backend.so compiled.
             import _cffi_backend as backend
             from . import __version__
             assert backend.__version__ == __version__, \
                "version mismatch, %s != %s" % (backend.__version__, __version__)
+            # (If you insist you can also try to pass the option
+            # 'backend=backend_ctypes.CTypesBackend()', but don't
+            # rely on it!  It's probably not going to work well.)
 
         self._backend = backend
         self._lock = allocate_lock()
@@ -76,6 +75,8 @@
         self._init_once_cache = {}
         self._cdef_version = None
         self._embedding = None
+        if hasattr(backend, 'set_ffi'):
+            backend.set_ffi(self)
         for name in backend.__dict__:
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
@@ -83,10 +84,15 @@
         with self._lock:
             self.BVoidP = self._get_cached_btype(model.voidp_type)
             self.BCharA = self._get_cached_btype(model.char_array_type)
-        # attach these constants to the class
-        if not hasattr(FFI, 'NULL'):
-            FFI.NULL = self.cast(self.BVoidP, 0)
-            FFI.CData, FFI.CType = backend._get_types()
+        if isinstance(backend, types.ModuleType):
+            # _cffi_backend: attach these constants to the class
+            if not hasattr(FFI, 'NULL'):
+                FFI.NULL = self.cast(self.BVoidP, 0)
+                FFI.CData, FFI.CType = backend._get_types()
+        else:
+            # ctypes backend: attach these constants to the instance
+            self.NULL = self.cast(self.BVoidP, 0)
+            self.CData, self.CType = backend._get_types()
 
     def cdef(self, csource, override=False, packed=False):
         """Parse the given C source.  This registers all declared functions,
diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
new file mode 100644
--- /dev/null
+++ b/cffi/backend_ctypes.py
@@ -0,0 +1,1097 @@
+import ctypes, ctypes.util, operator, sys
+from . import model
+
+if sys.version_info < (3,):
+    bytechr = chr
+else:
+    unicode = str
+    long = int
+    xrange = range
+    bytechr = lambda num: bytes([num])
+
+class CTypesType(type):
+    pass
+
+class CTypesData(object):
+    __metaclass__ = CTypesType
+    __slots__ = ['__weakref__']
+    __name__ = '<cdata>'
+
+    def __init__(self, *args):
+        raise TypeError("cannot instantiate %r" % (self.__class__,))
+
+    @classmethod
+    def _newp(cls, init):
+        raise TypeError("expected a pointer or array ctype, got '%s'"
+                        % (cls._get_c_name(),))
+
+    @staticmethod
+    def _to_ctypes(value):
+        raise TypeError
+
+    @classmethod
+    def _arg_to_ctypes(cls, *value):
+        try:
+            ctype = cls._ctype
+        except AttributeError:
+            raise TypeError("cannot create an instance of %r" % (cls,))
+        if value:
+            res = cls._to_ctypes(*value)
+            if not isinstance(res, ctype):
+                res = cls._ctype(res)
+        else:
+            res = cls._ctype()
+        return res
+
+    @classmethod
+    def _create_ctype_obj(cls, init):
+        if init is None:
+            return cls._arg_to_ctypes()
+        else:
+            return cls._arg_to_ctypes(init)
+
+    @staticmethod
+    def _from_ctypes(ctypes_value):
+        raise TypeError
+
+    @classmethod
+    def _get_c_name(cls, replace_with=''):
+        return cls._reftypename.replace(' &', replace_with)
+
+    @classmethod
+    def _fix_class(cls):
+        cls.__name__ = 'CData<%s>' % (cls._get_c_name(),)
+        cls.__qualname__ = 'CData<%s>' % (cls._get_c_name(),)
+        cls.__module__ = 'ffi'
+
+    def _get_own_repr(self):
+        raise NotImplementedError
+
+    def _addr_repr(self, address):
+        if address == 0:
+            return 'NULL'
+        else:
+            if address < 0:
+                address += 1 << (8*ctypes.sizeof(ctypes.c_void_p))
+            return '0x%x' % address
+
+    def __repr__(self, c_name=None):
+        own = self._get_own_repr()
+        return '<cdata %r %s>' % (c_name or self._get_c_name(), own)
+
+    def _convert_to_address(self, BClass):
+        if BClass is None:
+            raise TypeError("cannot convert %r to an address" % (
+                self._get_c_name(),))
+        else:
+            raise TypeError("cannot convert %r to %r" % (
+                self._get_c_name(), BClass._get_c_name()))
+
+    @classmethod
+    def _get_size(cls):
+        return ctypes.sizeof(cls._ctype)
+
+    def _get_size_of_instance(self):
+        return ctypes.sizeof(self._ctype)
+
+    @classmethod
+    def _cast_from(cls, source):
+        raise TypeError("cannot cast to %r" % (cls._get_c_name(),))
+
+    def _cast_to_integer(self):
+        return self._convert_to_address(None)
+
+    @classmethod
+    def _alignment(cls):
+        return ctypes.alignment(cls._ctype)
+
+    def __iter__(self):
+        raise TypeError("cdata %r does not support iteration" % (
+            self._get_c_name()),)
+
+    def _make_cmp(name):
+        cmpfunc = getattr(operator, name)
+        def cmp(self, other):
+            if isinstance(other, CTypesData):
+                return cmpfunc(self._convert_to_address(None),
+                               other._convert_to_address(None))
+            else:
+                return NotImplemented
+        cmp.func_name = name
+        return cmp
+
+    __eq__ = _make_cmp('__eq__')
+    __ne__ = _make_cmp('__ne__')
+    __lt__ = _make_cmp('__lt__')
+    __le__ = _make_cmp('__le__')
+    __gt__ = _make_cmp('__gt__')
+    __ge__ = _make_cmp('__ge__')
+
+    def __hash__(self):
+        return hash(type(self)) ^ hash(self._convert_to_address(None))
+
+    def _to_string(self, maxlen):
+        raise TypeError("string(): %r" % (self,))
+
+
+class CTypesGenericPrimitive(CTypesData):
+    __slots__ = []
+
+    def __eq__(self, other):
+        return self is other
+
+    def __ne__(self, other):
+        return self is not other
+
+    def __hash__(self):
+        return object.__hash__(self)
+
+    def _get_own_repr(self):
+        return repr(self._from_ctypes(self._value))
+
+
+class CTypesGenericArray(CTypesData):
+    __slots__ = []
+
+    @classmethod
+    def _newp(cls, init):
+        return cls(init)
+
+    def __iter__(self):
+        for i in xrange(len(self)):
+            yield self[i]
+
+    def _get_own_repr(self):
+        return self._addr_repr(ctypes.addressof(self._blob))
+
+
+class CTypesGenericPtr(CTypesData):
+    __slots__ = ['_address', '_as_ctype_ptr']
+    _automatic_casts = False
+    kind = "pointer"
+
+    @classmethod
+    def _newp(cls, init):
+        return cls(init)
+
+    @classmethod
+    def _cast_from(cls, source):
+        if source is None:
+            address = 0
+        elif isinstance(source, CTypesData):
+            address = source._cast_to_integer()
+        elif isinstance(source, (int, long)):
+            address = source
+        else:
+            raise TypeError("bad type for cast to %r: %r" %
+                            (cls, type(source).__name__))
+        return cls._new_pointer_at(address)
+
+    @classmethod
+    def _new_pointer_at(cls, address):
+        self = cls.__new__(cls)
+        self._address = address
+        self._as_ctype_ptr = ctypes.cast(address, cls._ctype)
+        return self
+
+    def _get_own_repr(self):
+        try:
+            return self._addr_repr(self._address)
+        except AttributeError:
+            return '???'
+
+    def _cast_to_integer(self):
+        return self._address
+
+    def __nonzero__(self):
+        return bool(self._address)
+    __bool__ = __nonzero__
+
+    @classmethod
+    def _to_ctypes(cls, value):
+        if not isinstance(value, CTypesData):
+            raise TypeError("unexpected %s object" % type(value).__name__)
+        address = value._convert_to_address(cls)
+        return ctypes.cast(address, cls._ctype)
+
+    @classmethod
+    def _from_ctypes(cls, ctypes_ptr):
+        address = ctypes.cast(ctypes_ptr, ctypes.c_void_p).value or 0
+        return cls._new_pointer_at(address)
+
+    @classmethod
+    def _initialize(cls, ctypes_ptr, value):
+        if value:
+            ctypes_ptr.contents = cls._to_ctypes(value).contents
+
+    def _convert_to_address(self, BClass):
+        if (BClass in (self.__class__, None) or BClass._automatic_casts
+            or self._automatic_casts):
+            return self._address
+        else:
+            return CTypesData._convert_to_address(self, BClass)
+
+
+class CTypesBaseStructOrUnion(CTypesData):
+    __slots__ = ['_blob']
+
+    @classmethod
+    def _create_ctype_obj(cls, init):
+        # may be overridden
+        raise TypeError("cannot instantiate opaque type %s" % (cls,))
+
+    def _get_own_repr(self):
+        return self._addr_repr(ctypes.addressof(self._blob))
+
+    @classmethod
+    def _offsetof(cls, fieldname):
+        return getattr(cls._ctype, fieldname).offset
+
+    def _convert_to_address(self, BClass):
+        if getattr(BClass, '_BItem', None) is self.__class__:
+            return ctypes.addressof(self._blob)
+        else:
+            return CTypesData._convert_to_address(self, BClass)
+
+    @classmethod
+    def _from_ctypes(cls, ctypes_struct_or_union):
+        self = cls.__new__(cls)
+        self._blob = ctypes_struct_or_union
+        return self
+
+    @classmethod
+    def _to_ctypes(cls, value):
+        return value._blob
+
+    def __repr__(self, c_name=None):
+        return CTypesData.__repr__(self, c_name or self._get_c_name(' &'))
+
+
+class CTypesBackend(object):
+
+    PRIMITIVE_TYPES = {
+        'char': ctypes.c_char,
+        'short': ctypes.c_short,
+        'int': ctypes.c_int,
+        'long': ctypes.c_long,
+        'long long': ctypes.c_longlong,
+        'signed char': ctypes.c_byte,
+        'unsigned char': ctypes.c_ubyte,
+        'unsigned short': ctypes.c_ushort,
+        'unsigned int': ctypes.c_uint,
+        'unsigned long': ctypes.c_ulong,
+        'unsigned long long': ctypes.c_ulonglong,
+        'float': ctypes.c_float,
+        'double': ctypes.c_double,
+        '_Bool': ctypes.c_bool,
+        }
+
+    for _name in ['unsigned long long', 'unsigned long',
+                  'unsigned int', 'unsigned short', 'unsigned char']:
+        _size = ctypes.sizeof(PRIMITIVE_TYPES[_name])
+        PRIMITIVE_TYPES['uint%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name]
+        if _size == ctypes.sizeof(ctypes.c_void_p):
+            PRIMITIVE_TYPES['uintptr_t'] = PRIMITIVE_TYPES[_name]
+        if _size == ctypes.sizeof(ctypes.c_size_t):
+            PRIMITIVE_TYPES['size_t'] = PRIMITIVE_TYPES[_name]
+
+    for _name in ['long long', 'long', 'int', 'short', 'signed char']:
+        _size = ctypes.sizeof(PRIMITIVE_TYPES[_name])
+        PRIMITIVE_TYPES['int%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name]
+        if _size == ctypes.sizeof(ctypes.c_void_p):
+            PRIMITIVE_TYPES['intptr_t'] = PRIMITIVE_TYPES[_name]
+            PRIMITIVE_TYPES['ptrdiff_t'] = PRIMITIVE_TYPES[_name]
+        if _size == ctypes.sizeof(ctypes.c_size_t):
+            PRIMITIVE_TYPES['ssize_t'] = PRIMITIVE_TYPES[_name]
+
+
+    def __init__(self):
+        self.RTLD_LAZY = 0   # not supported anyway by ctypes
+        self.RTLD_NOW  = 0
+        self.RTLD_GLOBAL = ctypes.RTLD_GLOBAL
+        self.RTLD_LOCAL = ctypes.RTLD_LOCAL
+
+    def set_ffi(self, ffi):
+        self.ffi = ffi
+
+    def _get_types(self):
+        return CTypesData, CTypesType
+
+    def load_library(self, path, flags=0):
+        cdll = ctypes.CDLL(path, flags)
+        return CTypesLibrary(self, cdll)
+
+    def new_void_type(self):
+        class CTypesVoid(CTypesData):
+            __slots__ = []
+            _reftypename = 'void &'
+            @staticmethod
+            def _from_ctypes(novalue):
+                return None
+            @staticmethod
+            def _to_ctypes(novalue):
+                if novalue is not None:
+                    raise TypeError("None expected, got %s object" %
+                                    (type(novalue).__name__,))
+                return None
+        CTypesVoid._fix_class()
+        return CTypesVoid
+
+    def new_primitive_type(self, name):
+        if name == 'wchar_t':
+            raise NotImplementedError(name)
+        ctype = self.PRIMITIVE_TYPES[name]
+        if name == 'char':
+            kind = 'char'
+        elif name in ('float', 'double'):
+            kind = 'float'
+        else:
+            if name in ('signed char', 'unsigned char'):
+                kind = 'byte'
+            elif name == '_Bool':
+                kind = 'bool'
+            else:
+                kind = 'int'
+            is_signed = (ctype(-1).value == -1)
+        #
+        def _cast_source_to_int(source):
+            if isinstance(source, (int, long, float)):
+                source = int(source)
+            elif isinstance(source, CTypesData):
+                source = source._cast_to_integer()
+            elif isinstance(source, bytes):
+                source = ord(source)
+            elif source is None:
+                source = 0
+            else:
+                raise TypeError("bad type for cast to %r: %r" %
+                                (CTypesPrimitive, type(source).__name__))
+            return source
+        #
+        kind1 = kind
+        class CTypesPrimitive(CTypesGenericPrimitive):
+            __slots__ = ['_value']
+            _ctype = ctype
+            _reftypename = '%s &' % name
+            kind = kind1
+
+            def __init__(self, value):
+                self._value = value
+
+            @staticmethod
+            def _create_ctype_obj(init):
+                if init is None:
+                    return ctype()
+                return ctype(CTypesPrimitive._to_ctypes(init))
+
+            if kind == 'int' or kind == 'byte':
+                @classmethod
+                def _cast_from(cls, source):
+                    source = _cast_source_to_int(source)
+                    source = ctype(source).value     # cast within range
+                    return cls(source)
+                def __int__(self):
+                    return self._value
+
+            if kind == 'bool':
+                @classmethod
+                def _cast_from(cls, source):
+                    if not isinstance(source, (int, long, float)):
+                        source = _cast_source_to_int(source)
+                    return cls(bool(source))
+                def __int__(self):
+                    return self._value
+
+            if kind == 'char':
+                @classmethod
+                def _cast_from(cls, source):
+                    source = _cast_source_to_int(source)
+                    source = bytechr(source & 0xFF)
+                    return cls(source)
+                def __int__(self):
+                    return ord(self._value)
+
+            if kind == 'float':
+                @classmethod
+                def _cast_from(cls, source):
+                    if isinstance(source, float):
+                        pass
+                    elif isinstance(source, CTypesGenericPrimitive):
+                        if hasattr(source, '__float__'):
+                            source = float(source)
+                        else:
+                            source = int(source)
+                    else:
+                        source = _cast_source_to_int(source)
+                    source = ctype(source).value     # fix precision
+                    return cls(source)
+                def __int__(self):
+                    return int(self._value)
+                def __float__(self):
+                    return self._value
+
+            _cast_to_integer = __int__
+
+            if kind == 'int' or kind == 'byte' or kind == 'bool':
+                @staticmethod
+                def _to_ctypes(x):
+                    if not isinstance(x, (int, long)):
+                        if isinstance(x, CTypesData):
+                            x = int(x)
+                        else:
+                            raise TypeError("integer expected, got %s" %
+                                            type(x).__name__)
+                    if ctype(x).value != x:
+                        if not is_signed and x < 0:
+                            raise OverflowError("%s: negative integer" % name)
+                        else:
+                            raise OverflowError("%s: integer out of bounds"
+                                                % name)
+                    return x
+
+            if kind == 'char':
+                @staticmethod
+                def _to_ctypes(x):
+                    if isinstance(x, bytes) and len(x) == 1:
+                        return x
+                    if isinstance(x, CTypesPrimitive):    # <CData <char>>
+                        return x._value
+                    raise TypeError("character expected, got %s" %
+                                    type(x).__name__)
+                def __nonzero__(self):
+                    return ord(self._value) != 0
+            else:
+                def __nonzero__(self):
+                    return self._value != 0
+            __bool__ = __nonzero__
+
+            if kind == 'float':
+                @staticmethod
+                def _to_ctypes(x):
+                    if not isinstance(x, (int, long, float, CTypesData)):
+                        raise TypeError("float expected, got %s" %
+                                        type(x).__name__)
+                    return ctype(x).value
+
+            @staticmethod
+            def _from_ctypes(value):
+                return getattr(value, 'value', value)
+
+            @staticmethod
+            def _initialize(blob, init):
+                blob.value = CTypesPrimitive._to_ctypes(init)
+
+            if kind == 'char':
+                def _to_string(self, maxlen):
+                    return self._value
+            if kind == 'byte':
+                def _to_string(self, maxlen):
+                    return chr(self._value & 0xff)
+        #
+        CTypesPrimitive._fix_class()
+        return CTypesPrimitive
+
+    def new_pointer_type(self, BItem):
+        getbtype = self.ffi._get_cached_btype
+        if BItem is getbtype(model.PrimitiveType('char')):
+            kind = 'charp'
+        elif BItem in (getbtype(model.PrimitiveType('signed char')),
+                       getbtype(model.PrimitiveType('unsigned char'))):
+            kind = 'bytep'
+        elif BItem is getbtype(model.void_type):
+            kind = 'voidp'
+        else:
+            kind = 'generic'
+        #
+        class CTypesPtr(CTypesGenericPtr):
+            __slots__ = ['_own']
+            if kind == 'charp':
+                __slots__ += ['__as_strbuf']
+            _BItem = BItem
+            if hasattr(BItem, '_ctype'):
+                _ctype = ctypes.POINTER(BItem._ctype)
+                _bitem_size = ctypes.sizeof(BItem._ctype)
+            else:
+                _ctype = ctypes.c_void_p
+            if issubclass(BItem, CTypesGenericArray):
+                _reftypename = BItem._get_c_name('(* &)')
+            else:
+                _reftypename = BItem._get_c_name(' * &')
+
+            def __init__(self, init):
+                ctypeobj = BItem._create_ctype_obj(init)
+                if kind == 'charp':
+                    self.__as_strbuf = ctypes.create_string_buffer(
+                        ctypeobj.value + b'\x00')
+                    self._as_ctype_ptr = ctypes.cast(
+                        self.__as_strbuf, self._ctype)
+                else:
+                    self._as_ctype_ptr = ctypes.pointer(ctypeobj)
+                self._address = ctypes.cast(self._as_ctype_ptr,
+                                            ctypes.c_void_p).value
+                self._own = True
+
+            def __add__(self, other):
+                if isinstance(other, (int, long)):
+                    return self._new_pointer_at(self._address +
+                                                other * self._bitem_size)
+                else:
+                    return NotImplemented
+
+            def __sub__(self, other):
+                if isinstance(other, (int, long)):
+                    return self._new_pointer_at(self._address -
+                                                other * self._bitem_size)
+                elif type(self) is type(other):
+                    return (self._address - other._address) // self._bitem_size
+                else:
+                    return NotImplemented
+
+            def __getitem__(self, index):
+                if getattr(self, '_own', False) and index != 0:
+                    raise IndexError
+                return BItem._from_ctypes(self._as_ctype_ptr[index])
+
+            def __setitem__(self, index, value):
+                self._as_ctype_ptr[index] = BItem._to_ctypes(value)
+
+            if kind == 'charp' or kind == 'voidp':
+                @classmethod
+                def _arg_to_ctypes(cls, *value):
+                    if value and isinstance(value[0], bytes):
+                        return ctypes.c_char_p(value[0])
+                    else:
+                        return super(CTypesPtr, cls)._arg_to_ctypes(*value)
+
+            if kind == 'charp' or kind == 'bytep':
+                def _to_string(self, maxlen):
+                    if maxlen < 0:
+                        maxlen = sys.maxsize
+                    p = ctypes.cast(self._as_ctype_ptr,
+                                    ctypes.POINTER(ctypes.c_char))
+                    n = 0
+                    while n < maxlen and p[n] != b'\x00':
+                        n += 1
+                    return b''.join([p[i] for i in range(n)])
+
+            def _get_own_repr(self):
+                if getattr(self, '_own', False):
+                    return 'owning %d bytes' % (
+                        ctypes.sizeof(self._as_ctype_ptr.contents),)
+                return super(CTypesPtr, self)._get_own_repr()
+        #
+        if (BItem is self.ffi._get_cached_btype(model.void_type) or
+            BItem is self.ffi._get_cached_btype(model.PrimitiveType('char'))):
+            CTypesPtr._automatic_casts = True
+        #
+        CTypesPtr._fix_class()
+        return CTypesPtr
+
+    def new_array_type(self, CTypesPtr, length):
+        if length is None:
+            brackets = ' &[]'
+        else:
+            brackets = ' &[%d]' % length
+        BItem = CTypesPtr._BItem
+        getbtype = self.ffi._get_cached_btype
+        if BItem is getbtype(model.PrimitiveType('char')):
+            kind = 'char'
+        elif BItem in (getbtype(model.PrimitiveType('signed char')),
+                       getbtype(model.PrimitiveType('unsigned char'))):
+            kind = 'byte'
+        else:
+            kind = 'generic'
+        #
+        class CTypesArray(CTypesGenericArray):
+            __slots__ = ['_blob', '_own']
+            if length is not None:
+                _ctype = BItem._ctype * length
+            else:
+                __slots__.append('_ctype')
+            _reftypename = BItem._get_c_name(brackets)
+            _declared_length = length
+            _CTPtr = CTypesPtr
+
+            def __init__(self, init):
+                if length is None:
+                    if isinstance(init, (int, long)):
+                        len1 = init
+                        init = None
+                    elif kind == 'char' and isinstance(init, bytes):
+                        len1 = len(init) + 1    # extra null
+                    else:
+                        init = tuple(init)
+                        len1 = len(init)
+                    self._ctype = BItem._ctype * len1
+                self._blob = self._ctype()
+                self._own = True
+                if init is not None:
+                    self._initialize(self._blob, init)
+
+            @staticmethod
+            def _initialize(blob, init):
+                if isinstance(init, bytes):
+                    init = [init[i:i+1] for i in range(len(init))]
+                else:
+                    init = tuple(init)
+                if len(init) > len(blob):
+                    raise IndexError("too many initializers")
+                addr = ctypes.cast(blob, ctypes.c_void_p).value
+                PTR = ctypes.POINTER(BItem._ctype)
+                itemsize = ctypes.sizeof(BItem._ctype)
+                for i, value in enumerate(init):
+                    p = ctypes.cast(addr + i * itemsize, PTR)
+                    BItem._initialize(p.contents, value)
+
+            def __len__(self):
+                return len(self._blob)
+
+            def __getitem__(self, index):
+                if not (0 <= index < len(self._blob)):
+                    raise IndexError
+                return BItem._from_ctypes(self._blob[index])
+
+            def __setitem__(self, index, value):
+                if not (0 <= index < len(self._blob)):
+                    raise IndexError
+                self._blob[index] = BItem._to_ctypes(value)
+
+            if kind == 'char' or kind == 'byte':
+                def _to_string(self, maxlen):
+                    if maxlen < 0:
+                        maxlen = len(self._blob)
+                    p = ctypes.cast(self._blob,
+                                    ctypes.POINTER(ctypes.c_char))
+                    n = 0
+                    while n < maxlen and p[n] != b'\x00':
+                        n += 1
+                    return b''.join([p[i] for i in range(n)])
+
+            def _get_own_repr(self):
+                if getattr(self, '_own', False):
+                    return 'owning %d bytes' % (ctypes.sizeof(self._blob),)
+                return super(CTypesArray, self)._get_own_repr()
+
+            def _convert_to_address(self, BClass):
+                if BClass in (CTypesPtr, None) or BClass._automatic_casts:
+                    return ctypes.addressof(self._blob)
+                else:
+                    return CTypesData._convert_to_address(self, BClass)
+
+            @staticmethod
+            def _from_ctypes(ctypes_array):
+                self = CTypesArray.__new__(CTypesArray)
+                self._blob = ctypes_array
+                return self
+
+            @staticmethod
+            def _arg_to_ctypes(value):
+                return CTypesPtr._arg_to_ctypes(value)
+
+            def __add__(self, other):
+                if isinstance(other, (int, long)):
+                    return CTypesPtr._new_pointer_at(
+                        ctypes.addressof(self._blob) +
+                        other * ctypes.sizeof(BItem._ctype))
+                else:
+                    return NotImplemented
+
+            @classmethod
+            def _cast_from(cls, source):
+                raise NotImplementedError("casting to %r" % (
+                    cls._get_c_name(),))
+        #
+        CTypesArray._fix_class()
+        return CTypesArray
+
+    def _new_struct_or_union(self, kind, name, base_ctypes_class):
+        #
+        class struct_or_union(base_ctypes_class):
+            pass
+        struct_or_union.__name__ = '%s_%s' % (kind, name)
+        kind1 = kind
+        #
+        class CTypesStructOrUnion(CTypesBaseStructOrUnion):
+            __slots__ = ['_blob']
+            _ctype = struct_or_union
+            _reftypename = '%s &' % (name,)
+            _kind = kind = kind1
+        #
+        CTypesStructOrUnion._fix_class()
+        return CTypesStructOrUnion
+
+    def new_struct_type(self, name):
+        return self._new_struct_or_union('struct', name, ctypes.Structure)
+
+    def new_union_type(self, name):
+        return self._new_struct_or_union('union', name, ctypes.Union)
+
+    def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
+                                 totalsize=-1, totalalignment=-1, sflags=0):
+        if totalsize >= 0 or totalalignment >= 0:
+            raise NotImplementedError("the ctypes backend of CFFI does not support "
+                                      "structures completed by verify(); please "
+                                      "compile and install the _cffi_backend module.")
+        struct_or_union = CTypesStructOrUnion._ctype
+        fnames = [fname for (fname, BField, bitsize) in fields]
+        btypes = [BField for (fname, BField, bitsize) in fields]
+        bitfields = [bitsize for (fname, BField, bitsize) in fields]
+        #
+        bfield_types = {}
+        cfields = []
+        for (fname, BField, bitsize) in fields:
+            if bitsize < 0:
+                cfields.append((fname, BField._ctype))
+                bfield_types[fname] = BField
+            else:
+                cfields.append((fname, BField._ctype, bitsize))
+                bfield_types[fname] = Ellipsis
+        if sflags & 8:
+            struct_or_union._pack_ = 1
+        struct_or_union._fields_ = cfields
+        CTypesStructOrUnion._bfield_types = bfield_types
+        #
+        @staticmethod
+        def _create_ctype_obj(init):
+            result = struct_or_union()
+            if init is not None:
+                initialize(result, init)
+            return result
+        CTypesStructOrUnion._create_ctype_obj = _create_ctype_obj
+        #
+        def initialize(blob, init):
+            if is_union:
+                if len(init) > 1:
+                    raise ValueError("union initializer: %d items given, but "
+                                    "only one supported (use a dict if needed)"
+                                     % (len(init),))
+            if not isinstance(init, dict):
+                if isinstance(init, (bytes, unicode)):
+                    raise TypeError("union initializer: got a str")
+                init = tuple(init)
+                if len(init) > len(fnames):
+                    raise ValueError("too many values for %s initializer" %
+                                     CTypesStructOrUnion._get_c_name())
+                init = dict(zip(fnames, init))
+            addr = ctypes.addressof(blob)
+            for fname, value in init.items():
+                BField, bitsize = name2fieldtype[fname]
+                assert bitsize < 0, \
+                       "not implemented: initializer with bit fields"
+                offset = CTypesStructOrUnion._offsetof(fname)
+                PTR = ctypes.POINTER(BField._ctype)
+                p = ctypes.cast(addr + offset, PTR)
+                BField._initialize(p.contents, value)
+        is_union = CTypesStructOrUnion._kind == 'union'
+        name2fieldtype = dict(zip(fnames, zip(btypes, bitfields)))
+        #
+        for fname, BField, bitsize in fields:
+            if fname == '':
+                raise NotImplementedError("nested anonymous structs/unions")
+            if hasattr(CTypesStructOrUnion, fname):
+                raise ValueError("the field name %r conflicts in "
+                                 "the ctypes backend" % fname)
+            if bitsize < 0:
+                def getter(self, fname=fname, BField=BField,
+                           offset=CTypesStructOrUnion._offsetof(fname),
+                           PTR=ctypes.POINTER(BField._ctype)):
+                    addr = ctypes.addressof(self._blob)
+                    p = ctypes.cast(addr + offset, PTR)
+                    return BField._from_ctypes(p.contents)
+                def setter(self, value, fname=fname, BField=BField):
+                    setattr(self._blob, fname, BField._to_ctypes(value))
+                #
+                if issubclass(BField, CTypesGenericArray):
+                    setter = None
+                    if BField._declared_length == 0:
+                        def getter(self, fname=fname, BFieldPtr=BField._CTPtr,
+                                   offset=CTypesStructOrUnion._offsetof(fname),
+                                   PTR=ctypes.POINTER(BField._ctype)):
+                            addr = ctypes.addressof(self._blob)
+                            p = ctypes.cast(addr + offset, PTR)
+                            return BFieldPtr._from_ctypes(p)
+                #
+            else:
+                def getter(self, fname=fname, BField=BField):
+                    return BField._from_ctypes(getattr(self._blob, fname))
+                def setter(self, value, fname=fname, BField=BField):
+                    # xxx obscure workaround
+                    value = BField._to_ctypes(value)
+                    oldvalue = getattr(self._blob, fname)
+                    setattr(self._blob, fname, value)
+                    if value != getattr(self._blob, fname):
+                        setattr(self._blob, fname, oldvalue)
+                        raise OverflowError("value too large for bitfield")
+            setattr(CTypesStructOrUnion, fname, property(getter, setter))
+        #
+        CTypesPtr = self.ffi._get_cached_btype(model.PointerType(tp))
+        for fname in fnames:
+            if hasattr(CTypesPtr, fname):
+                raise ValueError("the field name %r conflicts in "
+                                 "the ctypes backend" % fname)
+            def getter(self, fname=fname):
+                return getattr(self[0], fname)
+            def setter(self, value, fname=fname):
+                setattr(self[0], fname, value)
+            setattr(CTypesPtr, fname, property(getter, setter))
+
+    def new_function_type(self, BArgs, BResult, has_varargs):
+        nameargs = [BArg._get_c_name() for BArg in BArgs]
+        if has_varargs:
+            nameargs.append('...')
+        nameargs = ', '.join(nameargs)
+        #
+        class CTypesFunctionPtr(CTypesGenericPtr):
+            __slots__ = ['_own_callback', '_name']
+            _ctype = ctypes.CFUNCTYPE(getattr(BResult, '_ctype', None),
+                                      *[BArg._ctype for BArg in BArgs],
+                                      use_errno=True)
+            _reftypename = BResult._get_c_name('(* &)(%s)' % (nameargs,))
+
+            def __init__(self, init, error=None):
+                # create a callback to the Python callable init()
+                import traceback
+                assert not has_varargs, "varargs not supported for callbacks"
+                if getattr(BResult, '_ctype', None) is not None:
+                    error = BResult._from_ctypes(
+                        BResult._create_ctype_obj(error))
+                else:
+                    error = None
+                def callback(*args):
+                    args2 = []
+                    for arg, BArg in zip(args, BArgs):
+                        args2.append(BArg._from_ctypes(arg))
+                    try:
+                        res2 = init(*args2)
+                        res2 = BResult._to_ctypes(res2)
+                    except:
+                        traceback.print_exc()
+                        res2 = error
+                    if issubclass(BResult, CTypesGenericPtr):
+                        if res2:
+                            res2 = ctypes.cast(res2, ctypes.c_void_p).value
+                                # .value: http://bugs.python.org/issue1574593
+                        else:
+                            res2 = None
+                    #print repr(res2)
+                    return res2
+                if issubclass(BResult, CTypesGenericPtr):
+                    # The only pointers callbacks can return are void*s:
+                    # http://bugs.python.org/issue5710
+                    callback_ctype = ctypes.CFUNCTYPE(
+                        ctypes.c_void_p,
+                        *[BArg._ctype for BArg in BArgs],
+                        use_errno=True)
+                else:
+                    callback_ctype = CTypesFunctionPtr._ctype
+                self._as_ctype_ptr = callback_ctype(callback)
+                self._address = ctypes.cast(self._as_ctype_ptr,
+                                            ctypes.c_void_p).value
+                self._own_callback = init
+
+            @staticmethod
+            def _initialize(ctypes_ptr, value):
+                if value:
+                    raise NotImplementedError("ctypes backend: not supported: "
+                                          "initializers for function pointers")
+
+            def __repr__(self):
+                c_name = getattr(self, '_name', None)
+                if c_name:
+                    i = self._reftypename.index('(* &)')
+                    if self._reftypename[i-1] not in ' )*':
+                        c_name = ' ' + c_name
+                    c_name = self._reftypename.replace('(* &)', c_name)
+                return CTypesData.__repr__(self, c_name)
+
+            def _get_own_repr(self):
+                if getattr(self, '_own_callback', None) is not None:
+                    return 'calling %r' % (self._own_callback,)
+                return super(CTypesFunctionPtr, self)._get_own_repr()
+
+            def __call__(self, *args):
+                if has_varargs:
+                    assert len(args) >= len(BArgs)
+                    extraargs = args[len(BArgs):]
+                    args = args[:len(BArgs)]
+                else:
+                    assert len(args) == len(BArgs)
+                ctypes_args = []
+                for arg, BArg in zip(args, BArgs):
+                    ctypes_args.append(BArg._arg_to_ctypes(arg))
+                if has_varargs:
+                    for i, arg in enumerate(extraargs):
+                        if arg is None:
+                            ctypes_args.append(ctypes.c_void_p(0))  # NULL
+                            continue
+                        if not isinstance(arg, CTypesData):
+                            raise TypeError(
+                                "argument %d passed in the variadic part "
+                                "needs to be a cdata object (got %s)" %
+                                (1 + len(BArgs) + i, type(arg).__name__))
+                        ctypes_args.append(arg._arg_to_ctypes(arg))
+                result = self._as_ctype_ptr(*ctypes_args)
+                return BResult._from_ctypes(result)
+        #
+        CTypesFunctionPtr._fix_class()
+        return CTypesFunctionPtr
+
+    def new_enum_type(self, name, enumerators, enumvalues, CTypesInt):
+        assert isinstance(name, str)
+        reverse_mapping = dict(zip(reversed(enumvalues),
+                                   reversed(enumerators)))
+        #
+        class CTypesEnum(CTypesInt):
+            __slots__ = []
+            _reftypename = '%s &' % name
+
+            def _get_own_repr(self):
+                value = self._value
+                try:
+                    return '%d: %s' % (value, reverse_mapping[value])
+                except KeyError:
+                    return str(value)
+
+            def _to_string(self, maxlen):
+                value = self._value
+                try:
+                    return reverse_mapping[value]
+                except KeyError:
+                    return str(value)
+        #
+        CTypesEnum._fix_class()
+        return CTypesEnum
+
+    def get_errno(self):
+        return ctypes.get_errno()
+
+    def set_errno(self, value):
+        ctypes.set_errno(value)
+
+    def string(self, b, maxlen=-1):
+        return b._to_string(maxlen)
+
+    def buffer(self, bptr, size=-1):
+        raise NotImplementedError("buffer() with ctypes backend")
+
+    def sizeof(self, cdata_or_BType):
+        if isinstance(cdata_or_BType, CTypesData):
+            return cdata_or_BType._get_size_of_instance()
+        else:
+            assert issubclass(cdata_or_BType, CTypesData)
+            return cdata_or_BType._get_size()
+
+    def alignof(self, BType):
+        assert issubclass(BType, CTypesData)
+        return BType._alignment()
+
+    def newp(self, BType, source):
+        if not issubclass(BType, CTypesData):
+            raise TypeError
+        return BType._newp(source)
+
+    def cast(self, BType, source):
+        return BType._cast_from(source)
+
+    def callback(self, BType, source, error, onerror):
+        assert onerror is None   # XXX not implemented
+        return BType(source, error)
+
+    def gcp(self, cdata, destructor):
+        BType = self.typeof(cdata)
+
+        if destructor is None:
+            if not (hasattr(BType, '_gcp_type') and
+                    BType._gcp_type is BType):
+                raise TypeError("Can remove destructor only on a object "
+                                "previously returned by ffi.gc()")
+            cdata._destructor = None
+            return None
+
+        try:
+            gcp_type = BType._gcp_type
+        except AttributeError:
+            class CTypesDataGcp(BType):
+                __slots__ = ['_orig', '_destructor']
+                def __del__(self):
+                    if self._destructor is not None:
+                        self._destructor(self._orig)
+            gcp_type = BType._gcp_type = CTypesDataGcp
+        new_cdata = self.cast(gcp_type, cdata)
+        new_cdata._orig = cdata
+        new_cdata._destructor = destructor
+        return new_cdata
+
+    typeof = type
+
+    def getcname(self, BType, replace_with):
+        return BType._get_c_name(replace_with)
+
+    def typeoffsetof(self, BType, fieldname, num=0):
+        if isinstance(fieldname, str):
+            if num == 0 and issubclass(BType, CTypesGenericPtr):
+                BType = BType._BItem
+            if not issubclass(BType, CTypesBaseStructOrUnion):
+                raise TypeError("expected a struct or union ctype")
+            BField = BType._bfield_types[fieldname]
+            if BField is Ellipsis:
+                raise TypeError("not supported for bitfields")
+            return (BField, BType._offsetof(fieldname))
+        elif isinstance(fieldname, (int, long)):
+            if issubclass(BType, CTypesGenericArray):
+                BType = BType._CTPtr
+            if not issubclass(BType, CTypesGenericPtr):
+                raise TypeError("expected an array or ptr ctype")
+            BItem = BType._BItem
+            offset = BItem._get_size() * fieldname
+            if offset > sys.maxsize:
+                raise OverflowError
+            return (BItem, offset)
+        else:
+            raise TypeError(type(fieldname))
+
+    def rawaddressof(self, BTypePtr, cdata, offset=None):
+        if isinstance(cdata, CTypesBaseStructOrUnion):
+            ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata))
+        elif isinstance(cdata, CTypesGenericPtr):
+            if offset is None or not issubclass(type(cdata)._BItem,
+                                                CTypesBaseStructOrUnion):
+                raise TypeError("unexpected cdata type")
+            ptr = type(cdata)._to_ctypes(cdata)
+        elif isinstance(cdata, CTypesGenericArray):
+            ptr = type(cdata)._to_ctypes(cdata)
+        else:
+            raise TypeError("expected a <cdata 'struct-or-union'>")
+        if offset:
+            ptr = ctypes.cast(
+                ctypes.c_void_p(
+                    ctypes.cast(ptr, ctypes.c_void_p).value + offset),
+                type(ptr))
+        return BTypePtr._from_ctypes(ptr)
+
+
+class CTypesLibrary(object):
+
+    def __init__(self, backend, cdll):
+        self.backend = backend
+        self.cdll = cdll
+
+    def load_function(self, BType, name):
+        c_func = getattr(self.cdll, name)
+        funcobj = BType._from_ctypes(c_func)
+        funcobj._name = name
+        return funcobj
+
+    def read_variable(self, BType, name):
+        try:
+            ctypes_obj = BType._ctype.in_dll(self.cdll, name)
+        except AttributeError as e:
+            raise NotImplementedError(e)
+        return BType._from_ctypes(ctypes_obj)
+
+    def write_variable(self, BType, name, value):
+        new_ctypes_obj = BType._to_ctypes(value)
+        ctypes_obj = BType._ctype.in_dll(self.cdll, name)
+        ctypes.memmove(ctypes.addressof(ctypes_obj),
+                       ctypes.addressof(new_ctypes_obj),
+                       ctypes.sizeof(BType._ctype))
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
new file mode 100644
--- /dev/null
+++ b/testing/cffi0/backend_tests.py
@@ -0,0 +1,1868 @@
+import py
+import platform
+import sys, ctypes
+from cffi import FFI, CDefError, FFIError, VerificationMissing
+from testing.support import *
+
+SIZE_OF_INT   = ctypes.sizeof(ctypes.c_int)
+SIZE_OF_LONG  = ctypes.sizeof(ctypes.c_long)
+SIZE_OF_SHORT = ctypes.sizeof(ctypes.c_short)
+SIZE_OF_PTR   = ctypes.sizeof(ctypes.c_void_p)
+SIZE_OF_WCHAR = ctypes.sizeof(ctypes.c_wchar)
+
+
+class BackendTests:
+
+    def test_integer_ranges(self):
+        ffi = FFI(backend=self.Backend())
+        for (c_type, size) in [('char', 1),
+                               ('short', 2),
+                               ('short int', 2),
+                               ('', 4),
+                               ('int', 4),
+                               ('long', SIZE_OF_LONG),
+                               ('long int', SIZE_OF_LONG),
+                               ('long long', 8),
+                               ('long long int', 8),
+                               ]:
+            for unsigned in [None, False, True]:
+                c_decl = {None: '',
+                          False: 'signed ',
+                          True: 'unsigned '}[unsigned] + c_type
+                if c_decl == 'char' or c_decl == '':
+                    continue
+                self._test_int_type(ffi, c_decl, size, unsigned)
+
+    def test_fixedsize_int(self):
+        ffi = FFI(backend=self.Backend())
+        for size in [1, 2, 4, 8]:
+            self._test_int_type(ffi, 'int%d_t' % (8*size), size, False)
+            self._test_int_type(ffi, 'uint%d_t' % (8*size), size, True)
+        self._test_int_type(ffi, 'intptr_t', SIZE_OF_PTR, False)
+        self._test_int_type(ffi, 'uintptr_t', SIZE_OF_PTR, True)
+        self._test_int_type(ffi, 'ptrdiff_t', SIZE_OF_PTR, False)
+        self._test_int_type(ffi, 'size_t', SIZE_OF_PTR, True)
+        self._test_int_type(ffi, 'ssize_t', SIZE_OF_PTR, False)
+
+    def _test_int_type(self, ffi, c_decl, size, unsigned):
+        if unsigned:
+            min = 0
+            max = (1 << (8*size)) - 1
+        else:
+            min = -(1 << (8*size-1))
+            max = (1 << (8*size-1)) - 1
+        min = int(min)
+        max = int(max)
+        p = ffi.cast(c_decl, min)
+        assert p != min       # no __eq__(int)
+        assert bool(p) is bool(min)
+        assert int(p) == min
+        p = ffi.cast(c_decl, max)
+        assert int(p) == max
+        p = ffi.cast(c_decl, long(max))
+        assert int(p) == max
+        q = ffi.cast(c_decl, min - 1)
+        assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
+        q = ffi.cast(c_decl, long(min - 1))
+        assert ffi.typeof(q) is ffi.typeof(p) and int(q) == max
+        assert q != p
+        assert int(q) == int(p)
+        assert hash(q) != hash(p)   # unlikely
+        c_decl_ptr = '%s *' % c_decl
+        py.test.raises(OverflowError, ffi.new, c_decl_ptr, min - 1)
+        py.test.raises(OverflowError, ffi.new, c_decl_ptr, max + 1)
+        py.test.raises(OverflowError, ffi.new, c_decl_ptr, long(min - 1))
+        py.test.raises(OverflowError, ffi.new, c_decl_ptr, long(max + 1))
+        assert ffi.new(c_decl_ptr, min)[0] == min
+        assert ffi.new(c_decl_ptr, max)[0] == max
+        assert ffi.new(c_decl_ptr, long(min))[0] == min
+        assert ffi.new(c_decl_ptr, long(max))[0] == max
+
+    def test_new_unsupported_type(self):
+        ffi = FFI(backend=self.Backend())
+        e = py.test.raises(TypeError, ffi.new, "int")
+        assert str(e.value) == "expected a pointer or array ctype, got 'int'"
+
+    def test_new_single_integer(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int *")     # similar to ffi.new("int[1]")
+        assert p[0] == 0
+        p[0] = -123
+        assert p[0] == -123
+        p = ffi.new("int *", -42)
+        assert p[0] == -42
+        assert repr(p) == "<cdata 'int *' owning %d bytes>" % SIZE_OF_INT
+
+    def test_new_array_no_arg(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[10]")
+        # the object was zero-initialized:
+        for i in range(10):
+            assert p[i] == 0
+
+    def test_array_indexing(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[10]")
+        p[0] = 42
+        p[9] = 43
+        assert p[0] == 42
+        assert p[9] == 43
+        py.test.raises(IndexError, "p[10]")
+        py.test.raises(IndexError, "p[10] = 44")
+        py.test.raises(IndexError, "p[-1]")
+        py.test.raises(IndexError, "p[-1] = 44")
+
+    def test_new_array_args(self):
+        ffi = FFI(backend=self.Backend())
+        # this tries to be closer to C: where we say "int x[5] = {10, 20, ..}"
+        # then here we must enclose the items in a list
+        p = ffi.new("int[5]", [10, 20, 30, 40, 50])
+        assert p[0] == 10
+        assert p[1] == 20
+        assert p[2] == 30
+        assert p[3] == 40
+        assert p[4] == 50
+        p = ffi.new("int[4]", [25])
+        assert p[0] == 25
+        assert p[1] == 0     # follow C convention rather than LuaJIT's
+        assert p[2] == 0
+        assert p[3] == 0
+        p = ffi.new("int[4]", [ffi.cast("int", -5)])
+        assert p[0] == -5
+        assert repr(p) == "<cdata 'int[4]' owning %d bytes>" % (4*SIZE_OF_INT)
+
+    def test_new_array_varsize(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[]", 10)     # a single integer is the length
+        assert p[9] == 0
+        py.test.raises(IndexError, "p[10]")
+        #
+        py.test.raises(TypeError, ffi.new, "int[]")
+        #
+        p = ffi.new("int[]", [-6, -7])    # a list is all the items, like C
+        assert p[0] == -6
+        assert p[1] == -7
+        py.test.raises(IndexError, "p[2]")
+        assert repr(p) == "<cdata 'int[]' owning %d bytes>" % (2*SIZE_OF_INT)
+        #
+        p = ffi.new("int[]", 0)
+        py.test.raises(IndexError, "p[0]")
+        py.test.raises(ValueError, ffi.new, "int[]", -1)
+        assert repr(p) == "<cdata 'int[]' owning 0 bytes>"
+
+    def test_pointer_init(self):
+        ffi = FFI(backend=self.Backend())
+        n = ffi.new("int *", 24)
+        a = ffi.new("int *[10]", [ffi.NULL, ffi.NULL, n, n, ffi.NULL])
+        for i in range(10):
+            if i not in (2, 3):
+                assert a[i] == ffi.NULL
+        assert a[2] == a[3] == n
+
+    def test_cannot_cast(self):
+        ffi = FFI(backend=self.Backend())
+        a = ffi.new("short int[10]")
+        e = py.test.raises(TypeError, ffi.new, "long int **", a)
+        msg = str(e.value)
+        assert "'short[10]'" in msg and "'long *'" in msg
+
+    def test_new_pointer_to_array(self):
+        ffi = FFI(backend=self.Backend())
+        a = ffi.new("int[4]", [100, 102, 104, 106])
+        p = ffi.new("int **", a)
+        assert p[0] == ffi.cast("int *", a)
+        assert p[0][2] == 104
+        p = ffi.cast("int *", a)
+        assert p[0] == 100
+        assert p[1] == 102
+        assert p[2] == 104
+        assert p[3] == 106
+        # keepalive: a
+
+    def test_pointer_direct(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.cast("int*", 0)
+        assert p is not None
+        assert bool(p) is False
+        assert p == ffi.cast("int*", 0)
+        assert p != None
+        assert repr(p) == "<cdata 'int *' NULL>"
+        a = ffi.new("int[]", [123, 456])
+        p = ffi.cast("int*", a)
+        assert bool(p) is True
+        assert p == ffi.cast("int*", a)
+        assert p != ffi.cast("int*", 0)
+        assert p[0] == 123
+        assert p[1] == 456
+
+    def test_repr(self):
+        typerepr = self.TypeRepr
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { short a, b, c; };")
+        p = ffi.cast("short unsigned int", 0)
+        assert repr(p) == "<cdata 'unsigned short' 0>"
+        assert repr(ffi.typeof(p)) == typerepr % "unsigned short"
+        p = ffi.cast("unsigned short int", 0)
+        assert repr(p) == "<cdata 'unsigned short' 0>"
+        assert repr(ffi.typeof(p)) == typerepr % "unsigned short"
+        p = ffi.cast("int*", 0)
+        assert repr(p) == "<cdata 'int *' NULL>"
+        assert repr(ffi.typeof(p)) == typerepr % "int *"
+        #
+        p = ffi.new("int*")
+        assert repr(p) == "<cdata 'int *' owning %d bytes>" % SIZE_OF_INT
+        assert repr(ffi.typeof(p)) == typerepr % "int *"
+        p = ffi.new("int**")
+        assert repr(p) == "<cdata 'int * *' owning %d bytes>" % SIZE_OF_PTR
+        assert repr(ffi.typeof(p)) == typerepr % "int * *"
+        p = ffi.new("int [2]")
+        assert repr(p) == "<cdata 'int[2]' owning %d bytes>" % (2*SIZE_OF_INT)
+        assert repr(ffi.typeof(p)) == typerepr % "int[2]"
+        p = ffi.new("int*[2][3]")
+        assert repr(p) == "<cdata 'int *[2][3]' owning %d bytes>" % (
+            6*SIZE_OF_PTR)
+        assert repr(ffi.typeof(p)) == typerepr % "int *[2][3]"
+        p = ffi.new("struct foo *")
+        assert repr(p) == "<cdata 'struct foo *' owning %d bytes>" % (
+            3*SIZE_OF_SHORT)
+        assert repr(ffi.typeof(p)) == typerepr % "struct foo *"
+        #
+        q = ffi.cast("short", -123)
+        assert repr(q) == "<cdata 'short' -123>"
+        assert repr(ffi.typeof(q)) == typerepr % "short"
+        p = ffi.new("int*")
+        q = ffi.cast("short*", p)
+        assert repr(q).startswith("<cdata 'short *' 0x")
+        assert repr(ffi.typeof(q)) == typerepr % "short *"
+        p = ffi.new("int [2]")
+        q = ffi.cast("int*", p)
+        assert repr(q).startswith("<cdata 'int *' 0x")
+        assert repr(ffi.typeof(q)) == typerepr % "int *"
+        p = ffi.new("struct foo*")
+        q = ffi.cast("struct foo *", p)
+        assert repr(q).startswith("<cdata 'struct foo *' 0x")
+        assert repr(ffi.typeof(q)) == typerepr % "struct foo *"
+        prevrepr = repr(q)
+        q = q[0]
+        assert repr(q) == prevrepr.replace(' *', ' &')
+        assert repr(ffi.typeof(q)) == typerepr % "struct foo"
+
+    def test_new_array_of_array(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[3][4]")
+        p[0][0] = 10
+        p[2][3] = 33
+        assert p[0][0] == 10
+        assert p[2][3] == 33
+        py.test.raises(IndexError, "p[1][-1]")
+
+    def test_constructor_array_of_array(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[3][2]", [[10, 11], [12, 13], [14, 15]])
+        assert p[2][1] == 15
+
+    def test_new_array_of_pointer_1(self):
+        ffi = FFI(backend=self.Backend())
+        n = ffi.new("int*", 99)
+        p = ffi.new("int*[4]")
+        p[3] = n
+        a = p[3]
+        assert repr(a).startswith("<cdata 'int *' 0x")
+        assert a[0] == 99
+
+    def test_new_array_of_pointer_2(self):
+        ffi = FFI(backend=self.Backend())
+        n = ffi.new("int[1]", [99])
+        p = ffi.new("int*[4]")
+        p[3] = n
+        a = p[3]
+        assert repr(a).startswith("<cdata 'int *' 0x")
+        assert a[0] == 99
+
+    def test_char(self):
+        ffi = FFI(backend=self.Backend())
+        assert ffi.new("char*", b"\xff")[0] == b'\xff'
+        assert ffi.new("char*")[0] == b'\x00'
+        assert int(ffi.cast("char", 300)) == 300 - 256
+        assert not bool(ffi.cast("char", 0))
+        assert bool(ffi.cast("char", 1))
+        assert bool(ffi.cast("char", 255))
+        py.test.raises(TypeError, ffi.new, "char*", 32)
+        py.test.raises(TypeError, ffi.new, "char*", u+"x")
+        py.test.raises(TypeError, ffi.new, "char*", b"foo")
+        #
+        p = ffi.new("char[]", [b'a', b'b', b'\x9c'])
+        assert len(p) == 3
+        assert p[0] == b'a'
+        assert p[1] == b'b'
+        assert p[2] == b'\x9c'
+        p[0] = b'\xff'
+        assert p[0] == b'\xff'
+        p = ffi.new("char[]", b"abcd")
+        assert len(p) == 5
+        assert p[4] == b'\x00'    # like in C, with:  char[] p = "abcd";
+        #
+        p = ffi.new("char[4]", b"ab")
+        assert len(p) == 4
+        assert [p[i] for i in range(4)] == [b'a', b'b', b'\x00', b'\x00']
+        p = ffi.new("char[2]", b"ab")
+        assert len(p) == 2
+        assert [p[i] for i in range(2)] == [b'a', b'b']
+        py.test.raises(IndexError, ffi.new, "char[2]", b"abc")
+
+    def check_wchar_t(self, ffi):
+        try:
+            ffi.cast("wchar_t", 0)
+        except NotImplementedError:
+            py.test.skip("NotImplementedError: wchar_t")
+
+    def test_wchar_t(self):
+        ffi = FFI(backend=self.Backend())
+        self.check_wchar_t(ffi)
+        assert ffi.new("wchar_t*", u+'x')[0] == u+'x'
+        assert ffi.new("wchar_t*", u+'\u1234')[0] == u+'\u1234'
+        if SIZE_OF_WCHAR > 2:
+            assert ffi.new("wchar_t*", u+'\U00012345')[0] == u+'\U00012345'
+        else:
+            py.test.raises(TypeError, ffi.new, "wchar_t*", u+'\U00012345')
+        assert ffi.new("wchar_t*")[0] == u+'\x00'
+        assert int(ffi.cast("wchar_t", 300)) == 300
+        assert not bool(ffi.cast("wchar_t", 0))
+        assert bool(ffi.cast("wchar_t", 1))
+        assert bool(ffi.cast("wchar_t", 65535))
+        if SIZE_OF_WCHAR > 2:
+            assert bool(ffi.cast("wchar_t", 65536))
+        py.test.raises(TypeError, ffi.new, "wchar_t*", 32)
+        py.test.raises(TypeError, ffi.new, "wchar_t*", "foo")
+        #
+        p = ffi.new("wchar_t[]", [u+'a', u+'b', u+'\u1234'])
+        assert len(p) == 3
+        assert p[0] == u+'a'
+        assert p[1] == u+'b' and type(p[1]) is unicode
+        assert p[2] == u+'\u1234'
+        p[0] = u+'x'
+        assert p[0] == u+'x' and type(p[0]) is unicode
+        p[1] = u+'\u1357'
+        assert p[1] == u+'\u1357'
+        p = ffi.new("wchar_t[]", u+"abcd")
+        assert len(p) == 5
+        assert p[4] == u+'\x00'
+        p = ffi.new("wchar_t[]", u+"a\u1234b")
+        assert len(p) == 4
+        assert p[1] == u+'\u1234'
+        #
+        p = ffi.new("wchar_t[]", u+'\U00023456')
+        if SIZE_OF_WCHAR == 2:
+            assert sys.maxunicode == 0xffff
+            assert len(p) == 3
+            assert p[0] == u+'\ud84d'
+            assert p[1] == u+'\udc56'
+            assert p[2] == u+'\x00'
+        else:
+            assert len(p) == 2
+            assert p[0] == u+'\U00023456'
+            assert p[1] == u+'\x00'
+        #
+        p = ffi.new("wchar_t[4]", u+"ab")
+        assert len(p) == 4
+        assert [p[i] for i in range(4)] == [u+'a', u+'b', u+'\x00', u+'\x00']
+        p = ffi.new("wchar_t[2]", u+"ab")
+        assert len(p) == 2
+        assert [p[i] for i in range(2)] == [u+'a', u+'b']
+        py.test.raises(IndexError, ffi.new, "wchar_t[2]", u+"abc")
+
+    def test_none_as_null_doesnt_work(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int*[1]")
+        assert p[0] is not None
+        assert p[0] != None
+        assert p[0] == ffi.NULL
+        assert repr(p[0]) == "<cdata 'int *' NULL>"
+        #
+        n = ffi.new("int*", 99)
+        p = ffi.new("int*[]", [n])
+        assert p[0][0] == 99
+        py.test.raises(TypeError, "p[0] = None")
+        p[0] = ffi.NULL
+        assert p[0] == ffi.NULL
+
+    def test_float(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("float[]", [-2, -2.5])
+        assert p[0] == -2.0
+        assert p[1] == -2.5
+        p[1] += 17.75
+        assert p[1] == 15.25
+        #
+        p = ffi.new("float*", 15.75)
+        assert p[0] == 15.75
+        py.test.raises(TypeError, int, p)
+        py.test.raises(TypeError, float, p)
+        p[0] = 0.0
+        assert bool(p) is True
+        #
+        p = ffi.new("float*", 1.1)
+        f = p[0]
+        assert f != 1.1      # because of rounding effect
+        assert abs(f - 1.1) < 1E-7
+        #
+        INF = 1E200 * 1E200
+        assert 1E200 != INF
+        p[0] = 1E200
+        assert p[0] == INF     # infinite, not enough precision
+
+    def test_struct_simple(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { int a; short b, c; };")
+        s = ffi.new("struct foo*")
+        assert s.a == s.b == s.c == 0
+        s.b = -23
+        assert s.b == -23
+        py.test.raises(OverflowError, "s.b = 32768")
+        #
+        s = ffi.new("struct foo*", [-2, -3])
+        assert s.a == -2
+        assert s.b == -3
+        assert s.c == 0
+        py.test.raises((AttributeError, TypeError), "del s.a")
+        assert repr(s) == "<cdata 'struct foo *' owning %d bytes>" % (
+            SIZE_OF_INT + 2 * SIZE_OF_SHORT)
+        #
+        py.test.raises(ValueError, ffi.new, "struct foo*", [1, 2, 3, 4])
+
+    def test_constructor_struct_from_dict(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { int a; short b, c; };")
+        s = ffi.new("struct foo*", {'b': 123, 'c': 456})
+        assert s.a == 0
+        assert s.b == 123
+        assert s.c == 456
+        py.test.raises(KeyError, ffi.new, "struct foo*", {'d': 456})
+
+    def test_struct_pointer(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { int a; short b, c; };")
+        s = ffi.new("struct foo*")
+        assert s[0].a == s[0].b == s[0].c == 0
+        s[0].b = -23
+        assert s[0].b == s.b == -23
+        py.test.raises(OverflowError, "s[0].b = -32769")
+        py.test.raises(IndexError, "s[1]")
+
+    def test_struct_opaque(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(TypeError, ffi.new, "struct baz*")
+        p = ffi.new("struct baz **")    # this works
+        assert p[0] == ffi.NULL
+
+    def test_pointer_to_struct(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { int a; short b, c; };")
+        s = ffi.new("struct foo *")
+        s.a = -42
+        assert s[0].a == -42
+        p = ffi.new("struct foo **", s)
+        assert p[0].a == -42
+        assert p[0][0].a == -42
+        p[0].a = -43
+        assert s.a == -43
+        assert s[0].a == -43
+        p[0][0].a = -44
+        assert s.a == -44
+        assert s[0].a == -44
+        s.a = -45
+        assert p[0].a == -45
+        assert p[0][0].a == -45
+        s[0].a = -46
+        assert p[0].a == -46
+        assert p[0][0].a == -46
+
+    def test_constructor_struct_of_array(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { int a[2]; char b[3]; };")
+        s = ffi.new("struct foo *", [[10, 11], [b'a', b'b', b'c']])
+        assert s.a[1] == 11
+        assert s.b[2] == b'c'
+        s.b[1] = b'X'
+        assert s.b[0] == b'a'
+        assert s.b[1] == b'X'
+        assert s.b[2] == b'c'
+
+    def test_recursive_struct(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { int value; struct foo *next; };")
+        s = ffi.new("struct foo*")
+        t = ffi.new("struct foo*")
+        s.value = 123
+        s.next = t
+        t.value = 456
+        assert s.value == 123
+        assert s.next.value == 456
+
+    def test_union_simple(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("union foo { int a; short b, c; };")
+        u = ffi.new("union foo*")
+        assert u.a == u.b == u.c == 0
+        u.b = -23
+        assert u.b == -23
+        assert u.a != 0
+        py.test.raises(OverflowError, "u.b = 32768")
+        #
+        u = ffi.new("union foo*", [-2])
+        assert u.a == -2
+        py.test.raises((AttributeError, TypeError), "del u.a")
+        assert repr(u) == "<cdata 'union foo *' owning %d bytes>" % SIZE_OF_INT
+
+    def test_union_opaque(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(TypeError, ffi.new, "union baz *")
+        u = ffi.new("union baz **")   # this works
+        assert u[0] == ffi.NULL
+
+    def test_union_initializer(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("union foo { char a; int b; };")
+        py.test.raises(TypeError, ffi.new, "union foo*", b'A')
+        py.test.raises(TypeError, ffi.new, "union foo*", 5)
+        py.test.raises(ValueError, ffi.new, "union foo*", [b'A', 5])
+        u = ffi.new("union foo*", [b'A'])
+        assert u.a == b'A'
+        py.test.raises(TypeError, ffi.new, "union foo*", [1005])
+        u = ffi.new("union foo*", {'b': 12345})
+        assert u.b == 12345
+        u = ffi.new("union foo*", [])
+        assert u.a == b'\x00'
+        assert u.b == 0
+
+    def test_sizeof_type(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            struct foo { int a; short b, c, d; };
+            union foo { int a; short b, c, d; };
+        """)
+        for c_type, expected_size in [
+            ('char', 1),
+            ('unsigned int', 4),
+            ('char *', SIZE_OF_PTR),
+            ('int[5]', 20),
+            ('struct foo', 12),
+            ('union foo', 4),
+            ]:
+            size = ffi.sizeof(c_type)
+            assert size == expected_size, (size, expected_size, ctype)
+
+    def test_sizeof_cdata(self):
+        ffi = FFI(backend=self.Backend())
+        assert ffi.sizeof(ffi.new("short*")) == SIZE_OF_PTR
+        assert ffi.sizeof(ffi.cast("short", 123)) == SIZE_OF_SHORT
+        #
+        a = ffi.new("int[]", [10, 11, 12, 13, 14])
+        assert len(a) == 5
+        assert ffi.sizeof(a) == 5 * SIZE_OF_INT
+
+    def test_string_from_char_pointer(self):
+        ffi = FFI(backend=self.Backend())
+        x = ffi.new("char*", b"x")
+        assert str(x) == repr(x)
+        assert ffi.string(x) == b"x"
+        assert ffi.string(ffi.new("char*", b"\x00")) == b""
+        py.test.raises(TypeError, ffi.new, "char*", unicode("foo"))
+
+    def test_unicode_from_wchar_pointer(self):
+        ffi = FFI(backend=self.Backend())
+        self.check_wchar_t(ffi)
+        x = ffi.new("wchar_t*", u+"x")
+        assert unicode(x) == unicode(repr(x))
+        assert ffi.string(x) == u+"x"
+        assert ffi.string(ffi.new("wchar_t*", u+"\x00")) == u+""
+
+    def test_string_from_char_array(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("char[]", b"hello.")
+        p[5] = b'!'
+        assert ffi.string(p) == b"hello!"
+        p[6] = b'?'
+        assert ffi.string(p) == b"hello!?"
+        p[3] = b'\x00'
+        assert ffi.string(p) == b"hel"
+        assert ffi.string(p, 2) == b"he"
+        py.test.raises(IndexError, "p[7] = b'X'")
+        #
+        a = ffi.new("char[]", b"hello\x00world")
+        assert len(a) == 12
+        p = ffi.cast("char *", a)
+        assert ffi.string(p) == b'hello'
+
+    def test_string_from_wchar_array(self):
+        ffi = FFI(backend=self.Backend())
+        self.check_wchar_t(ffi)
+        assert ffi.string(ffi.cast("wchar_t", "x")) == u+"x"
+        assert ffi.string(ffi.cast("wchar_t", u+"x")) == u+"x"
+        x = ffi.cast("wchar_t", "x")
+        assert str(x) == repr(x)
+        assert ffi.string(x) == u+"x"
+        #
+        p = ffi.new("wchar_t[]", u+"hello.")
+        p[5] = u+'!'
+        assert ffi.string(p) == u+"hello!"
+        p[6] = u+'\u04d2'
+        assert ffi.string(p) == u+"hello!\u04d2"
+        p[3] = u+'\x00'
+        assert ffi.string(p) == u+"hel"
+        assert ffi.string(p, 123) == u+"hel"
+        py.test.raises(IndexError, "p[7] = u+'X'")
+        #
+        a = ffi.new("wchar_t[]", u+"hello\x00world")
+        assert len(a) == 12
+        p = ffi.cast("wchar_t *", a)
+        assert ffi.string(p) == u+'hello'
+        assert ffi.string(p, 123) == u+'hello'
+        assert ffi.string(p, 5) == u+'hello'
+        assert ffi.string(p, 2) == u+'he'
+
+    def test_fetch_const_char_p_field(self):
+        # 'const' is ignored so far
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { const char *name; };")
+        t = ffi.new("const char[]", b"testing")
+        s = ffi.new("struct foo*", [t])
+        assert type(s.name) not in (bytes, str, unicode)
+        assert ffi.string(s.name) == b"testing"
+        py.test.raises(TypeError, "s.name = None")
+        s.name = ffi.NULL
+        assert s.name == ffi.NULL
+
+    def test_fetch_const_wchar_p_field(self):
+        # 'const' is ignored so far
+        ffi = FFI(backend=self.Backend())
+        self.check_wchar_t(ffi)
+        ffi.cdef("struct foo { const wchar_t *name; };")
+        t = ffi.new("const wchar_t[]", u+"testing")
+        s = ffi.new("struct foo*", [t])
+        assert type(s.name) not in (bytes, str, unicode)
+        assert ffi.string(s.name) == u+"testing"
+        s.name = ffi.NULL
+        assert s.name == ffi.NULL
+
+    def test_voidp(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(TypeError, ffi.new, "void*")
+        p = ffi.new("void **")
+        assert p[0] == ffi.NULL
+        a = ffi.new("int[]", [10, 11, 12])
+        p = ffi.new("void **", a)
+        vp = p[0]
+        py.test.raises(TypeError, "vp[0]")
+        py.test.raises(TypeError, ffi.new, "short **", a)
+        #
+        ffi.cdef("struct foo { void *p; int *q; short *r; };")
+        s = ffi.new("struct foo *")
+        s.p = a    # works
+        s.q = a    # works
+        py.test.raises(TypeError, "s.r = a")    # fails
+        b = ffi.cast("int *", a)
+        s.p = b    # works
+        s.q = b    # works
+        py.test.raises(TypeError, "s.r = b")    # fails
+
+    def test_functionptr_simple(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(TypeError, ffi.callback, "int(*)(int)", 0)
+        def cb(n):
+            return n + 1
+        cb.__qualname__ = 'cb'
+        p = ffi.callback("int(*)(int)", cb)
+        res = p(41)     # calling an 'int(*)(int)', i.e. a function pointer
+        assert res == 42 and type(res) is int
+        res = p(ffi.cast("int", -41))
+        assert res == -40 and type(res) is int
+        assert repr(p).startswith(
+            "<cdata 'int(*)(int)' calling <function cb at 0x")
+        assert ffi.typeof(p) is ffi.typeof("int(*)(int)")
+        q = ffi.new("int(**)(int)", p)
+        assert repr(q) == "<cdata 'int(* *)(int)' owning %d bytes>" % (
+            SIZE_OF_PTR)
+        py.test.raises(TypeError, "q(43)")
+        res = q[0](43)
+        assert res == 44
+        q = ffi.cast("int(*)(int)", p)
+        assert repr(q).startswith("<cdata 'int(*)(int)' 0x")
+        res = q(45)
+        assert res == 46
+
+    def test_functionptr_advanced(self):
+        ffi = FFI(backend=self.Backend())
+        t = ffi.typeof("int(*(*)(int))(int)")
+        assert repr(t) == self.TypeRepr % "int(*(*)(int))(int)"
+
+    def test_functionptr_voidptr_return(self):
+        ffi = FFI(backend=self.Backend())
+        def cb():
+            return ffi.NULL
+        p = ffi.callback("void*(*)()", cb)
+        res = p()
+        assert res is not None
+        assert res == ffi.NULL
+        int_ptr = ffi.new('int*')
+        void_ptr = ffi.cast('void*', int_ptr)
+        def cb():
+            return void_ptr
+        p = ffi.callback("void*(*)()", cb)
+        res = p()
+        assert res == void_ptr
+
+    def test_functionptr_intptr_return(self):
+        ffi = FFI(backend=self.Backend())
+        def cb():
+            return ffi.NULL
+        p = ffi.callback("int*(*)()", cb)
+        res = p()
+        assert res == ffi.NULL
+        int_ptr = ffi.new('int*')
+        def cb():
+            return int_ptr
+        p = ffi.callback("int*(*)()", cb)
+        res = p()
+        assert repr(res).startswith("<cdata 'int *' 0x")
+        assert res == int_ptr
+        int_array_ptr = ffi.new('int[1]')
+        def cb():
+            return int_array_ptr
+        p = ffi.callback("int*(*)()", cb)
+        res = p()
+        assert repr(res).startswith("<cdata 'int *' 0x")
+        assert res == int_array_ptr
+
+    def test_functionptr_void_return(self):
+        ffi = FFI(backend=self.Backend())
+        def foo():
+            pass
+        foo_cb = ffi.callback("void foo()", foo)
+        result = foo_cb()
+        assert result is None
+
+    def test_char_cast(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.cast("int", b'\x01')
+        assert ffi.typeof(p) is ffi.typeof("int")
+        assert int(p) == 1
+        p = ffi.cast("int", ffi.cast("char", b"a"))
+        assert int(p) == ord("a")
+        p = ffi.cast("int", ffi.cast("char", b"\x80"))
+        assert int(p) == 0x80     # "char" is considered unsigned in this case
+        p = ffi.cast("int", b"\x81")
+        assert int(p) == 0x81
+
+    def test_wchar_cast(self):
+        ffi = FFI(backend=self.Backend())
+        self.check_wchar_t(ffi)
+        p = ffi.cast("int", ffi.cast("wchar_t", u+'\u1234'))
+        assert int(p) == 0x1234
+        p = ffi.cast("long long", ffi.cast("wchar_t", -1))
+        if SIZE_OF_WCHAR == 2:      # 2 bytes, unsigned
+            assert int(p) == 0xffff
+        elif (sys.platform.startswith('linux') and
+              platform.machine().startswith('x86')):   # known to be signed
+            assert int(p) == -1
+        else:                     # in general, it can be either signed or not
+            assert int(p) in [-1, 0xffffffff]  # e.g. on arm, both cases occur
+        p = ffi.cast("int", u+'\u1234')
+        assert int(p) == 0x1234
+
+    def test_cast_array_to_charp(self):
+        ffi = FFI(backend=self.Backend())
+        a = ffi.new("short int[]", [0x1234, 0x5678])
+        p = ffi.cast("char*", a)
+        data = b''.join([p[i] for i in range(4)])
+        if sys.byteorder == 'little':
+            assert data == b'\x34\x12\x78\x56'
+        else:
+            assert data == b'\x12\x34\x56\x78'
+
+    def test_cast_between_pointers(self):
+        ffi = FFI(backend=self.Backend())
+        a = ffi.new("short int[]", [0x1234, 0x5678])
+        p = ffi.cast("short*", a)
+        p2 = ffi.cast("int*", p)
+        q = ffi.cast("char*", p2)
+        data = b''.join([q[i] for i in range(4)])
+        if sys.byteorder == 'little':
+            assert data == b'\x34\x12\x78\x56'
+        else:
+            assert data == b'\x12\x34\x56\x78'
+
+    def test_cast_pointer_and_int(self):
+        ffi = FFI(backend=self.Backend())
+        a = ffi.new("short int[]", [0x1234, 0x5678])
+        l1 = ffi.cast("intptr_t", a)
+        p = ffi.cast("short*", a)
+        l2 = ffi.cast("intptr_t", p)
+        assert int(l1) == int(l2) != 0
+        q = ffi.cast("short*", l1)
+        assert q == ffi.cast("short*", int(l1))
+        assert q[0] == 0x1234
+        assert int(ffi.cast("intptr_t", ffi.NULL)) == 0
+
+    def test_cast_functionptr_and_int(self):
+        ffi = FFI(backend=self.Backend())
+        def cb(n):
+            return n + 1
+        a = ffi.callback("int(*)(int)", cb)
+        p = ffi.cast("void *", a)
+        assert p
+        b = ffi.cast("int(*)(int)", p)
+        assert b(41) == 42
+        assert a == b
+        assert hash(a) == hash(b)
+
+    def test_callback_crash(self):
+        ffi = FFI(backend=self.Backend())
+        def cb(n):
+            raise Exception
+        a = ffi.callback("int(*)(int)", cb, error=42)
+        res = a(1)    # and the error reported to stderr
+        assert res == 42
+
+    def test_structptr_argument(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo_s { int a, b; };")
+        def cb(p):
+            return p[0].a * 1000 + p[0].b * 100 + p[1].a * 10 + p[1].b
+        a = ffi.callback("int(*)(struct foo_s[])", cb)

From pypy.commits at gmail.com  Sat Sep  3 14:04:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 11:04:41 -0700 (PDT)
Subject: [pypy-commit] cffi default: Un-document the removal of the ctypes
 backend
Message-ID: <57cb10b9.0cce1c0a.b206c.2214@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2756:f959e7d3af34
Date: 2016-09-03 19:30 +0200
http://bitbucket.org/cffi/cffi/changeset/f959e7d3af34/

Log:	Un-document the removal of the ctypes backend

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -13,10 +13,6 @@
   name, you can rename it manually to ``NAME.abi3.so``, or use the very
   recent setuptools 26.
 
-* Removed the ctypes backend.  If ``_cffi_backend`` was not compiled,
-  you could ask (using an undocumented interface) for ``backend_ctypes``
-  instead.  That was never fully functional and long deprecated.
-
 * Added ``ffi.compile(debug=...)``, similar to ``python setup.py build
   --debug`` but defaulting to True if we are running a debugging
   version of Python itself.

From pypy.commits at gmail.com  Sat Sep  3 14:04:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 11:04:43 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #282: probable test and fix
Message-ID: <57cb10bb.229ec20a.918a5.91a1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2757:5fa1d8697d3e
Date: 2016-09-03 20:04 +0200
http://bitbucket.org/cffi/cffi/changeset/5fa1d8697d3e/

Log:	Issue #282: probable test and fix

diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
--- a/cffi/backend_ctypes.py
+++ b/cffi/backend_ctypes.py
@@ -997,29 +997,35 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    return self() is other()
+                def __ne__(self, other):
+                    return self() is not other()
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1478,6 +1478,7 @@
             assert p1[0] == 123
             seen.append(1)
         q = ffi.gc(p, destructor)
+        assert ffi.typeof(q) is ffi.typeof(p)
         import gc; gc.collect()
         assert seen == []
         del q

From pypy.commits at gmail.com  Sat Sep  3 14:10:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 11:10:55 -0700 (PDT)
Subject: [pypy-commit] cffi default: fix
Message-ID: <57cb122f.436ec20a.434cf.8c51@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2758:b81ca61b6de6
Date: 2016-09-03 20:10 +0200
http://bitbucket.org/cffi/cffi/changeset/b81ca61b6de6/

Log:	fix

diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
--- a/cffi/backend_ctypes.py
+++ b/cffi/backend_ctypes.py
@@ -1004,9 +1004,11 @@
             import weakref
             class MyRef(weakref.ref):
                 def __eq__(self, other):
-                    return self() is other()
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
                 def __ne__(self, other):
-                    return self() is not other()
+                    return not (self == other)
             self._weakref_cache_ref = {}, MyRef
         weak_cache, MyRef = self._weakref_cache_ref
 

From pypy.commits at gmail.com  Sat Sep  3 14:16:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 11:16:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: import cffi/b81ca61b6de6
Message-ID: <57cb1360.c398c20a.9932f.9d3b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86854:fdd46325fe3d
Date: 2016-09-03 20:14 +0200
http://bitbucket.org/pypy/pypy/changeset/fdd46325fe3d/

Log:	import cffi/b81ca61b6de6

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.1"
+__version_info__ = (1, 8, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,37 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1479,6 +1479,7 @@
             assert p1[0] == 123
             seen.append(1)
         q = ffi.gc(p, destructor)
+        assert ffi.typeof(q) is ffi.typeof(p)
         import gc; gc.collect()
         assert seen == []
         del q
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -149,3 +149,25 @@
         p = snip_setuptools_verify2.C.getpwuid(0)
         assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
         ''')
+
+    def test_set_py_limited_api(self):
+        from cffi.setuptools_ext import _set_py_limited_api
+        try:
+            import setuptools
+            orig_version = setuptools.__version__
+            setuptools.__version__ = '26.0.0'
+            from setuptools import Extension
+
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+            setuptools.__version__ = '25.0'
+            kwds = _set_py_limited_api(Extension, {})
+            assert not kwds
+
+            setuptools.__version__ = 'development'
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+        finally:
+            setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1975,9 +1975,9 @@
 
 def test_function_returns_partial_struct():
     ffi = FFI()
-    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    ffi.cdef("struct aaa { int a; ...; }; struct aaa f1(int);")
     lib = verify(ffi, "test_function_returns_partial_struct", """
-        struct a { int b, a, c; };
-        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+        struct aaa { int b, a, c; };
+        static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52

From pypy.commits at gmail.com  Sat Sep  3 15:07:59 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sat, 03 Sep 2016 12:07:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <57cb1f8f.ca11c30a.ff1fd.ad59@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r86855:83f439eeb461
Date: 2016-09-03 12:07 -0700
http://bitbucket.org/pypy/pypy/changeset/83f439eeb461/

Log:	merge default

diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.1"
+__version_info__ = (1, 8, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,37 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -187,6 +187,43 @@
 
         """)
 
+    def test_oldstyle_methcall(self):
+        def main():
+            def g(): pass
+            class A:
+                def f(self):
+                    return self.x + 1
+            class I(A):
+                pass
+            class J(I):
+                pass
+
+
+            class B(J):
+                def __init__(self, x):
+                    self.x = x
+
+            i = 0
+            b = B(1)
+            while i < 1000:
+                g()
+                v = b.f() # ID: meth
+                i += v
+            return i
+
+        log = self.run(main, [], threshold=80)
+        loop, = log.loops_by_filename(self.filepath, is_entry_bridge=True)
+        assert loop.match_by_id('meth',
+        '''
+    guard_nonnull_class(p18, ..., descr=...)
+    p52 = getfield_gc_r(p18, descr=...) # read map
+    guard_value(p52, ConstPtr(ptr53), descr=...)
+    p54 = getfield_gc_r(p18, descr=...) # read class
+    guard_value(p54, ConstPtr(ptr55), descr=...)
+    p56 = force_token() # done
+        ''')
+
+
     def test_oldstyle_newstyle_mix(self):
         def main():
             class A:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1479,6 +1479,7 @@
             assert p1[0] == 123
             seen.append(1)
         q = ffi.gc(p, destructor)
+        assert ffi.typeof(q) is ffi.typeof(p)
         import gc; gc.collect()
         assert seen == []
         del q
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -149,3 +149,25 @@
         p = snip_setuptools_verify2.C.getpwuid(0)
         assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
         ''')
+
+    def test_set_py_limited_api(self):
+        from cffi.setuptools_ext import _set_py_limited_api
+        try:
+            import setuptools
+            orig_version = setuptools.__version__
+            setuptools.__version__ = '26.0.0'
+            from setuptools import Extension
+
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+            setuptools.__version__ = '25.0'
+            kwds = _set_py_limited_api(Extension, {})
+            assert not kwds
+
+            setuptools.__version__ = 'development'
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+        finally:
+            setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1975,9 +1975,9 @@
 
 def test_function_returns_partial_struct():
     ffi = FFI()
-    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    ffi.cdef("struct aaa { int a; ...; }; struct aaa f1(int);")
     lib = verify(ffi, "test_function_returns_partial_struct", """
-        struct a { int b, a, c; };
-        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+        struct aaa { int b, a, c; };
+        static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
@@ -194,6 +194,29 @@
             _fields_ = [('t', enum)]
         assert isinstance(S().t, enum)
 
+    def test_no_missing_shape_to_ffi_type(self):
+        # whitebox test
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("only for pypy's ctypes")
+        skip("re-enable after adding 'g' to _shape_to_ffi_type.typemap, "
+             "which I think needs fighting all the way up from "
+             "rpython.rlib.libffi")
+        from _ctypes.basics import _shape_to_ffi_type
+        from _rawffi import Array
+        for i in range(1, 256):
+            try:
+                Array(chr(i))
+            except ValueError:
+                pass
+            else:
+                assert chr(i) in _shape_to_ffi_type.typemap
+
+    @py.test.mark.xfail
+    def test_pointer_to_long_double(self):
+        import ctypes
+        ctypes.POINTER(ctypes.c_longdouble)
+
 ##    def test_perf(self):
 ##        check_perf()
 
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -164,8 +164,15 @@
             # annotations that are passed in, and don't annotate the old
             # graph -- it's already low-level operations!
             for a, s_newarg in zip(block.inputargs, cells):
-                s_oldarg = self.binding(a)
-                assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
+                s_oldarg = a.annotation
+                # XXX: Should use s_oldarg.contains(s_newarg) but that breaks
+                # PyPy translation
+                if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg:
+                    raise annmodel.AnnotatorError(
+                        "Late-stage annotation is not allowed to modify the "
+                        "existing annotation for variable %s: %s" %
+                            (a, s_oldarg))
+
         else:
             assert not self.frozen
             if block not in self.annotated:
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -17,7 +17,7 @@
 from rpython.flowspace.model import Variable, Constant, const
 from rpython.flowspace.operation import op
 from rpython.rlib import rarithmetic
-from rpython.annotator.model import AnnotatorError
+from rpython.annotator.model import AnnotatorError, TLS
 
 BINARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values()
                         if oper.dispatch == 2])
@@ -436,6 +436,11 @@
 class __extend__(pairtype(SomeFloat, SomeFloat)):
 
     def union((flt1, flt2)):
+        if not TLS.allow_int_to_float:
+            # in this mode, if one of the two is actually the
+            # subclass SomeInteger, complain
+            if isinstance(flt1, SomeInteger) or isinstance(flt2, SomeInteger):
+                raise UnionError(flt1, flt2)
         return SomeFloat()
 
     add = sub = mul = union
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -44,6 +44,7 @@
     # A global attribute :-(  Patch it with 'True' to enable checking of
     # the no_nul attribute...
     check_str_without_nul = False
+    allow_int_to_float = True
 TLS = State()
 
 class SomeObject(object):
@@ -749,6 +750,7 @@
                 s1 = pair(s1, s2).union()
     else:
         # this is just a performance shortcut
+        # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
         if s1 != s2:
             s1 = pair(s1, s2).union()
     return s1
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -47,6 +47,8 @@
         cls.ulonglong = clibffi.cast_type_to_ffitype(rffi.ULONGLONG)
         cls.signed = clibffi.cast_type_to_ffitype(rffi.SIGNED)
         cls.wchar_t = clibffi.cast_type_to_ffitype(lltype.UniChar)
+        # XXX long double support: clibffi.ffi_type_longdouble, but then
+        # XXX fix the whole rest of this file to add a case for long double
         del cls._import
 
     @staticmethod
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -98,8 +98,15 @@
         try:
             ctypes.CDLL(name)
         except OSError as e:
+            # common case: ctypes fails too, with the real dlerror()
+            # message in str(e).  Return that error message.
             return str(e)
         else:
+            # uncommon case: may happen if 'name' is a linker script
+            # (which the C-level dlopen() can't handle) and we are
+            # directly running on pypy (whose implementation of ctypes
+            # or cffi will resolve linker scripts).  In that case, 
+            # unsure what we can do.
             return ("opening %r with ctypes.CDLL() works, "
                     "but not with c_dlopen()??" % (name,))
 
@@ -160,7 +167,18 @@
             mode = _dlopen_default_mode()
         elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
+        #
+        # haaaack for 'pypy py.test -A' if libm.so is a linker script
+        # (see reason in _dlerror_on_dlopen_untranslated())
+        must_free = False
+        if not we_are_translated() and platform.name == "linux":
+            if name and rffi.charp2str(name) == 'libm.so':
+                name = rffi.str2charp('libm.so.6')
+                must_free = True
+        #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
+        if must_free:
+            rffi.free_charp(name)
         if not res:
             if not we_are_translated():
                 err = _dlerror_on_dlopen_untranslated(name)
diff --git a/rpython/rlib/rmarshal.py b/rpython/rlib/rmarshal.py
--- a/rpython/rlib/rmarshal.py
+++ b/rpython/rlib/rmarshal.py
@@ -346,11 +346,15 @@
     # on s_bigger.  It relies on the fact that s_bigger was created with
     # an expression like 'annotation([s_item])' which returns a ListDef with
     # no bookkeeper, on which side-effects are not allowed.
+    saved = annmodel.TLS.allow_int_to_float
     try:
+        annmodel.TLS.allow_int_to_float = False
         s_union = annmodel.unionof(s_bigger, s_smaller)
         return s_bigger.contains(s_union)
     except (annmodel.UnionError, TooLateForChange):
         return False
+    finally:
+        annmodel.TLS.allow_int_to_float = saved
 
 
 class __extend__(pairtype(MTag, annmodel.SomeObject)):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -137,7 +137,27 @@
                                  result=result)
     return result.build(), pos
 
- at specialize.argtype(6)
+def _invalid_cont_byte(ordch):
+    return ordch>>6 != 0x2    # 0b10
+
+_invalid_byte_2_of_2 = _invalid_cont_byte
+_invalid_byte_3_of_3 = _invalid_cont_byte
+_invalid_byte_3_of_4 = _invalid_cont_byte
+_invalid_byte_4_of_4 = _invalid_cont_byte
+
+ at specialize.arg(2)
+def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xe0 and ordch2 < 0xa0)
+            # surrogates shouldn't be valid UTF-8!
+            or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f))
+
+def _invalid_byte_2_of_4(ordch1, ordch2):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xf0 and ordch2 < 0x90) or
+            (ordch1 == 0xf4 and ordch2 > 0x8f))
+
+ at specialize.arg(5)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:
@@ -157,22 +177,23 @@
         if pos + n > size:
             if not final:
                 break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
             charsleft = size - pos - 1 # either 0, 1, 2
-            # note: when we get the 'unexpected end of data' we don't care
-            # about the pos anymore and we just ignore the value
+            # note: when we get the 'unexpected end of data' we need
+            # to care about the pos returned; it can be lower than size,
+            # in case we need to continue running this loop
             if not charsleft:
                 # there's only the start byte and nothing else
                 r, pos = errorhandler(errors, 'utf8',
                                       'unexpected end of data',
                                       s, pos, pos+1)
                 result.append(r)
-                break
+                continue
             ordch2 = ord(s[pos+1])
             if n == 3:
                 # 3-bytes seq with only a continuation byte
-                if (ordch2>>6 != 0x2 or   # 0b10
-                    (ordch1 == 0xe0 and ordch2 < 0xa0)):
-                    # or (ordch1 == 0xed and ordch2 > 0x9f)
+                if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                     # second byte invalid, take the first and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
@@ -185,19 +206,17 @@
                                       'unexpected end of data',
                                       s, pos, pos+2)
                     result.append(r)
-                    break
+                    continue
             elif n == 4:
                 # 4-bytes seq with 1 or 2 continuation bytes
-                if (ordch2>>6 != 0x2 or    # 0b10
-                    (ordch1 == 0xf0 and ordch2 < 0x90) or
-                    (ordch1 == 0xf4 and ordch2 > 0x8f)):
+                if _invalid_byte_2_of_4(ordch1, ordch2):
                     # second byte invalid, take the first and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
                                           s, pos, pos+1)
                     result.append(r)
                     continue
-                elif charsleft == 2 and ord(s[pos+2])>>6 != 0x2:   # 0b10
+                elif charsleft == 2 and _invalid_byte_3_of_4(ord(s[pos+2])):
                     # third byte invalid, take the first two and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
@@ -210,7 +229,8 @@
                                       'unexpected end of data',
                                       s, pos, pos+charsleft+1)
                     result.append(r)
-                    break
+                    continue
+            raise AssertionError("unreachable")
 
         if n == 0:
             r, pos = errorhandler(errors, 'utf8',
@@ -223,7 +243,7 @@
 
         elif n == 2:
             ordch2 = ord(s[pos+1])
-            if ordch2>>6 != 0x2:   # 0b10
+            if _invalid_byte_2_of_2(ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
@@ -237,17 +257,13 @@
         elif n == 3:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
-            if (ordch2>>6 != 0x2 or    # 0b10
-                (ordch1 == 0xe0 and ordch2 < 0xa0)
-                # surrogates shouldn't be valid UTF-8!
-                or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f)
-                ):
+            if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif ordch3>>6 != 0x2:     # 0b10
+            elif _invalid_byte_3_of_3(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
@@ -263,21 +279,19 @@
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
             ordch4 = ord(s[pos+3])
-            if (ordch2>>6 != 0x2 or     # 0b10
-                (ordch1 == 0xf0 and ordch2 < 0x90) or
-                (ordch1 == 0xf4 and ordch2 > 0x8f)):
+            if _invalid_byte_2_of_4(ordch1, ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif ordch3>>6 != 0x2:     # 0b10
+            elif _invalid_byte_3_of_4(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
-            elif ordch4>>6 != 0x2:     # 0b10
+            elif _invalid_byte_4_of_4(ordch4):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+3)
diff --git a/rpython/rlib/test/test_rmarshal.py b/rpython/rlib/test/test_rmarshal.py
--- a/rpython/rlib/test/test_rmarshal.py
+++ b/rpython/rlib/test/test_rmarshal.py
@@ -128,10 +128,12 @@
 
 def test_llinterp_marshal():
     from rpython.rtyper.test.test_llinterp import interpret
-    marshaller = get_marshaller([(int, str, float)])
+    marshaller1 = get_marshaller([(int, str, float)])
+    marshaller2 = get_marshaller([(int, str, int)])
     def f():
         buf = []
-        marshaller(buf, [(5, "hello", -0.5), (7, "world", 1E100)])
+        marshaller1(buf, [(5, "hello", -0.5), (7, "world", 1E100)])
+        marshaller2(buf, [(5, "hello", 1)])
         return ''.join(buf)
     res = interpret(f, [])
     res = ''.join(res.chars)
@@ -139,14 +141,20 @@
         assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00'
                        's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00'
                        'i\x07\x00\x00\x00s\x05\x00\x00\x00world'
-                       'f\x061e+100')
+                       'f\x061e+100'
+                       '[\x01\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00'
+                       's\x05\x00\x00\x00helloi\x01\x00\x00\x00')
     else:
         assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00'
                        'I\x05\x00\x00\x00\x00\x00\x00\x00'
                        's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00'
                        'I\x07\x00\x00\x00\x00\x00\x00\x00'
                        's\x05\x00\x00\x00world'
-                       'f\x061e+100')
+                       'f\x061e+100'
+                       '[\x01\x00\x00\x00(\x03\x00\x00\x00'
+                       'I\x05\x00\x00\x00\x00\x00\x00\x00'
+                       's\x05\x00\x00\x00hello'
+                       'I\x01\x00\x00\x00\x00\x00\x00\x00')
 
 def test_llinterp_unmarshal():
     from rpython.rtyper.test.test_llinterp import interpret
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -286,9 +286,15 @@
 
 
 class TestUTF8Decoding(UnicodeTests):
-    def __init__(self):
+    def setup_method(self, meth):
         self.decoder = self.getdecoder('utf-8')
 
+    def custom_replace(self, errors, encoding, msg, s, startingpos, endingpos):
+        assert errors == 'custom'
+        # returns FOO, but consumes only one character (not up to endingpos)
+        FOO = u'\u1234'
+        return FOO, startingpos + 1
+
     def to_bytestring(self, bytes):
         return ''.join(chr(int(c, 16)) for c in bytes.split())
 
@@ -309,6 +315,7 @@
         E.g. <80> is a continuation byte and can appear only after a start byte.
         """
         FFFD = u'\ufffd'
+        FOO = u'\u1234'
         for byte in '\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF':
             py.test.raises(UnicodeDecodeError, self.decoder, byte, 1, None, final=True)
             self.checkdecodeerror(byte, 'utf-8', 0, 1, addstuff=False,
@@ -320,6 +327,11 @@
             assert self.decoder(byte, 1, 'ignore', final=True) == (u'', 1)
             assert (self.decoder('aaaa' + byte + 'bbbb', 9, 'ignore',
                         final=True) == (u'aaaabbbb', 9))
+            assert self.decoder(byte, 1, 'custom', final=True,
+                        errorhandler=self.custom_replace) == (FOO, 1)
+            assert (self.decoder('aaaa' + byte + 'bbbb', 9, 'custom',
+                        final=True, errorhandler=self.custom_replace) ==
+                        (u'aaaa'+ FOO + u'bbbb', 9))
 
     def test_unexpected_end_of_data(self):
         """
@@ -343,6 +355,7 @@
             'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF'
         ]
         FFFD = u'\ufffd'
+        FOO = u'\u1234'
         for seq in sequences:
             seq = self.to_bytestring(seq)
             py.test.raises(UnicodeDecodeError, self.decoder, seq, len(seq),
@@ -358,6 +371,12 @@
                                 ) == (u'', len(seq))
             assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'ignore',
                         final=True) == (u'aaaabbbb', len(seq) + 8))
+            assert (self.decoder(seq, len(seq), 'custom', final=True,
+                        errorhandler=self.custom_replace) == 
+                        (FOO * len(seq), len(seq)))
+            assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'custom',
+                        final=True, errorhandler=self.custom_replace) ==
+                        (u'aaaa'+ FOO * len(seq) + u'bbbb', len(seq) + 8))
 
     def test_invalid_cb_for_2bytes_seq(self):
         """
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -38,6 +38,24 @@
         xf = self.compile(f, [])
         assert xf() == 8+3
 
+    def test_no_float_to_int_conversion(self):
+        c_source = py.code.Source("""
+        int someexternalfunction(int x)
+        {
+            return (x + 3);
+        }
+        """)
+
+        eci = ExternalCompilationInfo(separate_module_sources=[c_source])
+        z = llexternal('someexternalfunction', [Signed], Signed,
+                       compilation_info=eci)
+
+        def f():
+            return z(8.2)
+
+        py.test.raises(TypeError, f)
+        py.test.raises(TypeError, self.compile, f, [])
+
     def test_hashdefine(self):
         h_source = """
         #define X(i) (i+3)
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -540,7 +540,7 @@
     def ll_ullong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
-        return llop.ullong_mod(UnsignedLongLong, x, y)
+        return ll_ullong_py_mod(x, y)
 
 @jit.dont_look_inside
 def ll_lllong_py_mod(x, y):
diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -532,8 +532,7 @@
         return sys.maxint
     else:
         res = Solution[blockmap[graph.startblock]]
-        assert res >= 0
-        return res
+        return max(res, 0.0)
 
 def static_instruction_count(graph):
     count = 0
diff --git a/rpython/translator/c/test/test_exception.py b/rpython/translator/c/test/test_exception.py
--- a/rpython/translator/c/test/test_exception.py
+++ b/rpython/translator/c/test/test_exception.py
@@ -9,7 +9,7 @@
 getcompiledopt = test_backendoptimized.TestTypedOptimizedTestCase().getcompiled
 
 
-class TestException(Exception):
+class InTestException(Exception):
     pass
 
 class MyException(Exception):
@@ -18,7 +18,7 @@
 def test_simple1():
     def raise_(i):
         if i == 0:
-            raise TestException()
+            raise InTestException()
         elif i == 1:
             raise MyException()
         else:
@@ -29,7 +29,7 @@
             b = raise_(i) + 12
             c = raise_(i) + 13
             return a+b+c
-        except TestException:
+        except InTestException:
             return 7
         except MyException:
             return 123
diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -213,6 +213,7 @@
         log.WARNING(warning)
 
 def main():
+    sys.setrecursionlimit(2000)  # PyPy can't translate within cpython's 1k limit
     targetspec_dic, translateconfig, config, args = parse_options_and_load_target()
     from rpython.translator import translator
     from rpython.translator import driver
diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -29,6 +29,7 @@
     assert msg == fnname
     msg = read_message(f)
     assert msg == args
+    assert [type(x) for x in msg] == [type(x) for x in args]
     if isinstance(result, Exception):
         write_exception(g, result)
     else:

From pypy.commits at gmail.com  Sat Sep  3 17:43:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Sep 2016 14:43:08 -0700 (PDT)
Subject: [pypy-commit] cffi default: Python 3 fix
Message-ID: <57cb43ec.a3a3c20a.2eacd.9364@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2759:d04043b6e7bc
Date: 2016-09-03 23:42 +0200
http://bitbucket.org/cffi/cffi/changeset/d04043b6e7bc/

Log:	Python 3 fix

diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
--- a/cffi/backend_ctypes.py
+++ b/cffi/backend_ctypes.py
@@ -1009,6 +1009,12 @@
                         myref is not None and myref is other())
                 def __ne__(self, other):
                     return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
             self._weakref_cache_ref = {}, MyRef
         weak_cache, MyRef = self._weakref_cache_ref
 

From pypy.commits at gmail.com  Sun Sep  4 01:29:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 03 Sep 2016 22:29:29 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: update version numbers,
 make test run untranslated
Message-ID: <57cbb139.ca11c30a.ff1fd.43cd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86856:b0cef2e770b9
Date: 2016-09-04 08:26 +0300
http://bitbucket.org/pypy/pypy/changeset/b0cef2e770b9/

Log:	update version numbers, make test run untranslated

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.0"
-#define PYPY_VERSION_NUM  0x05040000
+#define PYPY_VERSION "5.4.1"
+#define PYPY_VERSION_NUM  0x05040100
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -32,9 +32,11 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 4, 0, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 4, 1, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Sun Sep  4 01:29:31 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 03 Sep 2016 22:29:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: update version minor number,
 make test run untranslated
Message-ID: <57cbb13b.21b0c20a.5d2be.a57b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86857:3bbe61b5ea1c
Date: 2016-09-04 08:28 +0300
http://bitbucket.org/pypy/pypy/changeset/3bbe61b5ea1c/

Log:	update version minor number, make test run untranslated

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.1-alpha0"
-#define PYPY_VERSION_NUM  0x05040100
+#define PYPY_VERSION "5.5.0-alpha0"
+#define PYPY_VERSION_NUM  0x05050000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -32,9 +32,11 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 5, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Sun Sep  4 07:37:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Sep 2016 04:37:14 -0700 (PDT)
Subject: [pypy-commit] cffi default: Skip test if setuptools cannot be
 imported
Message-ID: <57cc076a.54bc1c0a.bc6d2.4187@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2760:cb6147abe7af
Date: 2016-09-04 13:37 +0200
http://bitbucket.org/cffi/cffi/changeset/cb6147abe7af/

Log:	Skip test if setuptools cannot be imported

diff --git a/testing/cffi0/test_zintegration.py b/testing/cffi0/test_zintegration.py
--- a/testing/cffi0/test_zintegration.py
+++ b/testing/cffi0/test_zintegration.py
@@ -153,7 +153,10 @@
         from cffi.setuptools_ext import _set_py_limited_api
         try:
             import setuptools
-            orig_version = setuptools.__version__
+        except ImportError as e:
+            py.test.skip(str(e))
+        orig_version = setuptools.__version__
+        try:
             setuptools.__version__ = '26.0.0'
             from setuptools import Extension
 

From pypy.commits at gmail.com  Sun Sep  4 07:38:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Sep 2016 04:38:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: import cffi/cb6147abe7af
Message-ID: <57cc07b9.81a2c20a.d8de6.aee3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86858:c224171fc33e
Date: 2016-09-04 13:37 +0200
http://bitbucket.org/pypy/pypy/changeset/c224171fc33e/

Log:	import cffi/cb6147abe7af

diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1009,6 +1009,12 @@
                         myref is not None and myref is other())
                 def __ne__(self, other):
                     return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
             self._weakref_cache_ref = {}, MyRef
         weak_cache, MyRef = self._weakref_cache_ref
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -154,7 +154,10 @@
         from cffi.setuptools_ext import _set_py_limited_api
         try:
             import setuptools
-            orig_version = setuptools.__version__
+        except ImportError as e:
+            py.test.skip(str(e))
+        orig_version = setuptools.__version__
+        try:
             setuptools.__version__ = '26.0.0'
             from setuptools import Extension
 

From pypy.commits at gmail.com  Sun Sep  4 07:48:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Sep 2016 04:48:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: These specialize.arg() are not valid,
 because callers inside runicode
Message-ID: <57cc0a08.68adc20a.966c2.a0e8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86859:1f8c8d955c04
Date: 2016-09-04 13:47 +0200
http://bitbucket.org/pypy/pypy/changeset/1f8c8d955c04/

Log:	These specialize.arg() are not valid, because callers inside
	runicode may not give a constant at all. See updated test.

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -145,7 +145,6 @@
 _invalid_byte_3_of_4 = _invalid_cont_byte
 _invalid_byte_4_of_4 = _invalid_cont_byte
 
- at specialize.arg(2)
 def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xe0 and ordch2 < 0xa0)
@@ -157,7 +156,6 @@
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
- at specialize.arg(5)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -823,9 +823,15 @@
         def f(x):
 
             s1 = "".join(["\xd7\x90\xd6\x96\xeb\x96\x95\xf0\x90\x91\x93"] * x)
-            u, consumed = runicode.str_decode_utf_8(s1, len(s1), True)
-            s2 = runicode.unicode_encode_utf_8(u, len(u), True)
-            return s1 == s2
+            u, consumed = runicode.str_decode_utf_8(s1, len(s1), 'strict',
+                                                    allow_surrogates=True)
+            s2 = runicode.unicode_encode_utf_8(u, len(u), 'strict',
+                                                    allow_surrogates=True)
+            u3, consumed3 = runicode.str_decode_utf_8(s1, len(s1), 'strict',
+                                                    allow_surrogates=False)
+            s3 = runicode.unicode_encode_utf_8(u3, len(u3), 'strict',
+                                                    allow_surrogates=False)
+            return s1 == s2 == s3
         res = interpret(f, [2])
         assert res
 

From pypy.commits at gmail.com  Sun Sep  4 08:25:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Sep 2016 05:25:20 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: Fix for test_zjit by
 explicitly disabling the W_MemoryView case here
Message-ID: <57cc12b0.04e21c0a.17845.7d95@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: buffer-interface
Changeset: r86860:73e7b767d5f7
Date: 2016-09-04 14:19 +0200
http://bitbucket.org/pypy/pypy/changeset/73e7b767d5f7/

Log:	Fix for test_zjit by explicitly disabling the W_MemoryView case here

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -97,7 +97,16 @@
             print "ERROR: did not implement return type for interpreter"
             raise TypeError(w_res)
 
-        if self.graph is None:
+        if self.graph is not None:
+            return
+
+        from pypy.module.micronumpy import ctors
+        def unimplemented(*args):
+            "Only for W_MemoryView objects, which are not compiled in"
+            raise NotImplementedError
+        prev_3118 = ctors._array_from_buffer_3118
+        ctors._array_from_buffer_3118 = unimplemented
+        try:
             interp, graph = self.meta_interp(f, [0],
                                              listops=True,
                                              listcomp=True,
@@ -107,6 +116,8 @@
                                              vec=True)
             self.__class__.interp = interp
             self.__class__.graph = graph
+        finally:
+            ctors._array_from_buffer_3118 = prev_3118
 
     def check_vectorized(self, expected_tried, expected_success):
         profiler = get_profiler()

From pypy.commits at gmail.com  Sun Sep  4 08:50:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Sep 2016 05:50:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Why final=False here?
Message-ID: <57cc188d.041f1c0a.fbb56.5e44@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86861:50051cb90662
Date: 2016-09-04 14:39 +0200
http://bitbucket.org/pypy/pypy/changeset/50051cb90662/

Log:	Why final=False here?

diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -30,7 +30,7 @@
         assert value is not None
         result = UnicodeBuilder(len(value))
         self.rstr_decode_utf_8(
-            value, len(value), 'strict', final=False,
+            value, len(value), 'strict', final=True,
             errorhandler=self.ll_raise_unicode_exception_decode,
             allow_surrogates=False, result=result)
         return self.ll.llunicode(result.build())
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -162,6 +162,18 @@
 
         assert self.ll_to_string(self.interpret(f, [0])) == f(0)
 
+    def test_unicode_decode_final(self):
+        strings = ['\xc3', '']
+        def f(n):
+            try:
+                strings[n].decode('utf-8')
+            except UnicodeDecodeError:
+                return True
+            return False
+
+        assert f(0)
+        assert self.interpret(f, [0])
+
     def test_utf_8_decoding_annotation(self):
         from rpython.rlib.runicode import str_decode_utf_8
         def errorhandler(errors, encoding, msg, s,

From pypy.commits at gmail.com  Sun Sep  4 08:50:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Sep 2016 05:50:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: Another attempt at fixing the original
 problem
Message-ID: <57cc1891.94071c0a.8c4bb.558a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86862:74b4b27aaa7b
Date: 2016-09-04 14:49 +0200
http://bitbucket.org/pypy/pypy/changeset/74b4b27aaa7b/

Log:	Another attempt at fixing the original problem

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1,5 +1,5 @@
 import sys
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, enforceargs
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
@@ -145,17 +145,21 @@
 _invalid_byte_3_of_4 = _invalid_cont_byte
 _invalid_byte_4_of_4 = _invalid_cont_byte
 
+ at enforceargs(allow_surrogates=bool)
 def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xe0 and ordch2 < 0xa0)
             # surrogates shouldn't be valid UTF-8!
-            or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f))
+            or (ordch1 == 0xed and ordch2 > 0x9f and not allow_surrogates))
 
 def _invalid_byte_2_of_4(ordch1, ordch2):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(4)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:
@@ -328,6 +332,9 @@
     return unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(3)
 def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                               allow_surrogates=False):
     assert(size >= 0)
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -55,7 +55,7 @@
                 s = s.encode(encoding)
         except LookupError as e:
             py.test.skip(e)
-        result, consumed = decoder(s, len(s), True)
+        result, consumed = decoder(s, len(s), 'strict', final=True)
         assert consumed == len(s)
         self.typeequals(trueresult, result)
 
@@ -69,7 +69,7 @@
                 s = s.decode(encoding)
         except LookupError as e:
             py.test.skip(e)
-        result = encoder(s, len(s), True)
+        result = encoder(s, len(s), 'strict')
         self.typeequals(trueresult, result)
 
     def checkencodeerror(self, s, encoding, start, stop):
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -35,7 +35,8 @@
             allow_surrogates=False, result=result)
         return self.ll.llunicode(result.build())
 
-    def ll_raise_unicode_exception_decode(self, errors, encoding, msg, s,
+    @staticmethod
+    def ll_raise_unicode_exception_decode(errors, encoding, msg, s,
                                        startingpos, endingpos):
         raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
 
@@ -411,7 +412,8 @@
             allow_surrogates=False)
         return self.ll.llstr(bytes)
 
-    def ll_raise_unicode_exception_encode(self, errors, encoding, msg, u,
+    @staticmethod
+    def ll_raise_unicode_exception_encode(errors, encoding, msg, u,
                                           startingpos, endingpos):
         raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 

From pypy.commits at gmail.com  Sun Sep  4 11:00:27 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 04 Sep 2016 08:00:27 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Issue #2388: the problem is obscure
 interaction with a different call (I
Message-ID: <57cc370b.a6a5c20a.90c9b.e2d3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86863:145c1bd68950
Date: 2016-09-04 17:55 +0300
http://bitbucket.org/pypy/pypy/changeset/145c1bd68950/

Log:	Issue #2388: the problem is obscure interaction with a different
	call (I don't know which one) with the signature (string, float),
	which was considered as more general than the signature (string,
	int) of os.access(). (grafted from
	7d6c66b1477085a45a3a64056a010adf3e018924)

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -17,7 +17,7 @@
 from rpython.flowspace.model import Variable, Constant, const
 from rpython.flowspace.operation import op
 from rpython.rlib import rarithmetic
-from rpython.annotator.model import AnnotatorError
+from rpython.annotator.model import AnnotatorError, TLS
 
 BINARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values()
                         if oper.dispatch == 2])
@@ -436,6 +436,11 @@
 class __extend__(pairtype(SomeFloat, SomeFloat)):
 
     def union((flt1, flt2)):
+        if not TLS.allow_int_to_float:
+            # in this mode, if one of the two is actually the
+            # subclass SomeInteger, complain
+            if isinstance(flt1, SomeInteger) or isinstance(flt2, SomeInteger):
+                raise UnionError(flt1, flt2)
         return SomeFloat()
 
     add = sub = mul = union
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -44,6 +44,7 @@
     # A global attribute :-(  Patch it with 'True' to enable checking of
     # the no_nul attribute...
     check_str_without_nul = False
+    allow_int_to_float = True
 TLS = State()
 
 class SomeObject(object):
diff --git a/rpython/rlib/rmarshal.py b/rpython/rlib/rmarshal.py
--- a/rpython/rlib/rmarshal.py
+++ b/rpython/rlib/rmarshal.py
@@ -346,11 +346,15 @@
     # on s_bigger.  It relies on the fact that s_bigger was created with
     # an expression like 'annotation([s_item])' which returns a ListDef with
     # no bookkeeper, on which side-effects are not allowed.
+    saved = annmodel.TLS.allow_int_to_float
     try:
+        annmodel.TLS.allow_int_to_float = False
         s_union = annmodel.unionof(s_bigger, s_smaller)
         return s_bigger.contains(s_union)
     except (annmodel.UnionError, TooLateForChange):
         return False
+    finally:
+        annmodel.TLS.allow_int_to_float = saved
 
 
 class __extend__(pairtype(MTag, annmodel.SomeObject)):
diff --git a/rpython/rlib/test/test_rmarshal.py b/rpython/rlib/test/test_rmarshal.py
--- a/rpython/rlib/test/test_rmarshal.py
+++ b/rpython/rlib/test/test_rmarshal.py
@@ -128,10 +128,12 @@
 
 def test_llinterp_marshal():
     from rpython.rtyper.test.test_llinterp import interpret
-    marshaller = get_marshaller([(int, str, float)])
+    marshaller1 = get_marshaller([(int, str, float)])
+    marshaller2 = get_marshaller([(int, str, int)])
     def f():
         buf = []
-        marshaller(buf, [(5, "hello", -0.5), (7, "world", 1E100)])
+        marshaller1(buf, [(5, "hello", -0.5), (7, "world", 1E100)])
+        marshaller2(buf, [(5, "hello", 1)])
         return ''.join(buf)
     res = interpret(f, [])
     res = ''.join(res.chars)
@@ -139,14 +141,20 @@
         assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00'
                        's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00'
                        'i\x07\x00\x00\x00s\x05\x00\x00\x00world'
-                       'f\x061e+100')
+                       'f\x061e+100'
+                       '[\x01\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00'
+                       's\x05\x00\x00\x00helloi\x01\x00\x00\x00')
     else:
         assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00'
                        'I\x05\x00\x00\x00\x00\x00\x00\x00'
                        's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00'
                        'I\x07\x00\x00\x00\x00\x00\x00\x00'
                        's\x05\x00\x00\x00world'
-                       'f\x061e+100')
+                       'f\x061e+100'
+                       '[\x01\x00\x00\x00(\x03\x00\x00\x00'
+                       'I\x05\x00\x00\x00\x00\x00\x00\x00'
+                       's\x05\x00\x00\x00hello'
+                       'I\x01\x00\x00\x00\x00\x00\x00\x00')
 
 def test_llinterp_unmarshal():
     from rpython.rtyper.test.test_llinterp import interpret

From pypy.commits at gmail.com  Sun Sep  4 11:00:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 04 Sep 2016 08:00:29 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Issue #2389: the custom error
 handler may return a 'pos' that is smaller
Message-ID: <57cc370d.45c8c20a.4d79b.ed10@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86864:110c3a4539c8
Date: 2016-09-04 17:55 +0300
http://bitbucket.org/pypy/pypy/changeset/110c3a4539c8/

Log:	Issue #2389: the custom error handler may return a 'pos' that is
	smaller than 'size', in which case we need to continue looping
	(grafted from e9dd5882eed653226e8108e4182949b3c58c5de7)

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -157,22 +157,26 @@
         if pos + n > size:
             if not final:
                 break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
             charsleft = size - pos - 1 # either 0, 1, 2
-            # note: when we get the 'unexpected end of data' we don't care
-            # about the pos anymore and we just ignore the value
+            # note: when we get the 'unexpected end of data' we need
+            # to care about the pos returned; it can be lower than size,
+            # in case we need to continue running this loop
             if not charsleft:
                 # there's only the start byte and nothing else
                 r, pos = errorhandler(errors, 'utf8',
                                       'unexpected end of data',
                                       s, pos, pos+1)
                 result.append(r)
-                break
+                continue
             ordch2 = ord(s[pos+1])
             if n == 3:
                 # 3-bytes seq with only a continuation byte
                 if (ordch2>>6 != 0x2 or   # 0b10
-                    (ordch1 == 0xe0 and ordch2 < 0xa0)):
-                    # or (ordch1 == 0xed and ordch2 > 0x9f)
+                    (ordch1 == 0xe0 and ordch2 < 0xa0)
+                or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f)
+                    ):
                     # second byte invalid, take the first and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
@@ -185,7 +189,7 @@
                                       'unexpected end of data',
                                       s, pos, pos+2)
                     result.append(r)
-                    break
+                    continue
             elif n == 4:
                 # 4-bytes seq with 1 or 2 continuation bytes
                 if (ordch2>>6 != 0x2 or    # 0b10
@@ -210,7 +214,8 @@
                                       'unexpected end of data',
                                       s, pos, pos+charsleft+1)
                     result.append(r)
-                    break
+                    continue
+            raise AssertionError("unreachable")
 
         if n == 0:
             r, pos = errorhandler(errors, 'utf8',
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -289,6 +289,12 @@
     def __init__(self):
         self.decoder = self.getdecoder('utf-8')
 
+    def custom_replace(self, errors, encoding, msg, s, startingpos, endingpos):
+        assert errors == 'custom'
+        # returns FOO, but consumes only one character (not up to endingpos)
+        FOO = u'\u1234'
+        return FOO, startingpos + 1
+
     def to_bytestring(self, bytes):
         return ''.join(chr(int(c, 16)) for c in bytes.split())
 
@@ -309,6 +315,7 @@
         E.g. <80> is a continuation byte and can appear only after a start byte.
         """
         FFFD = u'\ufffd'
+        FOO = u'\u1234'
         for byte in '\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF':
             py.test.raises(UnicodeDecodeError, self.decoder, byte, 1, None, final=True)
             self.checkdecodeerror(byte, 'utf-8', 0, 1, addstuff=False,
@@ -320,6 +327,11 @@
             assert self.decoder(byte, 1, 'ignore', final=True) == (u'', 1)
             assert (self.decoder('aaaa' + byte + 'bbbb', 9, 'ignore',
                         final=True) == (u'aaaabbbb', 9))
+            assert self.decoder(byte, 1, 'custom', final=True,
+                        errorhandler=self.custom_replace) == (FOO, 1)
+            assert (self.decoder('aaaa' + byte + 'bbbb', 9, 'custom',
+                        final=True, errorhandler=self.custom_replace) ==
+                        (u'aaaa'+ FOO + u'bbbb', 9))
 
     def test_unexpected_end_of_data(self):
         """
@@ -343,6 +355,7 @@
             'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF'
         ]
         FFFD = u'\ufffd'
+        FOO = u'\u1234'
         for seq in sequences:
             seq = self.to_bytestring(seq)
             py.test.raises(UnicodeDecodeError, self.decoder, seq, len(seq),
@@ -358,6 +371,12 @@
                                 ) == (u'', len(seq))
             assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'ignore',
                         final=True) == (u'aaaabbbb', len(seq) + 8))
+            assert (self.decoder(seq, len(seq), 'custom', final=True,
+                        errorhandler=self.custom_replace) == 
+                        (FOO * len(seq), len(seq)))
+            assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'custom',
+                        final=True, errorhandler=self.custom_replace) ==
+                        (u'aaaa'+ FOO * len(seq) + u'bbbb', len(seq) + 8))
 
     def test_invalid_cb_for_2bytes_seq(self):
         """

From pypy.commits at gmail.com  Sun Sep  4 11:00:31 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 04 Sep 2016 08:00:31 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Move the bit checking inside
 helpers, share it from the two places
Message-ID: <57cc370f.8bd51c0a.5659f.6d55@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86865:dfb434142f63
Date: 2016-09-04 17:55 +0300
http://bitbucket.org/pypy/pypy/changeset/dfb434142f63/

Log:	Move the bit checking inside helpers, share it from the two places
	(grafted from ee3a2fbec01afa109be9414e105ea7250a7e1b24)

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -137,6 +137,25 @@
                                  result=result)
     return result.build(), pos
 
+def _invalid_cont_byte(ordch):
+    return ordch>>6 != 0x2    # 0b10
+
+_invalid_byte_2_of_2 = _invalid_cont_byte
+_invalid_byte_3_of_3 = _invalid_cont_byte
+_invalid_byte_3_of_4 = _invalid_cont_byte
+_invalid_byte_4_of_4 = _invalid_cont_byte
+
+def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xe0 and ordch2 < 0xa0)
+            # surrogates shouldn't be valid UTF-8!
+            or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f))
+
+def _invalid_byte_2_of_4(ordch1, ordch2):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xf0 and ordch2 < 0x90) or
+            (ordch1 == 0xf4 and ordch2 > 0x8f))
+
 @specialize.argtype(6)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
@@ -173,10 +192,7 @@
             ordch2 = ord(s[pos+1])
             if n == 3:
                 # 3-bytes seq with only a continuation byte
-                if (ordch2>>6 != 0x2 or   # 0b10
-                    (ordch1 == 0xe0 and ordch2 < 0xa0)
-                or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f)
-                    ):
+                if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                     # second byte invalid, take the first and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
@@ -192,16 +208,14 @@
                     continue
             elif n == 4:
                 # 4-bytes seq with 1 or 2 continuation bytes
-                if (ordch2>>6 != 0x2 or    # 0b10
-                    (ordch1 == 0xf0 and ordch2 < 0x90) or
-                    (ordch1 == 0xf4 and ordch2 > 0x8f)):
+                if _invalid_byte_2_of_4(ordch1, ordch2):
                     # second byte invalid, take the first and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
                                           s, pos, pos+1)
                     result.append(r)
                     continue
-                elif charsleft == 2 and ord(s[pos+2])>>6 != 0x2:   # 0b10
+                elif charsleft == 2 and _invalid_byte_3_of_4(ord(s[pos+2])):
                     # third byte invalid, take the first two and continue
                     r, pos = errorhandler(errors, 'utf8',
                                           'invalid continuation byte',
@@ -228,7 +242,7 @@
 
         elif n == 2:
             ordch2 = ord(s[pos+1])
-            if ordch2>>6 != 0x2:   # 0b10
+            if _invalid_byte_2_of_2(ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
@@ -242,17 +256,13 @@
         elif n == 3:
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
-            if (ordch2>>6 != 0x2 or    # 0b10
-                (ordch1 == 0xe0 and ordch2 < 0xa0)
-                # surrogates shouldn't be valid UTF-8!
-                or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f)
-                ):
+            if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif ordch3>>6 != 0x2:     # 0b10
+            elif _invalid_byte_3_of_3(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
@@ -268,21 +278,19 @@
             ordch2 = ord(s[pos+1])
             ordch3 = ord(s[pos+2])
             ordch4 = ord(s[pos+3])
-            if (ordch2>>6 != 0x2 or     # 0b10
-                (ordch1 == 0xf0 and ordch2 < 0x90) or
-                (ordch1 == 0xf4 and ordch2 > 0x8f)):
+            if _invalid_byte_2_of_4(ordch1, ordch2):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+1)
                 result.append(r)
                 continue
-            elif ordch3>>6 != 0x2:     # 0b10
+            elif _invalid_byte_3_of_4(ordch3):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+2)
                 result.append(r)
                 continue
-            elif ordch4>>6 != 0x2:     # 0b10
+            elif _invalid_byte_4_of_4(ordch4):
                 r, pos = errorhandler(errors, 'utf8',
                                       'invalid continuation byte',
                                       s, pos, pos+3)

From pypy.commits at gmail.com  Sun Sep  4 11:00:33 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 04 Sep 2016 08:00:33 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: import cffi/b81ca61b6de6
Message-ID: <57cc3711.68adc20a.966c2.df9a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86866:b29dd462f6ed
Date: 2016-09-04 17:55 +0300
http://bitbucket.org/pypy/pypy/changeset/b29dd462f6ed/

Log:	import cffi/b81ca61b6de6 (grafted from
	fdd46325fe3d2c1d29496931411c453c0af576a2)

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.1"
+__version_info__ = (1, 8, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,37 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1479,6 +1479,7 @@
             assert p1[0] == 123
             seen.append(1)
         q = ffi.gc(p, destructor)
+        assert ffi.typeof(q) is ffi.typeof(p)
         import gc; gc.collect()
         assert seen == []
         del q
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -149,3 +149,25 @@
         p = snip_setuptools_verify2.C.getpwuid(0)
         assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
         ''')
+
+    def test_set_py_limited_api(self):
+        from cffi.setuptools_ext import _set_py_limited_api
+        try:
+            import setuptools
+            orig_version = setuptools.__version__
+            setuptools.__version__ = '26.0.0'
+            from setuptools import Extension
+
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+            setuptools.__version__ = '25.0'
+            kwds = _set_py_limited_api(Extension, {})
+            assert not kwds
+
+            setuptools.__version__ = 'development'
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+        finally:
+            setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1975,9 +1975,9 @@
 
 def test_function_returns_partial_struct():
     ffi = FFI()
-    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    ffi.cdef("struct aaa { int a; ...; }; struct aaa f1(int);")
     lib = verify(ffi, "test_function_returns_partial_struct", """
-        struct a { int b, a, c; };
-        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+        struct aaa { int b, a, c; };
+        static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52

From pypy.commits at gmail.com  Sun Sep  4 11:00:34 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 04 Sep 2016 08:00:34 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: import cffi/cb6147abe7af
Message-ID: <57cc3712.8628c20a.f6b0b.e4be@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86867:dfa482ad3f4c
Date: 2016-09-04 17:55 +0300
http://bitbucket.org/pypy/pypy/changeset/dfa482ad3f4c/

Log:	import cffi/cb6147abe7af (grafted from
	c224171fc33e53442553735d23896c4cd2d51b34)

diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1009,6 +1009,12 @@
                         myref is not None and myref is other())
                 def __ne__(self, other):
                     return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
             self._weakref_cache_ref = {}, MyRef
         weak_cache, MyRef = self._weakref_cache_ref
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -154,7 +154,10 @@
         from cffi.setuptools_ext import _set_py_limited_api
         try:
             import setuptools
-            orig_version = setuptools.__version__
+        except ImportError as e:
+            py.test.skip(str(e))
+        orig_version = setuptools.__version__
+        try:
             setuptools.__version__ = '26.0.0'
             from setuptools import Extension
 

From pypy.commits at gmail.com  Sun Sep  4 11:28:41 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 04 Sep 2016 08:28:41 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Disallow unions of char and
 unichar, since they make the annotator inconsistent
Message-ID: <57cc3da9.8fc51c0a.4b372.9990@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86868:ec11787ac32c
Date: 2016-09-04 16:27 +0100
http://bitbucket.org/pypy/pypy/changeset/ec11787ac32c/

Log:	Disallow unions of char and unichar, since they make the annotator
	inconsistent

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -378,11 +378,6 @@
         return SomeChar(no_nul=no_nul)
 
 
-class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint),
-                 pairtype(SomeUnicodeCodePoint, SomeChar)):
-    def union((uchr1, uchr2)):
-        return SomeUnicodeCodePoint()
-
 class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)):
     def union((uchr1, uchr2)):
         no_nul = uchr1.no_nul and uchr2.no_nul
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -3453,17 +3453,6 @@
         assert isinstance(s, annmodel.SomeUnicodeString)
         assert s.no_nul
 
-    def test_unicode_char(self):
-        def f(x, i):
-            for c in x:
-                if c == i:
-                    return c
-            return 'x'
-
-        a = self.RPythonAnnotator()
-        s = a.build_types(f, [unicode, str])
-        assert isinstance(s, annmodel.SomeUnicodeCodePoint)
-
     def test_strformatting_unicode(self):
         def f(x):
             return '%s' % unichr(x)
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -727,6 +727,11 @@
         count = 0
         n = end - start
         m = len(s2.chars)
+        tp = typeOf(s1)
+        if tp == string_repr.lowleveltype or tp == Char:
+            NUL = '\0'
+        else:
+            NUL = u'\0'
 
         if m == 0:
             if mode == FAST_COUNT:
@@ -771,7 +776,7 @@
                     if i + m < len(s1.chars):
                         c = s1.chars[i + m]
                     else:
-                        c = '\0'
+                        c = NUL
                     if not bloom(mask, c):
                         i += m
                     else:
@@ -780,7 +785,7 @@
                     if i + m < len(s1.chars):
                         c = s1.chars[i + m]
                     else:
-                        c = '\0'
+                        c = NUL
                     if not bloom(mask, c):
                         i += m
         else:
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -715,9 +715,7 @@
         return hop.genop('cast_unichar_to_int', vlist, resulttype=Signed)
 
 
-class __extend__(pairtype(AbstractUniCharRepr, AbstractUniCharRepr),
-                 pairtype(AbstractCharRepr, AbstractUniCharRepr),
-                 pairtype(AbstractUniCharRepr, AbstractCharRepr)):
+class __extend__(pairtype(AbstractUniCharRepr, AbstractUniCharRepr)):
     def rtype_eq(_, hop): return _rtype_unchr_compare_template(hop, 'eq')
     def rtype_ne(_, hop): return _rtype_unchr_compare_template(hop, 'ne')
     def rtype_lt(_, hop): return _rtype_unchr_compare_template_ord(hop, 'lt')
@@ -737,12 +735,7 @@
     vlist = hop.inputargs(*hop.args_r)
     vlist2 = []
     for v in vlist:
-        if v.concretetype == lltype.Char:
-            v = hop.genop('cast_char_to_int', [v], resulttype=lltype.Signed)
-        elif v.concretetype == lltype.UniChar:
-            v = hop.genop('cast_unichar_to_int', [v], resulttype=lltype.Signed)
-        else:
-            assert 0, v.concretetype
+        v = hop.genop('cast_unichar_to_int', [v], resulttype=lltype.Signed)
         vlist2.append(v)
     return hop.genop('int_' + func, vlist2, resulttype=Bool)
 
@@ -767,11 +760,6 @@
             return llops.gendirectcall(r_from.ll.ll_stritem_nonneg, v, c_zero)
         return NotImplemented
 
-class __extend__(pairtype(AbstractCharRepr, AbstractUniCharRepr)):
-    def convert_from_to((r_from, r_to), v, llops):
-        v2 = llops.genop('cast_char_to_int', [v], resulttype=Signed)
-        return llops.genop('cast_int_to_unichar', [v2], resulttype=UniChar)
-
 # ____________________________________________________________
 #
 #  Iteration.
diff --git a/rpython/rtyper/test/test_rlist.py b/rpython/rtyper/test/test_rlist.py
--- a/rpython/rtyper/test/test_rlist.py
+++ b/rpython/rtyper/test/test_rlist.py
@@ -1321,7 +1321,7 @@
 
     def test_unicharlist_extension_1(self):
         def f(n):
-            s = 'hello%d' % n
+            s = u'hello%d' % n
             l = [u'a', u'b']
             l += s
             return ''.join([chr(ord(c)) for c in l])
@@ -1348,7 +1348,7 @@
 
     def test_unicharlist_extension_2(self):
         def f(n, i):
-            s = 'hello%d' % n
+            s = u'hello%d' % n
             assert 0 <= i <= len(s)
             l = [u'a', u'b']
             l += s[i:]
@@ -1377,7 +1377,7 @@
 
     def test_unicharlist_extension_3(self):
         def f(n, i, j):
-            s = 'hello%d' % n
+            s = u'hello%d' % n
             assert 0 <= i <= j <= len(s)
             l = [u'a', u'b']
             l += s[i:j]
@@ -1396,7 +1396,7 @@
 
     def test_unicharlist_extension_4(self):
         def f(n):
-            s = 'hello%d' % n
+            s = u'hello%d' % n
             l = [u'a', u'b']
             l += s[:-1]
             return ''.join([chr(ord(c)) for c in l])
@@ -1416,7 +1416,7 @@
     def test_unicharlist_extension_5(self):
         def f(count):
             l = [u'a', u'b']
-            l += '.' * count     # NON-UNICODE-char * count
+            l += u'.' * count
             return ''.join([chr(ord(c)) for c in l])
         res = self.interpret(f, [7])
         assert self.ll_to_string(res) == 'ab.......'
diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py
--- a/rpython/rtyper/test/test_rstr.py
+++ b/rpython/rtyper/test/test_rstr.py
@@ -1120,7 +1120,7 @@
             for i, x in enumerate(s):
                 if i == n:
                     return x
-            return 'x'
+            return const('x')
         res = self.interpret(fn, [2])
         assert res == 'c'
 
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -216,33 +216,6 @@
             return d[c]
         assert self.interpret(fn, [u'\u03b1']) == 42
 
-    def test_convert_char_to_unichar(self):
-        def g(c):
-            return ord(c)
-        def fn(n):
-            if n < 0:
-                c = unichr(-n)
-            else:
-                c = chr(n)
-            return g(c)
-        assert self.interpret(fn, [65]) == 65
-        assert self.interpret(fn, [-5555]) == 5555
-
-    def test_char_unichar_eq(self):
-        def fn(c1, c2):
-            return c1 == c2
-        assert self.interpret(fn, [u'(', '(']) == True
-        assert self.interpret(fn, [u'\u1028', '(']) == False
-        assert self.interpret(fn, ['(', u'(']) == True
-        assert self.interpret(fn, ['(', u'\u1028']) == False
-
-    def test_char_unichar_eq_2(self):
-        def fn(c1):
-            return c1 == 'X'
-        assert self.interpret(fn, [u'(']) == False
-        assert self.interpret(fn, [u'\u1058']) == False
-        assert self.interpret(fn, [u'X']) == True
-
     def test_strformat_unicode_arg(self):
         const = self.const
         def percentS(s, i):

From pypy.commits at gmail.com  Sun Sep  4 11:48:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Sep 2016 08:48:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: Explicitly detect that we found OpenSSL
 1.1, and abort with a message
Message-ID: <57cc425d.87941c0a.801f7.8e9c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86869:48ca2469fc30
Date: 2016-09-04 17:48 +0200
http://bitbucket.org/pypy/pypy/changeset/48ca2469fc30/

Log:	Explicitly detect that we found OpenSSL 1.1, and abort with a
	message as explicit as possible, hopefully better than an obscure
	crash soon

diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,6 +97,21 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
+if OPENSSL_VERSION_NUMBER >= 0x10100000:
+    eci.pre_include_bits = ()
+    eci.post_include_bits = ()
+    raise Exception("""OpenSSL version >= 1.1 not supported yet.
+
+    This program requires OpenSSL version 1.0.x, and may also
+    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
+    some work to update to; contributions are welcome.  Sorry,
+    you need to install an older version of OpenSSL for now.
+    Make sure this older version is the one picked up by this
+    program when it runs the compiler.
+    
+    This is the configuration used: %r""" % (eci,))
+
+
 class CConfig:
     _compilation_info_ = eci
 

From pypy.commits at gmail.com  Sun Sep  4 14:11:46 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 04 Sep 2016 11:11:46 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Explicitly detect that we found
 OpenSSL 1.1, and abort with a message
Message-ID: <57cc63e2.21b0c20a.5d2be.8e11@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86870:050d84dd7899
Date: 2016-09-04 21:10 +0300
http://bitbucket.org/pypy/pypy/changeset/050d84dd7899/

Log:	Explicitly detect that we found OpenSSL 1.1, and abort with a
	message as explicit as possible, hopefully better than an obscure
	crash soon (grafted from 48ca2469fc30053c39d92b8d4ccef6e82800fb57)

diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,6 +97,21 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
+if OPENSSL_VERSION_NUMBER >= 0x10100000:
+    eci.pre_include_bits = ()
+    eci.post_include_bits = ()
+    raise Exception("""OpenSSL version >= 1.1 not supported yet.
+
+    This program requires OpenSSL version 1.0.x, and may also
+    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
+    some work to update to; contributions are welcome.  Sorry,
+    you need to install an older version of OpenSSL for now.
+    Make sure this older version is the one picked up by this
+    program when it runs the compiler.
+    
+    This is the configuration used: %r""" % (eci,))
+
+
 class CConfig:
     _compilation_info_ = eci
 

From pypy.commits at gmail.com  Sun Sep  4 14:31:55 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 04 Sep 2016 11:31:55 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Add test for constants
Message-ID: <57cc689b.434bc20a.b8022.3185@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86871:6cee69266cee
Date: 2016-09-04 19:31 +0100
http://bitbucket.org/pypy/pypy/changeset/6cee69266cee/

Log:	Add test for constants

diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -1,6 +1,6 @@
 import pytest
 
-from hypothesis import given
+from hypothesis import given, assume, example
 from hypothesis import strategies as st
 
 from rpython.flowspace.model import Variable
@@ -143,16 +143,21 @@
 def compatible_pair(pair_s):
     return compatible(*pair_s)
 
-st_float = st.just(SomeFloat()) | st.builds(const_float, st.floats())
-st_int = st.one_of(st.builds(SomeInteger, st.booleans(), st.booleans()),
-                   st.builds(const_int, st.integers()))
+st_const_float = st.builds(const_float, st.floats())
+st_float = st.just(SomeFloat()) | st_const_float
+st_const_int = st.builds(const_int, st.integers())
+st_int = st.builds(SomeInteger, st.booleans(), st.booleans()) | st_const_int
+st_const_bool = st.sampled_from([s_True, s_False])
 st_bool = st.sampled_from([s_Bool, s_True, s_False])
 st_numeric = st.one_of(st_float, st_int, st_bool)
-st_str = (st.builds(SomeString, st.booleans(), st.booleans())
-          | st.builds(const_str, st.binary()))
+st_const_str = st.builds(const_str, st.binary())
+st_str = st.builds(SomeString, st.booleans(), st.booleans()) | st_const_str
+st_const_unicode = st.builds(const_unicode, st.text())
 st_unicode = (st.builds(SomeUnicodeString, st.booleans(), st.booleans())
-              | st.builds(const_unicode, st.text()))
+              | st_const_unicode)
 st_simple = st.one_of(st_numeric, st_str, st_unicode, st.just(s_ImpossibleValue), st.just(s_None))
+st_const = st.one_of(st_const_float, st_const_int, st_const_bool,
+                     st_const_str, st_const_unicode, st.just(s_None))
 
 def valid_unions(st_ann):
     """From a strategy generating annotations, create a strategy returning
@@ -188,6 +193,11 @@
     s1, s2, s3 = t
     assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
 
+ at given(s_const=st_const, s_obj=st_annotation)
+def test_constants_are_atoms(s_const, s_obj):
+    assume(s_const.contains(s_obj))
+    assert s_const == s_obj or s_obj == s_ImpossibleValue
+
 
 def compile_function(function, annotation=[]):
     t = TranslationContext()

From pypy.commits at gmail.com  Sun Sep  4 14:34:59 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 04 Sep 2016 11:34:59 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: hg merge default
Message-ID: <57cc6953.45c8c20a.4d79b.2fc4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86872:be656798450c
Date: 2016-09-04 19:34 +0100
http://bitbucket.org/pypy/pypy/changeset/be656798450c/

Log:	hg merge default

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.1"
+__version_info__ = (1, 8, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,43 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.1-alpha0"
-#define PYPY_VERSION_NUM  0x05040100
+#define PYPY_VERSION "5.5.0-alpha0"
+#define PYPY_VERSION_NUM  0x05050000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -32,9 +32,11 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 5, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1479,6 +1479,7 @@
             assert p1[0] == 123
             seen.append(1)
         q = ffi.gc(p, destructor)
+        assert ffi.typeof(q) is ffi.typeof(p)
         import gc; gc.collect()
         assert seen == []
         del q
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -149,3 +149,28 @@
         p = snip_setuptools_verify2.C.getpwuid(0)
         assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
         ''')
+
+    def test_set_py_limited_api(self):
+        from cffi.setuptools_ext import _set_py_limited_api
+        try:
+            import setuptools
+        except ImportError as e:
+            py.test.skip(str(e))
+        orig_version = setuptools.__version__
+        try:
+            setuptools.__version__ = '26.0.0'
+            from setuptools import Extension
+
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+            setuptools.__version__ = '25.0'
+            kwds = _set_py_limited_api(Extension, {})
+            assert not kwds
+
+            setuptools.__version__ = 'development'
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+        finally:
+            setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1975,9 +1975,9 @@
 
 def test_function_returns_partial_struct():
     ffi = FFI()
-    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    ffi.cdef("struct aaa { int a; ...; }; struct aaa f1(int);")
     lib = verify(ffi, "test_function_returns_partial_struct", """
-        struct a { int b, a, c; };
-        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+        struct aaa { int b, a, c; };
+        static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,6 +97,21 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
+if OPENSSL_VERSION_NUMBER >= 0x10100000:
+    eci.pre_include_bits = ()
+    eci.post_include_bits = ()
+    raise Exception("""OpenSSL version >= 1.1 not supported yet.
+
+    This program requires OpenSSL version 1.0.x, and may also
+    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
+    some work to update to; contributions are welcome.  Sorry,
+    you need to install an older version of OpenSSL for now.
+    Make sure this older version is the one picked up by this
+    program when it runs the compiler.
+    
+    This is the configuration used: %r""" % (eci,))
+
+
 class CConfig:
     _compilation_info_ = eci
 
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1,5 +1,5 @@
 import sys
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, enforceargs
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
@@ -145,19 +145,21 @@
 _invalid_byte_3_of_4 = _invalid_cont_byte
 _invalid_byte_4_of_4 = _invalid_cont_byte
 
- at specialize.arg(2)
+ at enforceargs(allow_surrogates=bool)
 def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xe0 and ordch2 < 0xa0)
             # surrogates shouldn't be valid UTF-8!
-            or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f))
+            or (ordch1 == 0xed and ordch2 > 0x9f and not allow_surrogates))
 
 def _invalid_byte_2_of_4(ordch1, ordch2):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
- at specialize.arg(5)
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(4)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:
@@ -330,6 +332,9 @@
     return unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(3)
 def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                               allow_surrogates=False):
     assert(size >= 0)
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -55,7 +55,7 @@
                 s = s.encode(encoding)
         except LookupError as e:
             py.test.skip(e)
-        result, consumed = decoder(s, len(s), True)
+        result, consumed = decoder(s, len(s), 'strict', final=True)
         assert consumed == len(s)
         self.typeequals(trueresult, result)
 
@@ -69,7 +69,7 @@
                 s = s.decode(encoding)
         except LookupError as e:
             py.test.skip(e)
-        result = encoder(s, len(s), True)
+        result = encoder(s, len(s), 'strict')
         self.typeequals(trueresult, result)
 
     def checkencodeerror(self, s, encoding, start, stop):
@@ -823,9 +823,15 @@
         def f(x):
 
             s1 = "".join(["\xd7\x90\xd6\x96\xeb\x96\x95\xf0\x90\x91\x93"] * x)
-            u, consumed = runicode.str_decode_utf_8(s1, len(s1), True)
-            s2 = runicode.unicode_encode_utf_8(u, len(u), True)
-            return s1 == s2
+            u, consumed = runicode.str_decode_utf_8(s1, len(s1), 'strict',
+                                                    allow_surrogates=True)
+            s2 = runicode.unicode_encode_utf_8(u, len(u), 'strict',
+                                                    allow_surrogates=True)
+            u3, consumed3 = runicode.str_decode_utf_8(s1, len(s1), 'strict',
+                                                    allow_surrogates=False)
+            s3 = runicode.unicode_encode_utf_8(u3, len(u3), 'strict',
+                                                    allow_surrogates=False)
+            return s1 == s2 == s3
         res = interpret(f, [2])
         assert res
 
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -540,7 +540,7 @@
     def ll_ullong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
-        return llop.ullong_mod(UnsignedLongLong, x, y)
+        return ll_ullong_py_mod(x, y)
 
 @jit.dont_look_inside
 def ll_lllong_py_mod(x, y):
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -30,12 +30,13 @@
         assert value is not None
         result = UnicodeBuilder(len(value))
         self.rstr_decode_utf_8(
-            value, len(value), 'strict', final=False,
+            value, len(value), 'strict', final=True,
             errorhandler=self.ll_raise_unicode_exception_decode,
             allow_surrogates=False, result=result)
         return self.ll.llunicode(result.build())
 
-    def ll_raise_unicode_exception_decode(self, errors, encoding, msg, s,
+    @staticmethod
+    def ll_raise_unicode_exception_decode(errors, encoding, msg, s,
                                        startingpos, endingpos):
         raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
 
@@ -411,7 +412,8 @@
             allow_surrogates=False)
         return self.ll.llstr(bytes)
 
-    def ll_raise_unicode_exception_encode(self, errors, encoding, msg, u,
+    @staticmethod
+    def ll_raise_unicode_exception_encode(errors, encoding, msg, u,
                                           startingpos, endingpos):
         raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -162,6 +162,18 @@
 
         assert self.ll_to_string(self.interpret(f, [0])) == f(0)
 
+    def test_unicode_decode_final(self):
+        strings = ['\xc3', '']
+        def f(n):
+            try:
+                strings[n].decode('utf-8')
+            except UnicodeDecodeError:
+                return True
+            return False
+
+        assert f(0)
+        assert self.interpret(f, [0])
+
     def test_utf_8_decoding_annotation(self):
         from rpython.rlib.runicode import str_decode_utf_8
         def errorhandler(errors, encoding, msg, s,

From pypy.commits at gmail.com  Sun Sep  4 19:17:10 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 04 Sep 2016 16:17:10 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Explicitly forbid
 comparisons between str and unicode
Message-ID: <57ccab76.2146c20a.7f80b.6e31@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86873:e1f64f1f25cc
Date: 2016-09-05 00:16 +0100
http://bitbucket.org/pypy/pypy/changeset/e1f64f1f25cc/

Log:	Explicitly forbid comparisons between str and unicode

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -641,6 +641,20 @@
             result.const = str1.const + str2.const
         return result
 
+for cmp_op in [op.lt, op.le, op.eq, op.ne, op.gt, op.ge]:
+    @cmp_op.register(SomeUnicodeString, SomeString)
+    @cmp_op.register(SomeUnicodeString, SomeChar)
+    @cmp_op.register(SomeString, SomeUnicodeString)
+    @cmp_op.register(SomeChar, SomeUnicodeString)
+    @cmp_op.register(SomeUnicodeCodePoint, SomeString)
+    @cmp_op.register(SomeUnicodeCodePoint, SomeChar)
+    @cmp_op.register(SomeString, SomeUnicodeCodePoint)
+    @cmp_op.register(SomeChar, SomeUnicodeCodePoint)
+    def cmp_str_unicode(annotator, v1, v2):
+        raise AnnotatorError(
+            "Comparing byte strings with unicode strings is not RPython")
+
+
 class __extend__(pairtype(SomeInteger, SomeList)):
 
     def mul((int1, lst2)):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1367,7 +1367,7 @@
             oc = ord(ch)
 
             # Escape quotes
-            if quotes and (oc == quote or ch == '\\'):
+            if quotes and (oc == quote or ch == u'\\'):
                 result.append(STR('\\'))
                 result.append(CHR(oc))
                 pos += 1
@@ -1390,13 +1390,13 @@
                 pos -= 1
 
             # Map special whitespace to '\t', \n', '\r'
-            if ch == '\t':
+            if ch == u'\t':
                 result.append(STR('\\t'))
-            elif ch == '\n':
+            elif ch == u'\n':
                 result.append(STR('\\n'))
-            elif ch == '\r':
+            elif ch == u'\r':
                 result.append(STR('\\r'))
-            elif ch == '\\':
+            elif ch == u'\\':
                 result.append(STR('\\\\'))
 
             # Map non-printable or non-ascii to '\xhh' or '\uhhhh'

From pypy.commits at gmail.com  Sun Sep  4 22:25:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 04 Sep 2016 19:25:01 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Fix str vs unicode
 correctness issues
Message-ID: <57ccd77d.c3f0c20a.54624.a917@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86874:350957bdef41
Date: 2016-09-05 03:24 +0100
http://bitbucket.org/pypy/pypy/changeset/350957bdef41/

Log:	Fix str vs unicode correctness issues

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -231,16 +231,16 @@
             while True:
                 # Fast path for non-control chars. The loop always ends
                 # since the Py_UNICODE storage is NUL-terminated.
-                while i < size and line[start + i] > '\r':
+                while i < size and line[start + i] > u'\r':
                     i += 1
                 if i >= size:
                     return -1, size
                 ch = line[start + i]
                 i += 1
-                if ch == '\n':
+                if ch == u'\n':
                     return i, 0
-                if ch == '\r':
-                    if line[start + i] == '\n':
+                if ch == u'\r':
+                    if line[start + i] == u'\n':
                         return i + 1, 0
                     else:
                         return i, 0
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -184,11 +184,11 @@
                 except IndexError:
                     space = self.space
                     raise oefmt(space.w_ValueError, "incomplete format key")
-                if c == ')':
+                if c == const(')'):
                     pcount -= 1
                     if pcount == 0:
                         break
-                elif c == '(':
+                elif c == const('('):
                     pcount += 1
                 i += 1
             self.fmtpos = i + 1   # first character after ')'
@@ -203,7 +203,7 @@
             return space.getitem(self.w_valuedict, w_key)
 
         def parse_fmt(self):
-            if self.peekchr() == '(':
+            if self.peekchr() == const('('):
                 w_value = self.getmappingvalue(self.getmappingkey())
             else:
                 w_value = None
@@ -216,7 +216,7 @@
                 self.f_ljust = True
                 self.width = -self.width
 
-            if self.peekchr() == '.':
+            if self.peekchr() == const('.'):
                 self.forward()
                 self.prec = self.peel_num('prec', INT_MAX)
                 if self.prec < 0:
@@ -225,7 +225,7 @@
                 self.prec = -1
 
             c = self.peekchr()
-            if c == 'h' or c == 'l' or c == 'L':
+            if c == const('h') or c == const('l') or c == const('L'):
                 self.forward()
 
             return w_value
@@ -240,15 +240,15 @@
             self.f_zero  = False
             while True:
                 c = self.peekchr()
-                if c == '-':
+                if c == const('-'):
                     self.f_ljust = True
-                elif c == '+':
+                elif c == const('+'):
                     self.f_sign = True
-                elif c == ' ':
+                elif c == const(' '):
                     self.f_blank = True
-                elif c == '#':
+                elif c == const('#'):
                     self.f_alt = True
-                elif c == '0':
+                elif c == const('0'):
                     self.f_zero = True
                 else:
                     break
@@ -259,7 +259,7 @@
         def peel_num(self, name, maxval):
             space = self.space
             c = self.peekchr()
-            if c == '*':
+            if c == const('*'):
                 self.forward()
                 w_value = self.nextinputvalue()
                 if name == 'width':
@@ -293,7 +293,7 @@
                 fmt = self.fmt
                 i = i0 = self.fmtpos
                 while i < len(fmt):
-                    if fmt[i] == '%':
+                    if fmt[i] == const('%'):
                         break
                     i += 1
                 else:
@@ -306,7 +306,7 @@
                 w_value = self.parse_fmt()
                 c = self.peekchr()
                 self.forward()
-                if c == '%':
+                if c == const('%'):
                     self.std_wp(const('%'))
                     continue
                 if w_value is None:
@@ -315,7 +315,7 @@
                 # dispatch on the formatter
                 # (this turns into a switch after translation)
                 for c1 in FORMATTER_CHARS:
-                    if c == c1:
+                    if c == const(c1):
                         # 'c1' is an annotation constant here,
                         # so this getattr() is ok
                         do_fmt = getattr(self, 'fmt_' + c1)
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -45,6 +45,7 @@
 
 
 def make_template_formatting_class(for_unicode):
+    STR = unicode if for_unicode else str
     class TemplateFormatter(object):
         is_unicode = for_unicode
 
@@ -90,19 +91,19 @@
             while i < end:
                 c = s[i]
                 i += 1
-                if c == "{" or c == "}":
+                if c == STR("{") or c == STR("}"):
                     at_end = i == end
                     # Find escaped "{" and "}"
                     markup_follows = True
-                    if c == "}":
-                        if at_end or s[i] != "}":
+                    if c == STR("}"):
+                        if at_end or s[i] != STR("}"):
                             raise oefmt(space.w_ValueError, "Single '}'")
                         i += 1
                         markup_follows = False
-                    if c == "{":
+                    if c == STR("{"):
                         if at_end:
                             raise oefmt(space.w_ValueError, "Single '{'")
-                        if s[i] == "{":
+                        if s[i] == STR("{"):
                             i += 1
                             markup_follows = False
                     # Attach literal data, ending with { or }
@@ -124,10 +125,10 @@
                     recursive = False
                     while i < end:
                         c = s[i]
-                        if c == "{":
+                        if c == STR("{"):
                             recursive = True
                             nested += 1
-                        elif c == "}":
+                        elif c == STR("}"):
                             nested -= 1
                             if not nested:
                                 break
@@ -150,9 +151,9 @@
             i = start
             while i < end:
                 c = s[i]
-                if c == ":" or c == "!":
+                if c == STR(":") or c == STR("!"):
                     end_name = i
-                    if c == "!":
+                    if c == STR("!"):
                         i += 1
                         if i == end:
                             raise oefmt(self.space.w_ValueError,
@@ -160,7 +161,7 @@
                         conversion = s[i]
                         i += 1
                         if i < end:
-                            if s[i] != ':':
+                            if s[i] != STR(':'):
                                 raise oefmt(self.space.w_ValueError,
                                             "expected ':' after format "
                                             "specifier")
@@ -180,7 +181,7 @@
             end = len(name)
             while i < end:
                 c = name[i]
-                if c == "[" or c == ".":
+                if c == STR("[") or c == STR("."):
                     break
                 i += 1
             empty = not i
@@ -232,12 +233,12 @@
             i = start
             while i < end:
                 c = name[i]
-                if c == ".":
+                if c == STR("."):
                     i += 1
                     start = i
                     while i < end:
                         c = name[i]
-                        if c == "[" or c == ".":
+                        if c == STR("[") or c == STR("."):
                             break
                         i += 1
                     if start == i:
@@ -249,13 +250,13 @@
                     else:
                         self.parser_list_w.append(space.newtuple([
                             space.w_True, w_attr]))
-                elif c == "[":
+                elif c == STR("["):
                     got_bracket = False
                     i += 1
                     start = i
                     while i < end:
                         c = name[i]
-                        if c == "]":
+                        if c == STR("]"):
                             got_bracket = True
                             break
                         i += 1
@@ -284,7 +285,7 @@
             end = len(name)
             while i < end:
                 c = name[i]
-                if c == "[" or c == ".":
+                if c == STR("[") or c == STR("."):
                     break
                 i += 1
             if i == 0:
@@ -307,9 +308,9 @@
         def _convert(self, w_obj, conversion):
             space = self.space
             conv = conversion[0]
-            if conv == "r":
+            if conv == STR("r"):
                 return space.repr(w_obj)
-            elif conv == "s":
+            elif conv == STR("s"):
                 if self.is_unicode:
                     return space.call_function(space.w_unicode, w_obj)
                 return space.str(w_obj)
@@ -400,6 +401,7 @@
 LONG_DIGITS = string.digits + string.ascii_lowercase
 
 def make_formatting_class(for_unicode):
+    _lit = unicode if for_unicode else str
     class Formatter(BaseFormatter):
         """__format__ implementation for builtin types."""
 
@@ -412,22 +414,22 @@
             self.spec = spec
 
         def _is_alignment(self, c):
-            return (c == "<" or
-                    c == ">" or
-                    c == "=" or
-                    c == "^")
+            return (c == _lit("<") or
+                    c == _lit(">") or
+                    c == _lit("=") or
+                    c == _lit("^"))
 
         def _is_sign(self, c):
-            return (c == " " or
-                    c == "+" or
-                    c == "-")
+            return (c == _lit(" ") or
+                    c == _lit("+") or
+                    c == _lit("-"))
 
         def _parse_spec(self, default_type, default_align):
             space = self.space
             self._fill_char = self._lit(" ")[0]
             self._align = default_align
             self._alternate = False
-            self._sign = "\0"
+            self._sign = _lit("\0")
             self._thousands_sep = False
             self._precision = -1
             the_type = default_type
@@ -451,19 +453,19 @@
             if length - i >= 1 and self._is_sign(spec[i]):
                 self._sign = spec[i]
                 i += 1
-            if length - i >= 1 and spec[i] == "#":
+            if length - i >= 1 and spec[i] == _lit("#"):
                 self._alternate = True
                 i += 1
-            if not got_fill_char and length - i >= 1 and spec[i] == "0":
+            if not got_fill_char and length - i >= 1 and spec[i] == _lit("0"):
                 self._fill_char = self._lit("0")[0]
                 if not got_align:
-                    self._align = "="
+                    self._align = _lit("=")
                 i += 1
             self._width, i = _parse_int(self.space, spec, i, length)
-            if length != i and spec[i] == ",":
+            if length != i and spec[i] == _lit(","):
                 self._thousands_sep = True
                 i += 1
-            if length != i and spec[i] == ".":
+            if length != i and spec[i] == _lit("."):
                 i += 1
                 self._precision, i = _parse_int(self.space, spec, i, length)
                 if self._precision == -1:
@@ -471,28 +473,20 @@
             if length - i > 1:
                 raise oefmt(space.w_ValueError, "invalid format spec")
             if length - i == 1:
-                presentation_type = spec[i]
-                if self.is_unicode:
-                    try:
-                        the_type = spec[i].encode("ascii")[0]
-                    except UnicodeEncodeError:
-                        raise oefmt(space.w_ValueError,
-                                    "invalid presentation type")
-                else:
-                    the_type = presentation_type
+                the_type = spec[i]
                 i += 1
             self._type = the_type
             if self._thousands_sep:
                 tp = self._type
-                if (tp == "d" or
-                    tp == "e" or
-                    tp == "f" or
-                    tp == "g" or
-                    tp == "E" or
-                    tp == "G" or
-                    tp == "%" or
-                    tp == "F" or
-                    tp == "\0"):
+                if (tp == _lit("d") or
+                    tp == _lit("e") or
+                    tp == _lit("f") or
+                    tp == _lit("g") or
+                    tp == _lit("E") or
+                    tp == _lit("G") or
+                    tp == _lit("%") or
+                    tp == _lit("F") or
+                    tp == _lit("\0")):
                     # ok
                     pass
                 else:
@@ -506,11 +500,11 @@
             else:
                 total = length
             align = self._align
-            if align == ">":
+            if align == _lit(">"):
                 left = total - length
-            elif align == "^":
+            elif align == _lit("^"):
                 left = (total - length) / 2
-            elif align == "<" or align == "=":
+            elif align == _lit("<") or align == _lit("="):
                 left = 0
             else:
                 raise AssertionError("shouldn't be here")
@@ -539,23 +533,24 @@
                 return rstring.StringBuilder()
 
         def _unknown_presentation(self, tp):
+            spec = self._type.encode('ascii') if for_unicode else self._type
             raise oefmt(self.space.w_ValueError,
-                        "unknown presentation for %s: '%s'", tp, self._type)
+                        "unknown presentation for %s: '%s'", tp, spec)
 
         def format_string(self, string):
             space = self.space
-            if self._parse_spec("s", "<"):
+            if self._parse_spec(_lit("s"), _lit("<")):
                 return space.wrap(string)
-            if self._type != "s":
+            if self._type != _lit("s"):
                 self._unknown_presentation("string")
-            if self._sign != "\0":
+            if self._sign != _lit("\0"):
                 raise oefmt(space.w_ValueError,
                             "Sign not allowed in string format specifier")
             if self._alternate:
                 raise oefmt(space.w_ValueError,
                             "Alternate form (#) not allowed in string format "
                             "specifier")
-            if self._align == "=":
+            if self._align == _lit("="):
                 raise oefmt(space.w_ValueError,
                             "'=' alignment not allowed in string format "
                             "specifier")
@@ -760,8 +755,8 @@
                             "precision not allowed in integer type")
             sign_char = "\0"
             tp = self._type
-            if tp == "c":
-                if self._sign != "\0":
+            if tp == _lit("c"):
+                if self._sign != _lit("\0"):
                     raise oefmt(space.w_ValueError,
                                 "sign not allowed with 'c' presentation type")
                 value = space.int_w(w_num)
@@ -776,16 +771,16 @@
                 to_prefix = 0
                 to_numeric = 0
             else:
-                if tp == "b":
+                if tp == _lit("b"):
                     base = 2
                     skip_leading = 2
-                elif tp == "o":
+                elif tp == _lit("o"):
                     base = 8
                     skip_leading = 2
-                elif tp == "x" or tp == "X":
+                elif tp == _lit("x") or tp == _lit("X"):
                     base = 16
                     skip_leading = 2
-                elif tp == "n" or tp == "d":
+                elif tp == _lit("n") or tp == _lit("d"):
                     base = 10
                     skip_leading = 0
                 else:
@@ -808,7 +803,7 @@
             spec = self._calc_num_width(n_prefix, sign_char, to_numeric, n_digits,
                                         n_remainder, False, result)
             fill = self._fill_char
-            upper = self._type == "X"
+            upper = self._type == _lit("X")
             return self.space.wrap(self._fill_number(spec, result, to_numeric,
                                      to_prefix, fill, to_remainder, upper))
 
@@ -874,7 +869,7 @@
 
         def format_int_or_long(self, w_num, kind):
             space = self.space
-            if self._parse_spec("d", ">"):
+            if self._parse_spec(_lit("d"), _lit(">")):
                 if self.is_unicode:
                     return space.call_function(space.w_unicode, w_num)
                 return self.space.str(w_num)
@@ -922,15 +917,15 @@
                             "Alternate form (#) not allowed in float formats")
             tp = self._type
             self._get_locale(tp)
-            if tp == "\0":
-                tp = "g"
+            if tp == _lit("\0"):
+                tp = _lit("g")
                 default_precision = 12
                 flags |= rfloat.DTSF_ADD_DOT_0
-            elif tp == "n":
-                tp = "g"
+            elif tp == _lit("n"):
+                tp = _lit("g")
             value = space.float_w(w_float)
-            if tp == "%":
-                tp = "f"
+            if tp == _lit("%"):
+                tp = _lit("f")
                 value *= 100
                 add_pct = True
             else:
@@ -963,7 +958,7 @@
 
         def format_float(self, w_float):
             space = self.space
-            if self._parse_spec("\0", ">"):
+            if self._parse_spec(_lit("\0"), _lit(">")):
                 if self.is_unicode:
                     return space.call_function(space.w_unicode, w_float)
                 return space.str(w_float)
@@ -1002,9 +997,9 @@
                             "specifier")
             skip_re = 0
             add_parens = 0
-            if tp == "\0":
+            if tp == _lit("\0"):
                 #should mirror str() output
-                tp = "g"
+                tp = _lit("g")
                 default_precision = 12
                 #test if real part is non-zero
                 if (w_complex.realval == 0 and
@@ -1013,9 +1008,9 @@
                 else:
                     add_parens = 1
 
-            if tp == "n":
+            if tp == _lit("n"):
                 #same as 'g' except for locale, taken care of later
-                tp = "g"
+                tp = _lit("g")
 
             #check if precision not set
             if self._precision == -1:
@@ -1073,7 +1068,7 @@
             #self._grouped_digits will get overwritten in imaginary calc_num_width
             re_grouped_digits = self._grouped_digits
             if not skip_re:
-                self._sign = "+"
+                self._sign = _lit("+")
             im_spec = self._calc_num_width(0, im_sign, to_imag_number, n_im_digits,
                                            im_n_remainder, im_have_dec,
                                            im_num)
@@ -1127,7 +1122,7 @@
             """return the string representation of a complex number"""
             space = self.space
             #parse format specification, set associated variables
-            if self._parse_spec("\0", ">"):
+            if self._parse_spec(_lit("\0"), _lit(">")):
                 return space.str(w_complex)
             tp = self._type
             if (tp == "\0" or
@@ -1148,9 +1143,5 @@
 
 @specialize.arg(2)
 def run_formatter(space, w_format_spec, meth, *args):
-    if space.isinstance_w(w_format_spec, space.w_unicode):
-        formatter = unicode_formatter(space, space.unicode_w(w_format_spec))
-        return getattr(formatter, meth)(*args)
-    else:
-        formatter = str_formatter(space, space.str_w(w_format_spec))
-        return getattr(formatter, meth)(*args)
+    formatter = str_formatter(space, space.str_w(w_format_spec))
+    return getattr(formatter, meth)(*args)
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -220,7 +220,7 @@
             offset = len(token)
 
             while 1:
-                if token[offset-1] == "\n" or token[offset-1] == "\r":
+                if token[offset-1] == self._chr("\n") or token[offset-1] == self._chr("\r"):
                     break
                 distance += 1
                 offset -= 1
@@ -598,7 +598,8 @@
             eol = pos
             pos += 1
             # read CRLF as one line break
-            if pos < length and value[eol] == '\r' and value[pos] == '\n':
+            if (pos < length and value[eol] == self._chr('\r')
+                    and value[pos] == self._chr('\n')):
                 pos += 1
             if keepends:
                 eol = pos
@@ -780,7 +781,8 @@
             return self._new(selfval)
 
         builder = self._builder(width)
-        if len(selfval) > 0 and (selfval[0] == '+' or selfval[0] == '-'):
+        if len(selfval) > 0 and (
+                selfval[0] == self._chr('+') or selfval[0] == self._chr('-')):
             # copy sign to first position
             builder.append(selfval[0])
             start = 1

From pypy.commits at gmail.com  Mon Sep  5 03:29:26 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Sep 2016 00:29:26 -0700 (PDT)
Subject: [pypy-commit] pypy redirect-assembler-jitlog: translation issues
Message-ID: <57cd1ed6.68adc20a.966c2.f187@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: redirect-assembler-jitlog
Changeset: r86875:adb14149ca02
Date: 2016-09-05 08:34 +0200
http://bitbucket.org/pypy/pypy/changeset/adb14149ca02/

Log:	translation issues

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1123,8 +1123,8 @@
     """
     jitcell_token = make_jitcell_token(jitdriver_sd)
     #
-    logger = jitdriver_sd.metainterp_sd.jitlog
-    jitcell_token.number = logger.next_id()
+    #logger = jitdriver_sd.metainterp_sd.jitlog
+    #jitcell_token.number = logger.next_id()
     jl.tmp_callback(jitcell_token)
     #
     nb_red_args = jitdriver_sd.num_red_args
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -255,12 +255,13 @@
 
 if __name__ == "__main__":
     print("# generated constants from rpython/rlib/jitlog.py")
-    print 'MARK_JITLOG_START = struct.pack("b", "%s")' % hex(0x10)
+    print('import struct')
+    print('MARK_JITLOG_START = struct.pack("b", %s)' % hex(0x10))
     for mark, in marks:
         nmr = globals()['MARK_' + mark]
         h = hex(ord(nmr))
-        print '%s = struct.pack("b", "%s")' % ('MARK_' + mark, h)
-    print 'MARK_JITLOG_END = struct.pack("b", "%s")' % hex(start)
+        print '%s = struct.pack("b", %s)' % ('MARK_' + mark, h)
+    print 'MARK_JITLOG_END = struct.pack("b", %s)' % hex(start)
     for key,value in locals().items():
         if key.startswith("MP_"):
             print '%s = (%s,"%s")' % (key, hex(value[0]), value[1])
@@ -323,10 +324,10 @@
 
 def tmp_callback(looptoken):
     mark_tmp_callback = ''.join([
-        jl.MARK_TMP_CALLBACK,
+        MARK_TMP_CALLBACK,
         encode_le_addr(compute_unique_id(looptoken)),
         encode_le_64bit(looptoken.number)])
-    jl.jitlog_write_marked(mark_tmp_callback, len(mark_tmp_callback))
+    jitlog_write_marked(mark_tmp_callback, len(mark_tmp_callback))
 
 class JitLogger(object):
     def __init__(self, cpu=None):

From pypy.commits at gmail.com  Mon Sep  5 03:29:31 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Sep 2016 00:29:31 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57cd1edb.121a1c0a.10647.88f5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86876:ab4c074e8f5f
Date: 2016-09-05 08:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ab4c074e8f5f/

Log:	merge default

diff too long, truncating to 2000 out of 213218 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,6 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,12 +360,15 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
 
   Heinrich-Heine University, Germany 
   Open End AB (formerly AB Strakt), Sweden
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.1"
+__version_info__ = (1, 8, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,43 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -137,6 +137,8 @@
             lib.gdbm_sync(self.__ll_dbm)
 
 def open(filename, flags='r', mode=0666):
+    if isinstance(filename, unicode):
+        filename = filename.encode()
     if flags[0] == 'r':
         iflags = lib.GDBM_READER
     elif flags[0] == 'w':
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,218 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,152 +1,9 @@
 ==========================
-What's new in PyPy2.7 5.3+
+What's new in PyPy2.7 5.4+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
-
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
-
-
-.. branch: const-fold-we-are-jitted
-
-Reduce the size of the generated C code by constant-folding ``we_are_jitted``
-in non-jitcode.
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 5.3+
-==========================
+=========================
+What's new in PyPy2.7 5.4
+=========================
 
 .. this is a revision shortly after release-pypy2.7-v5.3
 .. startrev: 873218a739f1
@@ -150,3 +150,16 @@
 
 Reduce the size of the generated C code by constant-folding ``we_are_jitted``
 in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -208,7 +208,8 @@
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self, 
+                                        space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
@@ -216,7 +217,8 @@
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
@@ -224,7 +226,8 @@
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
@@ -232,7 +235,8 @@
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -23,6 +23,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -84,7 +92,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -38,6 +38,8 @@
 
 
 class W_ClassObject(W_Root):
+    _immutable_fields_ = ['bases_w?[*]', 'w_dict?']
+
     def __init__(self, space, w_name, bases, w_dict):
         self.name = space.str_w(w_name)
         make_sure_not_resized(bases)
@@ -75,6 +77,7 @@
                             "__bases__ items must be classes")
         self.bases_w = bases_w
 
+    @jit.unroll_safe
     def is_subclass_of(self, other):
         assert isinstance(other, W_ClassObject)
         if self is other:
@@ -313,7 +316,7 @@
         # This method ignores the instance dict and the __getattr__.
         # Returns None if not found.
         assert isinstance(name, str)
-        w_value = self.w_class.lookup(space, name)
+        w_value = jit.promote(self.w_class).lookup(space, name)
         if w_value is None:
             return None
         w_descr_get = space.lookup(w_value, '__get__')
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -250,6 +250,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -358,9 +358,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -460,7 +466,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -612,7 +618,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -151,6 +151,7 @@
 
     def test_readValues(self):
         from _winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from _winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -164,7 +165,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -120,8 +120,8 @@
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
 """.split()
 for name in constant_names:
@@ -649,6 +649,7 @@
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
@@ -976,8 +977,10 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
     add_fork_hook('child', reinit_tls)
 
 def init_function(func):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,13 +1,17 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer)
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
+def PyObject_CheckBuffer(space, pyobj):
     """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
+    as_buffer = pyobj.c_ob_type.c_tp_as_buffer
+    flags = pyobj.c_ob_type.c_tp_flags
+    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
+        return 1
+    return 0  
 
 @cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
              rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -123,5 +123,4 @@
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
     space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname))
-    importing.exec_code_module(space, w_mod, code)
-    return w_mod
+    return importing.exec_code_module(space, w_mod, code, w_name)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.3.2-alpha0"
-#define PYPY_VERSION_NUM  0x05030200
+#define PYPY_VERSION "5.5.0-alpha0"
+#define PYPY_VERSION_NUM  0x05050000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -12,7 +12,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
-    raise NotImplementedError
+    raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -3,15 +3,16 @@
 import re
 
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES,
-    mangle_name, pypy_decl)
+    mangle_name, pypy_decl, Py_buffer, Py_bufferP)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc,
     getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-    readbufferproc, ssizessizeobjargproc)
+    readbufferproc, getbufferproc, ssizessizeobjargproc)
 from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.state import State
@@ -22,6 +23,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_renamer
 from rpython.rtyper.annlowlevel import llhelper
+from pypy.module.sys.version import CPYTHON_VERSION
+
+PY3 = CPYTHON_VERSION[0] == 3
 
 # XXX: Also defined in object.h
 Py_LT = 0
@@ -298,11 +302,23 @@
     # Similar to Py_buffer
     _immutable_ = True
 
-    def __init__(self, ptr, size, w_obj):
+    def __init__(self, ptr, size, w_obj, format='B', shape=None,
+                strides=None, ndim=1, itemsize=1, readonly=True):
         self.ptr = ptr
         self.size = size
         self.w_obj = w_obj # kept alive
-        self.readonly = True
+        self.format = format
+        if not shape:
+            self.shape = [size]
+        else:
+            self.shape = shape
+        if not strides:
+            self.strides = [1]
+        else:
+            self.strides = strides
+        self.ndim = ndim 
+        self.itemsize = itemsize
+        self.readonly = readonly
 
     def getlength(self):
         return self.size
@@ -313,6 +329,15 @@
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
 
+    def getformat(self):
+        return self.format
+
+    def getshape(self):
+        return self.shape
+
+    def getitemsize(self):
+        return self.itemsize
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
@@ -322,6 +347,30 @@
             space.fromcache(State).check_and_raise_exception(always=True)
         return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
 
+def wrap_getbuffer(space, w_self, w_args, func):
+    func_target = rffi.cast(getbufferproc, func)
+    with lltype.scoped_alloc(Py_buffer) as pybuf:
+        _flags = 0
+        if space.len_w(w_args) > 0:
+            _flags = space.int_w(space.listview(w_args)[0])
+        flags = rffi.cast(rffi.INT_real,_flags)
+        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+        if widen(size) < 0:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        ptr = pybuf.c_buf
+        size = pybuf.c_len
+        ndim = widen(pybuf.c_ndim)
+        shape =   [pybuf.c_shape[i]   for i in range(ndim)]
+        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_format:
+            format = rffi.charp2str(pybuf.c_format)
+        else:
+            format = 'B'
+        return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format,
+                            ndim=ndim, shape=shape, strides=strides,
+                            itemsize=pybuf.c_itemsize,
+                            readonly=widen(pybuf.c_readonly)))
+
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
         func_target = rffi.cast(richcmpfunc, func)
@@ -486,7 +535,6 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         api_func = slot_tp_getattro.api_func
-
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -542,6 +590,21 @@
                              w_stararg=w_args, w_starstararg=w_kwds)
             return space.call_args(space.get(new_fn, w_self), args)
         api_func = slot_tp_new.api_func
+    elif name == 'tp_as_buffer.c_bf_getbuffer':
+        buff_fn = w_type.getdictvalue(space, '__buffer__')
+        if buff_fn is None:
+            return
+        @cpython_api([PyObject, Py_bufferP, rffi.INT_real], 
+                rffi.INT_real, header=None, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def buff_w(space, w_self, pybuf, flags):
+            # XXX this is wrong, needs a test
+            raise oefmt(space.w_NotImplemented, 
+                "calling bf_getbuffer on a builtin type not supported yet")
+            #args = Arguments(space, [w_self],
+            #                 w_stararg=w_args, w_starstararg=w_kwds)
+            #return space.call_args(space.get(buff_fn, w_self), args)
+        api_func = buff_w.api_func
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
@@ -850,11 +913,19 @@
 slotdefs = eval(slotdefs_str)
 # PyPy addition
 slotdefs += (
-    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    # XXX that might not be what we want!
+    TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""),
 )
 
+if not PY3:
+    slotdefs += (
+        TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+    )
+
+
 # partial sort to solve some slot conflicts:
 # Number slots before Mapping slots before Sequence slots.
+# also prefer the new buffer interface
 # These are the only conflicts between __name__ methods
 def slotdef_sort_key(slotdef):
     if slotdef.slot_name.startswith('tp_as_number'):
@@ -863,6 +934,10 @@
         return 2
     if slotdef.slot_name.startswith('tp_as_sequence'):
         return 3
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer':
+        return 100
+    if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer':
+        return 101
     return 0
 slotdefs = sorted(slotdefs, key=slotdef_sort_key)
 
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -0,0 +1,248 @@
+#ifdef _MSC_VER
+#define _CRT_SECURE_NO_WARNINGS 1
+#endif
+#include <Python.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/* 
+ * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol,
+ * which is copyright Jake Vanderplas and released under the BSD license
+ */
+
+/* Structure defines a 1-dimensional strided array */
+typedef struct{
+    int* arr;
+    Py_ssize_t length;
+} MyArray;
+
+/* initialize the array with integers 0...length */
+void initialize_MyArray(MyArray* a, long length){
+    int i;
+    a->length = length;
+    a->arr = (int*)malloc(length * sizeof(int));
+    for(i=0; i<length; i++){
+        a->arr[i] = i;
+    }
+}
+
+/* free the memory when finished */
+void deallocate_MyArray(MyArray* a){
+    free(a->arr);
+    a->arr = NULL;
+}
+
+/* tools to print the array */
+char* stringify(MyArray* a, int nmax){
+    char* output = (char*) malloc(nmax * 20);
+    int k, pos = sprintf(&output[0], "[");
+
+    for (k=0; k < a->length && k < nmax; k++){
+        pos += sprintf(&output[pos], " %d", a->arr[k]);
+    }
+    if(a->length > nmax)
+        pos += sprintf(&output[pos], "...");
+    sprintf(&output[pos], " ]");
+    return output;
+}
+
+void print_MyArray(MyArray* a, int nmax){
+    char* s = stringify(a, nmax);
+    printf("%s", s);
+    free(s);
+}
+
+/* This is where we define the PyMyArray object structure */
+typedef struct {
+    PyObject_HEAD
+    /* Type-specific fields go below. */
+    MyArray arr;
+} PyMyArray;
+
+
+/* This is the __init__ function, implemented in C */
+static int
+PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds)
+{
+    int length = 0;
+    static char *kwlist[] = {"length", NULL};
+    // init may have already been called
+    if (self->arr.arr != NULL) {
+        deallocate_MyArray(&self->arr);
+    }
+
+    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length))
+        return -1;
+
+    if (length < 0)
+        length = 0;
+
+    initialize_MyArray(&self->arr, length);
+
+    return 0;
+}
+
+
+/* this function is called when the object is deallocated */
+static void
+PyMyArray_dealloc(PyMyArray* self)
+{
+    deallocate_MyArray(&self->arr);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+
+/* This function returns the string representation of our object */
+static PyObject *
+PyMyArray_str(PyMyArray * self)
+{
+  char* s = stringify(&self->arr, 10);
+  PyObject* ret = PyUnicode_FromString(s);
+  free(s);
+  return ret;
+}
+
+/* Here is the buffer interface function */
+static int
+PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
+{
+  PyMyArray* self = (PyMyArray*)obj;
+  fprintf(stdout, "in PyMyArray_getbuffer\n");
+  if (view == NULL) {
+    fprintf(stdout, "view is NULL\n");
+    PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
+    return -1;
+  }
+  if (flags == 0) {
+    fprintf(stdout, "flags is 0\n");
+    PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
+    return -1;
+  }
+
+  view->obj = (PyObject*)self;
+  view->buf = (void*)self->arr.arr;
+  view->len = self->arr.length * sizeof(int);
+  view->readonly = 0;
+  view->itemsize = sizeof(int);
+  view->format = "i";  // integer
+  view->ndim = 1;
+  view->shape = &self->arr.length;  // length-1 sequence of dimensions
+  view->strides = &view->itemsize;  // for the simple case we can do this
+  view->suboffsets = NULL;
+  view->internal = NULL;
+
+  Py_INCREF(self);  // need to increase the reference count
+  return 0;
+}
+
+static PyBufferProcs PyMyArray_as_buffer = {
+#if PY_MAJOR_VERSION < 3
+  (readbufferproc)0,
+  (writebufferproc)0,
+  (segcountproc)0,
+  (charbufferproc)0,
+#endif
+  (getbufferproc)PyMyArray_getbuffer,
+  (releasebufferproc)0,  // we do not require any special release function
+};
+
+
+/* Here is the type structure: we put the above functions in the appropriate place
+   in order to actually define the Python object type */
+static PyTypeObject PyMyArrayType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "pymyarray.PyMyArray",        /* tp_name */
+    sizeof(PyMyArray),            /* tp_basicsize */
+    0,                            /* tp_itemsize */
+    (destructor)PyMyArray_dealloc,/* tp_dealloc */
+    0,                            /* tp_print */
+    0,                            /* tp_getattr */
+    0,                            /* tp_setattr */
+    0,                            /* tp_reserved */
+    (reprfunc)PyMyArray_str,      /* tp_repr */
+    0,                            /* tp_as_number */
+    0,                            /* tp_as_sequence */
+    0,                            /* tp_as_mapping */
+    0,                            /* tp_hash  */
+    0,                            /* tp_call */
+    (reprfunc)PyMyArray_str,      /* tp_str */
+    0,                            /* tp_getattro */
+    0,                            /* tp_setattro */
+    &PyMyArray_as_buffer,         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */
+    "PyMyArray object",           /* tp_doc */
+    0,                            /* tp_traverse */
+    0,                            /* tp_clear */
+    0,                            /* tp_richcompare */
+    0,                            /* tp_weaklistoffset */
+    0,                            /* tp_iter */
+    0,                            /* tp_iternext */
+    0,                            /* tp_methods */
+    0,                            /* tp_members */
+    0,                            /* tp_getset */
+    0,                            /* tp_base */
+    0,                            /* tp_dict */
+    0,                            /* tp_descr_get */
+    0,                            /* tp_descr_set */
+    0,                            /* tp_dictoffset */
+    (initproc)PyMyArray_init,     /* tp_init */
+};
+
+static PyMethodDef buffer_functions[] = {
+    {NULL,        NULL}    /* Sentinel */
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "buffer_test",
+    "Module Doc",
+    -1,
+    buffer_functions,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+};
+#define INITERROR return NULL
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+extern __declspec(dllexport)
+#endif
+
+PyMODINIT_FUNC
+PyInit_buffer_test(void)
+
+#else
+
+#define INITERROR return
+
+/* Initialize this module. */
+#ifdef __GNUC__
+extern __attribute__((visibility("default")))
+#else
+#endif
+
+PyMODINIT_FUNC
+initbuffer_test(void)
+#endif
+{
+#if PY_MAJOR_VERSION >= 3
+    PyObject *m= PyModule_Create(&moduledef);
+#else
+    PyObject *m= Py_InitModule("buffer_test", buffer_functions);
+#endif
+    if (m == NULL)
+        INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&PyMyArrayType) < 0)
+        INITERROR;
+    Py_INCREF(&PyMyArrayType);
+    PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType);
+#if PY_MAJOR_VERSION >=3
+    return m;
+#endif
+}
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -87,4 +87,13 @@
         module.switch_multiply()
         res = [1, 2, 3] * arr
         assert res == [2, 4, 6]
+
+    def test_subclass(self):
+        module = self.import_module(name='array')
+        class Sub(module.array):
+            pass
+
+        arr = Sub('i', [2])
+        res = [1, 2, 3] * arr
+        assert res == [1, 2, 3, 1, 2, 3]
         
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
@@ -145,6 +151,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -433,21 +457,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,17 +1,26 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
+        assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
+        w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
+        assert space.eq_w(w_char, space.wrap('h'))
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
-    @pytest.mark.skipif(True, reason='write a test for this')
-    def test_get_base_and_get_buffer(self, space, api):
-        assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+    def test_buffer_protocol(self):
+        import struct
+        module = self.import_module(name='buffer_test')
+        arr = module.PyMyArray(10)
+        y = memoryview(arr)
+        assert y.format == 'i'
+        assert y.shape == (10,)
+        s = y[3]
+        assert len(s) == struct.calcsize('i')
+        assert s == struct.pack('i', 3)
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -32,9 +32,11 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -17,7 +17,9 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder,
-    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr)
+    PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr,
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HAVE_INPLACEOPS)
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -385,6 +387,8 @@
         pto.c_tp_basicsize = base_pto.c_tp_basicsize
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+    pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
     flags = rffi.cast(lltype.Signed, pto.c_tp_flags)
     base_object_pyo = make_ref(space, space.w_object)
     base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
@@ -608,6 +612,7 @@
                 bf_getwritebuffer.api_func.get_wrapper(space))
     pto.c_tp_as_buffer = c_buf
     pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
+    pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def type_dealloc(space, obj):
@@ -774,6 +779,8 @@
             pto.c_tp_setattro = base.c_tp_setattro
         if not pto.c_tp_getattro:
             pto.c_tp_getattro = base.c_tp_getattro
+        if not pto.c_tp_as_buffer:
+            pto.c_tp_as_buffer = base.c_tp_as_buffer
     finally:
         Py_DecRef(space, base_pyo)
 
@@ -810,8 +817,13 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number
-        if not py_type.c_tp_as_sequence: py_type.c_tp_as_sequence = base.c_tp_as_sequence
+        if not py_type.c_tp_as_number: 
+            py_type.c_tp_as_number = base.c_tp_as_number
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+        if not py_type.c_tp_as_sequence:
+            py_type.c_tp_as_sequence = base.c_tp_as_sequence
+            py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
         if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -5,6 +5,7 @@
     Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.modsupport import PyMethodDef
+from pypy.module.cpyext.api import Py_bufferP
 
 
 P, FT, PyO = Ptr, FuncType, PyObject
@@ -58,8 +59,7 @@
 writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t))
 segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t))
 charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t))
-## We don't support new buffer interface for now
-getbufferproc = rffi.VOIDP
+getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real))
 releasebufferproc = rffi.VOIDP
 
 
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -597,6 +597,11 @@
 
 @jit.dont_look_inside
 def load_module(space, w_modulename, find_info, reuse=False):
+    """Like load_module() in CPython's import.c, this will normally
+    make a module object, store it in sys.modules, execute code in it,
+    and then fetch it again from sys.modules.  But this logic is not
+    used if we're calling a PEP302 loader.
+    """
     if find_info is None:
         return
 
@@ -625,17 +630,15 @@
 
         try:
             if find_info.modtype == PY_SOURCE:
-                load_source_module(
+                return load_source_module(
                     space, w_modulename, w_mod,
                     find_info.filename, find_info.stream.readall(),
                     find_info.stream.try_to_find_file_descriptor())
-                return w_mod
             elif find_info.modtype == PY_COMPILED:
                 magic = _r_long(find_info.stream)
                 timestamp = _r_long(find_info.stream)
-                load_compiled_module(space, w_modulename, w_mod, find_info.filename,
+                return load_compiled_module(space, w_modulename, w_mod, find_info.filename,
                                      magic, timestamp, find_info.stream.readall())

From pypy.commits at gmail.com  Mon Sep  5 03:29:33 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Sep 2016 00:29:33 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: killed a bug in unpacking
 multiple
Message-ID: <57cd1edd.06a81c0a.332c.3865@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86877:0e4fa815252e
Date: 2016-09-05 09:28 +0200
http://bitbucket.org/pypy/pypy/changeset/0e4fa815252e/

Log:	killed a bug in unpacking multiple

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -534,13 +534,13 @@
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
             if count * size == 8:
-                stidx = 0
                 if not IS_BIG_ENDIAN:
-                    idx = (16 // size) - 1 - idx
-                    stidx = 0
-                off = PARAM_SAVE_AREA_OFFSET + idx * size
+                    endian_off = 8
+                off = PARAM_SAVE_AREA_OFFSET
+                off = off + endian_off - (idx * size)
+                assert idx * size + 8 <= 16
                 self.mc.load(r.SCRATCH.value, r.SP.value, off)
-                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+stidx)
+                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
                 self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
                 return
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -826,7 +826,6 @@
                        (2**31-1 if i%2==0 else 0)
 
     def test_unpack_several(self):
-        # count == 2
         values = [1,2,3,4]
         for i,v in enumerate(values):
             j = (i // 2) * 2

From pypy.commits at gmail.com  Mon Sep  5 07:51:15 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Sep 2016 04:51:15 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: not_implemented takes only one
 argument, remove pxor(result, result) which contained valid data
Message-ID: <57cd5c33.87941c0a.801f7.fb89@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86878:179c6a65d492
Date: 2016-09-05 13:50 +0200
http://bitbucket.org/pypy/pypy/changeset/179c6a65d492/

Log:	not_implemented takes only one argument, remove pxor(result,result)
	which contained valid data

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -322,8 +322,7 @@
             assert lhsloc is xmm0
             maskloc = X86_64_XMM_SCRATCH_REG
             self.mc.MOVAPD(maskloc, heap(self.element_ones[get_scale(size)]))
-            self.mc.PXOR(resloc, resloc)
-            # note that xmm0 contains true false for each element by the last compare operation
+            # note that resloc contains true false for each element by the last compare operation
             self.mc.PBLENDVB_xx(resloc.value, maskloc.value)
 
     def genop_vec_float_ne(self, op, arglocs, resloc):
@@ -532,7 +531,7 @@
                             self.mc.UNPCKHPD(resloc, srcloc)
                         # if they are equal nothing is to be done
         else:
-            not_implemented("pack/unpack for size %d", size)
+            not_implemented("pack/unpack for size %d" % size)
 
     genop_vec_unpack_f = genop_vec_pack_f
 

From pypy.commits at gmail.com  Mon Sep  5 09:53:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 06:53:29 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: rewrite
 _array_from_buffer_3118 without explicit W_MemeoryView (arigato)
Message-ID: <57cd78d9.041f1c0a.fbb56.3d45@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86879:52eb4d9855de
Date: 2016-09-04 22:26 +0300
http://bitbucket.org/pypy/pypy/changeset/52eb4d9855de/

Log:	rewrite _array_from_buffer_3118 without explicit W_MemeoryView
	(arigato)

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
     def getdictvalue(self, space, key):
         return self.items[key]
 
+    def descr_memoryview(self, space, buf):
+        raise oefmt(space.w_TypeError, "error")
+
 class IterDictObject(W_Root):
     def __init__(self, space, w_dict):
         self.space = space
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstring import strip_spaces
@@ -11,7 +12,6 @@
 from pypy.module.micronumpy.converters import shape_converter, order_converter
 import pypy.module.micronumpy.constants as NPY
 from .casting import scalar2dtype
-from pypy.objspace.std.memoryobject import W_MemoryView
 
 
 def build_scalar(space, w_dtype, w_state):
@@ -123,16 +123,23 @@
     return None 
 
 def _array_from_buffer_3118(space, w_object, dtype):
-    buf = w_object.buf
-    if buf.getformat():
-        descr = _descriptor_from_pep3118_format(space, buf.getformat())
+    try:
+        w_buf = space.call_method(space.builtin, "memoryview", w_object)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            # object does not have buffer interface
+            return w_object
+        raise
+    format = space.getattr(w_buf,space.newbytes('format'))
+    if format:
+        descr = _descriptor_from_pep3118_format(space, space.str_w(format))
         if not descr:
             return w_object
         if dtype and descr:
             raise oefmt(space.w_NotImplementedError,
                 "creating an array from a memoryview while specifying dtype "
                 "not supported")
-        if descr.elsize != buf.getitemsize(): 
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
             msg = ("Item size computed from the PEP 3118 buffer format "
                   "string does not match the actual item size.")
             space.warn(space.wrap(msg), space.w_RuntimeWarning)
@@ -140,14 +147,17 @@
         dtype = descr 
     elif not dtype:
         dtype = descriptor.get_dtype_cache(space).w_stringdtype
-        dtype.elsize = buf.getitemsize()
-    nd = buf.getndim()
-    shape = buf.getshape()
+        dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
+    nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim')))
+    shape = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('shape')))]
     strides = []
+    buflen = space.len_w(w_buf) * dtype.elsize
     if shape:
-        strides = buf.getstrides()
+        strides = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('strides')))]
         if not strides:
-            d = len(buf)
+            d = buflen
             strides = [0] * nd
             for k in range(nd):
                 if shape[k] > 0:
@@ -155,17 +165,23 @@
                     strides[k] = d
     else:
         if nd == 1:
-            shape = [len(buf) / dtype.elsize, ]
+            shape = [buflen / dtype.elsize, ]
             strides = [dtype.elsize, ]
         elif nd > 1:
             msg = ("ndim computed from the PEP 3118 buffer format "
                    "is greater than 1, but shape is NULL.")
             space.warn(space.wrap(msg), space.w_RuntimeWarning)
             return w_object
-    storage = buf.get_raw_address()
-    writable = not buf.readonly
-    w_ret = W_NDimArray.from_shape_and_storage(space, shape, storage,
-               storage_bytes=len(buf), dtype=dtype, w_base=w_object, 
+    try:
+        w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
+    except OperationError as e:
+        if e.match(space, space.w_ValueError):
+            return w_object
+        else:
+            raise e
+    writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
+    w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
+               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
                writable=writable, strides=strides)
     if w_ret:
         return w_ret
@@ -187,6 +203,7 @@
 
 def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
 
+    from pypy.module.micronumpy.boxes import W_GenericBox
     # numpy testing calls array(type(array([]))) and expects a ValueError
     if space.isinstance_w(w_object, space.w_type):
         raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
@@ -194,14 +211,17 @@
     dtype = descriptor.decode_w_dtype(space, w_dtype)
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
-        if w_array is not None:
+        if w_array is None:
+            if (    not space.isinstance_w(w_object, space.w_str) and 
+                    not space.isinstance_w(w_object, space.w_unicode) and
+                    not isinstance(w_object, W_GenericBox)):
+                # use buffer interface
+                w_object = _array_from_buffer_3118(space, w_object, dtype)
+        else:
             # continue with w_array, but do further operations in place
             w_object = w_array
             copy = False
             dtype = w_object.get_dtype()
-        elif isinstance(w_object, W_MemoryView):
-            # use buffer interface
-            w_object = _array_from_buffer_3118(space, w_object, dtype)
     if not isinstance(w_object, W_NDimArray):
         w_array, _copy = try_interface_method(space, w_object, copy)
         if w_array is not None:
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -97,16 +97,7 @@
             print "ERROR: did not implement return type for interpreter"
             raise TypeError(w_res)
 
-        if self.graph is not None:
-            return
-
-        from pypy.module.micronumpy import ctors
-        def unimplemented(*args):
-            "Only for W_MemoryView objects, which are not compiled in"
-            raise NotImplementedError
-        prev_3118 = ctors._array_from_buffer_3118
-        ctors._array_from_buffer_3118 = unimplemented
-        try:
+        if self.graph is None:
             interp, graph = self.meta_interp(f, [0],
                                              listops=True,
                                              listcomp=True,
@@ -116,8 +107,6 @@
                                              vec=True)
             self.__class__.interp = interp
             self.__class__.graph = graph
-        finally:
-            ctors._array_from_buffer_3118 = prev_3118
 
     def check_vectorized(self, expected_tried, expected_success):
         profiler = get_profiler()

From pypy.commits at gmail.com  Mon Sep  5 09:55:45 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 06:55:45 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: use tobytes if get_raw_address
 fails, for StringBuffer type buffers
Message-ID: <57cd7961.0558c20a.67473.4ed2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86880:da7d464c1703
Date: 2016-09-05 16:54 +0300
http://bitbucket.org/pypy/pypy/changeset/da7d464c1703/

Log:	use tobytes if get_raw_address fails, for StringBuffer type buffers

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -26,13 +26,17 @@
         return view
     try:
         view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = as_pyobj(space, w_obj)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        isstr = False
     except ValueError:
-        return view
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = as_pyobj(space, w_s)
+        rffi.setintfield(view, 'c_readonly', 1)
+        isstr = True
     view.c_len = w_obj.getlength()
-    view.c_obj = as_pyobj(space, w_obj)
     incref(space, view.c_obj)
     view.c_itemsize = w_obj.buf.getitemsize()
-    rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
     ndim = w_obj.buf.getndim()
     rffi.setintfield(view, 'c_ndim', ndim)
     view.c_format = rffi.str2charp(w_obj.buf.getformat())
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
@@ -12,6 +12,12 @@
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
+    def test_frombuffer(self, space, api):
+        w_buf = space.newbuffer(StringBuffer("hello"))
+        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        ndim = w_view.c_ndim
+        assert ndim == 1
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):

From pypy.commits at gmail.com  Mon Sep  5 10:40:43 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 07:40:43 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: use make_ref which has a
 keepalive_until_here() (arigato)
Message-ID: <57cd83eb.081dc20a.e249d.049e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86881:8b66ea6779d0
Date: 2016-09-05 17:17 +0300
http://bitbucket.org/pypy/pypy/changeset/8b66ea6779d0/

Log:	use make_ref which has a keepalive_until_here() (arigato)

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
                                     build_type_checkers, Py_ssize_tP)
-from pypy.module.cpyext.pyobject import PyObject, as_pyobj, incref
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.objspace.std.memoryobject import W_MemoryView
 
@@ -26,16 +26,15 @@
         return view
     try:
         view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
-        view.c_obj = as_pyobj(space, w_obj)
+        view.c_obj = make_ref(space, w_obj)
         rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
         isstr = False
     except ValueError:
         w_s = w_obj.descr_tobytes(space)
-        view.c_obj = as_pyobj(space, w_s)
+        view.c_obj = make_ref(space, w_s)
         rffi.setintfield(view, 'c_readonly', 1)
         isstr = True
     view.c_len = w_obj.getlength()
-    incref(space, view.c_obj)
     view.c_itemsize = w_obj.buf.getitemsize()
     ndim = w_obj.buf.getndim()
     rffi.setintfield(view, 'c_ndim', ndim)

From pypy.commits at gmail.com  Mon Sep  5 10:40:45 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 07:40:45 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: use make_ref which has a
 keepalive_until_here() (arigato)
Message-ID: <57cd83ed.915c1c0a.bba0e.40c1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86882:948af0843de0
Date: 2016-09-05 17:40 +0300
http://bitbucket.org/pypy/pypy/changeset/948af0843de0/

Log:	use make_ref which has a keepalive_until_here() (arigato)

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -3,7 +3,7 @@
 from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
-from pypy.module.cpyext.pyobject import PyObject, as_pyobj, incref
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -41,8 +41,7 @@
     except ValueError:
         raise BufferError("could not create buffer from object")
     view.c_len = buf.getlength()
-    view.c_obj = as_pyobj(space, w_obj)
-    incref(space, view.c_obj)
+    view.c_obj = make_ref(space, w_obj)
     ndim = buf.getndim()
     view.c_itemsize = buf.getitemsize()
     rffi.setintfield(view, 'c_readonly', int(buf.readonly))

From pypy.commits at gmail.com  Mon Sep  5 10:42:27 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Sep 2016 07:42:27 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: return register immediately if
 it is the right one (base case)
Message-ID: <57cd8453.e16ec20a.59a7a.a8e4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86883:199d8fd839ac
Date: 2016-09-05 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/199d8fd839ac/

Log:	return register immediately if it is the right one (base case)

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -308,7 +308,7 @@
             self.mc.CMPPD_xxi(lhsloc.value, rhsloc.value, 0)
         self.flush_vec_cc(rx86.Conditions["E"], lhsloc, resloc, sizeloc.value)
 
-    def flush_vec_cc(self, rev_cond, lhsloc, resloc, size):
+    def flush_vec_cc(self, rev_cond, lhsloc, resloc, size)
         # After emitting an instruction that leaves a boolean result in
         # a condition code (cc), call this.  In the common case, result_loc
         # will be set to SPP by the regalloc, which in this case means
@@ -322,6 +322,7 @@
             assert lhsloc is xmm0
             maskloc = X86_64_XMM_SCRATCH_REG
             self.mc.MOVAPD(maskloc, heap(self.element_ones[get_scale(size)]))
+            self.mc.PXOR(resloc, resloc)
             # note that resloc contains true false for each element by the last compare operation
             self.mc.PBLENDVB_xx(resloc.value, maskloc.value)
 
@@ -643,7 +644,12 @@
             instructions.
         """
         xrm = self.xrm
+        curloc = self.loc(arg)
+        if curloc is selected_reg:
+            # nothing to do, it is already in the correct register
+            return selected_reg
         if selected_reg not in xrm.free_regs:
+            # we need to move some registers
             variable = None
             candidate_to_spill = None
             for var, reg in self.xrm.reg_bindings.items():
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -156,9 +156,9 @@
                 i += size
 
         la = data.draw(st.lists(st.floats(), min_size=10, max_size=150))
-        #la = [0.0,0.0,0.0,0.0,0.0,0.0,0.0]
-        #lb = [0.0,0.0,0.0,0.0,1.7976931348623157e+308,0.0,0.0]
+        #la = [0.0, 0.0, 0.0, 0.0, 5e-324, 0.0, 0.0, 5e-324, 0.0, 0.0]
         l = len(la)
+        #lb = [0.0] * l
         lb = data.draw(st.lists(st.floats(), min_size=l, max_size=l))
 
         rawstorage = RawStorage()
@@ -215,6 +215,9 @@
         la = data.draw(st.lists(integers, min_size=10, max_size=150))
         l = len(la)
         lb = data.draw(st.lists(integers, min_size=l, max_size=l))
+        #la = [0] * 10
+        #l = 10
+        #lb = [0] * 10
 
         rawstorage = RawStorage()
         va = rawstorage.new(la, type)

From pypy.commits at gmail.com  Mon Sep  5 11:18:54 2016
From: pypy.commits at gmail.com (sbauman)
Date: Mon, 05 Sep 2016 08:18:54 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Explicitly pass force_boxes
 arg in all cases (suggested by cfbolz)
Message-ID: <57cd8cde.262ec20a.f163a.afb3@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86884:3ae2ff6dceac
Date: 2016-09-05 11:18 -0400
http://bitbucket.org/pypy/pypy/changeset/3ae2ff6dceac/

Log:	Explicitly pass force_boxes arg in all cases (suggested by cfbolz)

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -184,8 +184,8 @@
                     self.optimizer._newoperations)
 
         try:
-            new_virtual_state = self.jump_to_existing_trace(end_jump, label_op,
-                                                            state.runtime_boxes)
+            new_virtual_state = self.jump_to_existing_trace(
+                    end_jump, label_op, state.runtime_boxes force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
@@ -252,7 +252,8 @@
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
         try:
-            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes, False)
+            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                             force_boxes=False)
         except InvalidLoop:
             return self.jump_to_preamble(cell_token, jump_op, info)
         if vs is None:
@@ -265,7 +266,8 @@
         else:
             # Try forcing boxes to avoid jumping to the preamble
             try:
-                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes, True)
+                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                                 force_boxes=True)
             except InvalidLoop:
                 pass
             if vs is None:

From pypy.commits at gmail.com  Mon Sep  5 12:36:33 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 09:36:33 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Added tag release-pypy2.7-v5.4.1
 for changeset 050d84dd7899
Message-ID: <57cd9f11.02d31c0a.cb3a3.7a23@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86885:4efaca9d74d6
Date: 2016-09-05 19:34 +0300
http://bitbucket.org/pypy/pypy/changeset/4efaca9d74d6/

Log:	Added tag release-pypy2.7-v5.4.1 for changeset 050d84dd7899

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -29,3 +29,4 @@
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1

From pypy.commits at gmail.com  Mon Sep  5 12:36:35 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 09:36:35 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: update for 5.4.1
Message-ID: <57cd9f13.6937c20a.8dc4e.8207@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86886:a58f1f2dc35b
Date: 2016-09-05 19:35 +0300
http://bitbucket.org/pypy/pypy/changeset/a58f1f2dc35b/

Log:	update for 5.4.1

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
 min=4
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Mon Sep  5 12:36:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 09:36:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v5.4.1 for
 changeset 050d84dd7899
Message-ID: <57cd9f15.121a1c0a.10647.7144@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86887:5558faa5f5f9
Date: 2016-09-05 19:35 +0300
http://bitbucket.org/pypy/pypy/changeset/5558faa5f5f9/

Log:	Added tag release-pypy2.7-v5.4.1 for changeset 050d84dd7899 (grafted
	from 4efaca9d74d6913fcd60f0af46988fee4317e295)

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -30,3 +30,4 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1

From pypy.commits at gmail.com  Mon Sep  5 12:36:39 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 09:36:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: update for 5.4.1
Message-ID: <57cd9f17.c41f1c0a.59642.6d6f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86888:1a16bd6a1d86
Date: 2016-09-05 19:35 +0300
http://bitbucket.org/pypy/pypy/changeset/1a16bd6a1d86/

Log:	update for 5.4.1 (grafted from
	a58f1f2dc35bd2d3b5bda83ed86317676bbb55b4)

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
 min=4
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Mon Sep  5 13:04:02 2016
From: pypy.commits at gmail.com (sbauman)
Date: Mon, 05 Sep 2016 10:04:02 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Merge with known working
 version
Message-ID: <57cda582.919a1c0a.2dc4b.83c5@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86889:e06660b2588f
Date: 2016-09-05 13:03 -0400
http://bitbucket.org/pypy/pypy/changeset/e06660b2588f/

Log:	Merge with known working version

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.1"
+__version_info__ = (1, 8, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,43 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.1-alpha0"
-#define PYPY_VERSION_NUM  0x05040100
+#define PYPY_VERSION "5.5.0-alpha0"
+#define PYPY_VERSION_NUM  0x05050000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -32,9 +32,11 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 5, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1479,6 +1479,7 @@
             assert p1[0] == 123
             seen.append(1)
         q = ffi.gc(p, destructor)
+        assert ffi.typeof(q) is ffi.typeof(p)
         import gc; gc.collect()
         assert seen == []
         del q
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -149,3 +149,28 @@
         p = snip_setuptools_verify2.C.getpwuid(0)
         assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
         ''')
+
+    def test_set_py_limited_api(self):
+        from cffi.setuptools_ext import _set_py_limited_api
+        try:
+            import setuptools
+        except ImportError as e:
+            py.test.skip(str(e))
+        orig_version = setuptools.__version__
+        try:
+            setuptools.__version__ = '26.0.0'
+            from setuptools import Extension
+
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+            setuptools.__version__ = '25.0'
+            kwds = _set_py_limited_api(Extension, {})
+            assert not kwds
+
+            setuptools.__version__ = 'development'
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+        finally:
+            setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1975,9 +1975,9 @@
 
 def test_function_returns_partial_struct():
     ffi = FFI()
-    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    ffi.cdef("struct aaa { int a; ...; }; struct aaa f1(int);")
     lib = verify(ffi, "test_function_returns_partial_struct", """
-        struct a { int b, a, c; };
-        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+        struct aaa { int b, a, c; };
+        static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,6 +97,21 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
+if OPENSSL_VERSION_NUMBER >= 0x10100000:
+    eci.pre_include_bits = ()
+    eci.post_include_bits = ()
+    raise Exception("""OpenSSL version >= 1.1 not supported yet.
+
+    This program requires OpenSSL version 1.0.x, and may also
+    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
+    some work to update to; contributions are welcome.  Sorry,
+    you need to install an older version of OpenSSL for now.
+    Make sure this older version is the one picked up by this
+    program when it runs the compiler.
+    
+    This is the configuration used: %r""" % (eci,))
+
+
 class CConfig:
     _compilation_info_ = eci
 
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1,5 +1,5 @@
 import sys
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, enforceargs
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
@@ -145,19 +145,21 @@
 _invalid_byte_3_of_4 = _invalid_cont_byte
 _invalid_byte_4_of_4 = _invalid_cont_byte
 
- at specialize.arg(2)
+ at enforceargs(allow_surrogates=bool)
 def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xe0 and ordch2 < 0xa0)
             # surrogates shouldn't be valid UTF-8!
-            or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f))
+            or (ordch1 == 0xed and ordch2 > 0x9f and not allow_surrogates))
 
 def _invalid_byte_2_of_4(ordch1, ordch2):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
- at specialize.arg(5)
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(4)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:
@@ -330,6 +332,9 @@
     return unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(3)
 def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                               allow_surrogates=False):
     assert(size >= 0)
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -55,7 +55,7 @@
                 s = s.encode(encoding)
         except LookupError as e:
             py.test.skip(e)
-        result, consumed = decoder(s, len(s), True)
+        result, consumed = decoder(s, len(s), 'strict', final=True)
         assert consumed == len(s)
         self.typeequals(trueresult, result)
 
@@ -69,7 +69,7 @@
                 s = s.decode(encoding)
         except LookupError as e:
             py.test.skip(e)
-        result = encoder(s, len(s), True)
+        result = encoder(s, len(s), 'strict')
         self.typeequals(trueresult, result)
 
     def checkencodeerror(self, s, encoding, start, stop):
@@ -823,9 +823,15 @@
         def f(x):
 
             s1 = "".join(["\xd7\x90\xd6\x96\xeb\x96\x95\xf0\x90\x91\x93"] * x)
-            u, consumed = runicode.str_decode_utf_8(s1, len(s1), True)
-            s2 = runicode.unicode_encode_utf_8(u, len(u), True)
-            return s1 == s2
+            u, consumed = runicode.str_decode_utf_8(s1, len(s1), 'strict',
+                                                    allow_surrogates=True)
+            s2 = runicode.unicode_encode_utf_8(u, len(u), 'strict',
+                                                    allow_surrogates=True)
+            u3, consumed3 = runicode.str_decode_utf_8(s1, len(s1), 'strict',
+                                                    allow_surrogates=False)
+            s3 = runicode.unicode_encode_utf_8(u3, len(u3), 'strict',
+                                                    allow_surrogates=False)
+            return s1 == s2 == s3
         res = interpret(f, [2])
         assert res
 
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -540,7 +540,7 @@
     def ll_ullong_py_mod_zer(x, y):
         if y == 0:
             raise ZeroDivisionError
-        return llop.ullong_mod(UnsignedLongLong, x, y)
+        return ll_ullong_py_mod(x, y)
 
 @jit.dont_look_inside
 def ll_lllong_py_mod(x, y):
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -30,12 +30,13 @@
         assert value is not None
         result = UnicodeBuilder(len(value))
         self.rstr_decode_utf_8(
-            value, len(value), 'strict', final=False,
+            value, len(value), 'strict', final=True,
             errorhandler=self.ll_raise_unicode_exception_decode,
             allow_surrogates=False, result=result)
         return self.ll.llunicode(result.build())
 
-    def ll_raise_unicode_exception_decode(self, errors, encoding, msg, s,
+    @staticmethod
+    def ll_raise_unicode_exception_decode(errors, encoding, msg, s,
                                        startingpos, endingpos):
         raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
 
@@ -411,7 +412,8 @@
             allow_surrogates=False)
         return self.ll.llstr(bytes)
 
-    def ll_raise_unicode_exception_encode(self, errors, encoding, msg, u,
+    @staticmethod
+    def ll_raise_unicode_exception_encode(errors, encoding, msg, u,
                                           startingpos, endingpos):
         raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -162,6 +162,18 @@
 
         assert self.ll_to_string(self.interpret(f, [0])) == f(0)
 
+    def test_unicode_decode_final(self):
+        strings = ['\xc3', '']
+        def f(n):
+            try:
+                strings[n].decode('utf-8')
+            except UnicodeDecodeError:
+                return True
+            return False
+
+        assert f(0)
+        assert self.interpret(f, [0])
+
     def test_utf_8_decoding_annotation(self):
         from rpython.rlib.runicode import str_decode_utf_8
         def errorhandler(errors, encoding, msg, s,

From pypy.commits at gmail.com  Mon Sep  5 13:31:49 2016
From: pypy.commits at gmail.com (sbauman)
Date: Mon, 05 Sep 2016 10:31:49 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Missing comma
Message-ID: <57cdac05.c3f0c20a.54624.f434@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86890:63719b0c0290
Date: 2016-09-05 13:30 -0400
http://bitbucket.org/pypy/pypy/changeset/63719b0c0290/

Log:	Missing comma

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -185,7 +185,7 @@
 
         try:
             new_virtual_state = self.jump_to_existing_trace(
-                    end_jump, label_op, state.runtime_boxes force_boxes=False)
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)

From pypy.commits at gmail.com  Mon Sep  5 14:32:17 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 05 Sep 2016 11:32:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Skip test_random_attr.py on pypy3
Message-ID: <57cdba31.4676c20a.2e4e.3266@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86891:0b9f47589e08
Date: 2016-08-31 16:50 +0100
http://bitbucket.org/pypy/pypy/changeset/0b9f47589e08/

Log:	Skip test_random_attr.py on pypy3

diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,4 +1,5 @@
 import pytest
+pytest.skip("This cannot possibly work on pypy3")
 import sys
 try:
     import __pypy__

From pypy.commits at gmail.com  Mon Sep  5 14:32:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Sep 2016 11:32:43 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: md5/sha1/sha256 for pypy 5.4.1
Message-ID: <57cdba4b.861b1c0a.6b21.9b0b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r785:f8f81b14f7ad
Date: 2016-09-05 20:32 +0200
http://bitbucket.org/pypy/pypy.org/changeset/f8f81b14f7ad/

Log:	md5/sha1/sha256 for pypy 5.4.1

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -448,20 +448,20 @@
 
 Here are the checksums for each of the downloads
 
-pypy2.7-v5.4.0 md5::
+pypy2.7-v5.4.1 md5::
 
-    50ea504e66f4d9297f5228d7a3b026ec  pypy2-v5.4.0-linux-armel.tar.bz2
-    e838ba554bc53c793f23c378a898fa0f  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
-    b1b9b755631ef85d400d7690ece50210  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
-    df7180d5070ac19a234fc6c39b88f420  pypy2-v5.4.0-linux32.tar.bz2
-    5e228ba05b6eaa0af37321fd3f425891  pypy2-v5.4.0-linux64.tar.bz2
-    b32d4c97275901665945f1f2813b6f26  pypy2-v5.4.0-osx64.tar.bz2
-    1d32ef8036a9fe718f397813bd070be8  pypy2-v5.4.0-ppc64.tar.bz2
-    d8abb09416b4370ea40c51a710d12b18  pypy2-v5.4.0-ppc64le.tar.bz2
-    b560c2811a3089f22b21db9beea7f273  pypy2-v5.4.0-s390x.tar.bz2
-    c806bea7ecbb999fffeea3a06e6462e8  pypy2-v5.4.0-src.tar.bz2
-    26c2ab1c891651eb620dbde499088c1f  pypy2-v5.4.0-src.zip
-    bd25b15c0d6c0f7c7f6fa75f1da35014  pypy2-v5.4.0-win32.zip
+    70958c05af6628a66db5072ef1c72522  pypy2-v5.4.1-linux-armel.tar.bz2
+    5246b7b963689ec5b70291c2b104476a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+    106cfa49756df7ae3bf531ce6659d0ed  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+    f561bedb338fa09f011eaf1edac0faf1  pypy2-v5.4.1-linux32.tar.bz2
+    df1327fc3cd41a1ff860e90b5a901899  pypy2-v5.4.1-linux64.tar.bz2
+    b24ebe9f4825fc05afd76a9e8f47018c  pypy2-v5.4.1-osx64.tar.bz2
+    b8e90edf11639d1757413c8bf5a11d49  pypy2-v5.4.1-ppc64.tar.bz2
+    55f6747988d981699e52a7c60aed8a7f  pypy2-v5.4.1-ppc64le.tar.bz2
+    082cca9bb948c8b1389c35db7174396a  pypy2-v5.4.1-s390x.tar.bz2
+    129e730c84f55133b9694bc48e2d4812  pypy2-v5.4.1-src.tar.bz2
+    f20a420d90475b72c6ef8b4ab90377f0  pypy2-v5.4.1-src.zip
+    125874d61b4ac4e2fd7d0b7c2db3b041  pypy2-v5.4.1-win32.zip
 
 pypy3.3-v5.2-alpha md5::
 
@@ -481,20 +481,20 @@
     009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 
 
-pypy2.7-5.4.0 sha1::
+pypy2.7-5.4.1 sha1::
 
-    c50062a83e4bb9fc59b76901c92e7bf1ecd0351f  pypy2-v5.4.0-linux-armel.tar.bz2
-    f4ebad7a9a31dfa55b35cc01b0533ef8e31ab7c4  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
-    c0becdcb7f44e09947afab9df759313ec94563ef  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
-    63be7254bdecd4f3272bcc47f0da7f5db82435a0  pypy2-v5.4.0-linux32.tar.bz2
-    b0e0405ca8f3b143e16122767eb5605d3388af0c  pypy2-v5.4.0-linux64.tar.bz2
-    9c97f54d492886fcaae8611733bcc40a625c8245  pypy2-v5.4.0-osx64.tar.bz2
-    4a263167bbc89447e5adc2ed687ed44798bbca08  pypy2-v5.4.0-ppc64.tar.bz2
-    b3554db74a826fd8e86f1132e9c2cb2e49caac1c  pypy2-v5.4.0-ppc64le.tar.bz2
-    165920a2d0eeda83e8808e7fce93f2a9db7f736a  pypy2-v5.4.0-s390x.tar.bz2
-    95163f8f3c8e9e52e126fc1807d8d94e3d224aec  pypy2-v5.4.0-src.tar.bz2
-    b26546821836cb4bfda0160d37d4dd31fd3aace8  pypy2-v5.4.0-src.zip
-    5ec0ca235cc68b557770b8cf5e1e49bd7b1a0aad  pypy2-v5.4.0-win32.zip
+    74fea5a7a0a3d8c899404ddc8c0296d4bf4ca3d0  pypy2-v5.4.1-linux-armel.tar.bz2
+    4c72d94325567d2079ca5021da3cba6cbc835744  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+    18486da20d2513c083be308f8222f83f80c74671  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+    c38c40dfcbe9cba712a508fed0ea3dc6da5e2b7c  pypy2-v5.4.1-linux32.tar.bz2
+    2c9a45c1bf67d8f2fac9d09e082f059ce892f291  pypy2-v5.4.1-linux64.tar.bz2
+    765bed9e45fa58a6f65ae20ff28b5e39beb56793  pypy2-v5.4.1-osx64.tar.bz2
+    6e73fa972ca01d58e95b36f133319bff0cc66876  pypy2-v5.4.1-ppc64.tar.bz2
+    4410c7514328d936084b00ac9d01af1aecdc7289  pypy2-v5.4.1-ppc64le.tar.bz2
+    8726412fcbaea859f21fb4b1c21fd5832e4c56d7  pypy2-v5.4.1-s390x.tar.bz2
+    0d865c16a3779f492b7f4687cd46c21bbfc05609  pypy2-v5.4.1-src.tar.bz2
+    8f898a052786d3b60e9effe162c15fa572a5f52d  pypy2-v5.4.1-src.zip
+    21958b782dc727a0be3bbc248e0ca9af18305654  pypy2-v5.4.1-win32.zip
 
 pypy3.3-v5.2-alpha sha1::
 
@@ -508,20 +508,20 @@
     4b31ab492716ea375dd090bbacdf3d7c2d483059  pypy3.3-v5.2.0-alpha1-src.tar.bz2
     d9f5b64f144ebec1a200156809fbbe04fdf7eb7e  pypy3.3-v5.2.0-alpha1-src.zip
 
-pypy2.7-5.4.0 sha256::
+pypy2.7-5.4.1 sha256::
 
-    04509044f21bb41ee6d3fafcf637fc0c586c248d4cdae6ac3357606a7b660fdb  pypy2-v5.4.0-linux-armel.tar.bz2
-    95c690bcae6771ebce6cf06c7c2842e0662e007e35162afc963337aa597b471a  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
-    839b08db89b7e20cb670b8cf02596e033ea0b76fb8336af7bedfbb04b6b502da  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
-    ce581270464b14cdecd13dedb9bd7bf98232f767ac4ac282229a405d8e807af1  pypy2-v5.4.0-linux32.tar.bz2
-    bdfea513d59dcd580970cb6f79f3a250d00191fd46b68133d5327e924ca845f8  pypy2-v5.4.0-linux64.tar.bz2
-    3adf21c2bf3432759c99123f21240d71a72aba81d73129e48ef912c34631b723  pypy2-v5.4.0-osx64.tar.bz2
-    dc09a057264dafb7e4bceca57b6a6ba3b0a5273e125a9b29da32b8439f980270  pypy2-v5.4.0-ppc64.tar.bz2
-    4feb0711e7c235b247f8ea0b22e8a676f89e8831488b7a4e9c7f3a6943d07052  pypy2-v5.4.0-ppc64le.tar.bz2
-    6bceb2760b1c7d6105d20207102862160ddddfd9b1a2707b3a8d866ac29e08d3  pypy2-v5.4.0-s390x.tar.bz2
-    d9568ebe9a14d0eaefde887d78f3cba63d665e95c0d234bb583932341f55a655  pypy2-v5.4.0-src.tar.bz2
-    3c165676be8df3b482727438836a9a240ea641392ddd60593f825e1d50029022  pypy2-v5.4.0-src.zip
-    442c0a917781b6155bf78d2648f1ccd9a36c321926a043f83efcea22a99960b4  pypy2-v5.4.0-win32.zip
+    8925b76fe9ca6f960d8f914ed67f7a3c52ce2b4c65fa71a5ef7d4b285c2c3a36  pypy2-v5.4.1-linux-armel.tar.bz2
+    0213b0d948ae0afea8b4cb93f08e55b0562522b3ab8f2706c4e22ffe8cd86f84  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+    2daee13ec1836c1041c89c18d9514134ff606dc3648fc6304611eb1ec0819289  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+    85bccf8679f908c08850115fe74325474fe1b2e1e793c147d1fa484b56472b12  pypy2-v5.4.1-linux32.tar.bz2
+    06c29d59565d9fdb618ed8aa730e05cf975da21158955591dff38d9e305af074  pypy2-v5.4.1-linux64.tar.bz2
+    507c81af9ca302c67a582255306529f88fba56760d353e17d667a114eee1f7e2  pypy2-v5.4.1-osx64.tar.bz2
+    c11b37f5e97b647003426987e223d75a0dc0da1ecc35675ddad0af8a9add972d  pypy2-v5.4.1-ppc64.tar.bz2
+    652d97fbd574d0349f7aa8b37c8c5a1238ed0cd3d6b68cf2ea8280b7ead4c7ad  pypy2-v5.4.1-ppc64le.tar.bz2
+    987b3354dcbed5fd3f0d8d9d1484a259f0dff97da5d11a84b354c6e61a4af891  pypy2-v5.4.1-s390x.tar.bz2
+    92af82664ace96d721c66dbe8726d4f39c7d01f568d9df56c11149be2960238f  pypy2-v5.4.1-src.tar.bz2
+    08148d1157dd16f402c7844fc0cdfde9e7d187c7fd1549a93e888e2fd13828bf  pypy2-v5.4.1-src.zip
+    b703224af4e99243d090783a7b685063da7ba01ef28bb99a89cacfce2fb0dfc2  pypy2-v5.4.1-win32.zip
 
 pypy3.3-v5.2-alpha sha256::
 

From pypy.commits at gmail.com  Mon Sep  5 15:16:08 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 05 Sep 2016 12:16:08 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase
Message-ID: <57cdc478.c75dc20a.fc357.14ed@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86892:37cdb0f205ea
Date: 2016-09-05 21:15 +0200
http://bitbucket.org/pypy/pypy/changeset/37cdb0f205ea/

Log:	Update qcgc codebase

diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -2,7 +2,8 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper import rmodel
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.memory.gctransform.framework import (BaseFrameworkGCTransformer, BaseRootWalker)
+from rpython.memory.gctransform.framework import (BaseFrameworkGCTransformer,
+        BaseRootWalker, TYPE_ID, WEAKREF, WEAKREFPTR)
 
 VISIT_FPTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
 
@@ -34,6 +35,12 @@
                   [SomeAddress(), SomePtr(VISIT_FPTR)],
                   s_None))
 
+    #Compilation error when overriding, no idea why
+    #def finish_tables(self):
+    #    BaseFrameworkGCTransformer.finish_tables(self)
+        #Makes test fail, works when translating pypy (but compiling still fails)
+        #assert len(self.layoutbuilder.addresses_of_static_ptrs_in_nongc) == 2
+
     def gc_header_for(self, obj, needs_hash=False):
         hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj)
         if needs_hash:
@@ -77,6 +84,40 @@
 #        hop.genop("cast_adr_to_ptr", [v_adr],
 #                  resultvar = hop.spaceop.result)
 
+    def gct_weakref_create(self, hop):
+        # Custom weakref creation as their registration is slightly different
+        op = hop.spaceop
+
+        type_id = self.get_type_id(WEAKREF)
+
+        c_type_id = rmodel.inputconst(TYPE_ID, type_id)
+        info = self.layoutbuilder.get_info(type_id)
+        c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
+        malloc_ptr = self.malloc_fixedsize_ptr
+        c_false = rmodel.inputconst(lltype.Bool, False)
+        c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
+        args = [self.c_const_gc, c_type_id, c_size,
+                c_false, c_false, c_has_weakptr]
+
+        # push and pop the current live variables *including* the argument
+        # to the weakref_create operation, which must be kept alive if the GC
+        # needs to collect
+        livevars = self.push_roots(hop, keep_current_args=True)
+        v_result = hop.genop("direct_call", [malloc_ptr] + args,
+                             resulttype=llmemory.GCREF)
+        v_result = hop.genop("cast_opaque_ptr", [v_result],
+                            resulttype=WEAKREFPTR)
+        self.pop_roots(hop, livevars)
+        #
+        v_instance, = op.args
+        v_addr = hop.genop("cast_ptr_to_adr", [v_instance],
+                           resulttype=llmemory.Address)
+        hop.genop("bare_setfield",
+                  [v_result, rmodel.inputconst(lltype.Void, "weakptr"), v_addr])
+        v_weakref = hop.genop("cast_ptr_to_weakrefptr", [v_result],
+                              resulttype=llmemory.WeakRefPtr)
+        hop.cast_result(v_weakref)
+
 class QcgcRootWalker(BaseRootWalker):
     def walk_stack_roots(self, collect_stack_root, is_minor=False):
         raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -515,10 +515,9 @@
     # can malloc a GC object.
 
     # __________ qcgc operations __________
-    'qcgc_allocate':    LLOp(canmallocgc=True),
-    'qcgc_collect':     LLOp(canmallocgc=True),
-    'qcgc_is_prebuilt': LLOp(),
-    'qcgc_write_barrier':   LLOp(),
+    'qcgc_allocate':            LLOp(canmallocgc=True),
+    'qcgc_collect':             LLOp(canmallocgc=True),
+    'qcgc_write_barrier':       LLOp(),
 
     # __________ weakrefs __________
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -960,12 +960,6 @@
     def OP_QCGC_COLLECT(self, op):
         return 'qcgc_collect();'
 
-    def OP_QCGC_IS_PREBUILT(self, op):
-        obj = self.expr(op.args[0])
-        result = self.expr(op.result)
-        return '%s = (((object_t *) %s)->flags & QCGC_PREBUILT_OBJECT) != 0;' % (
-                result, obj)
-
     def OP_QCGC_WRITE_BARRIER(self, op):
         obj = self.expr(op.args[0])
         return 'qcgc_write(%s);' % (obj,)
diff --git a/rpython/translator/c/src/qcgc/bag.c b/rpython/translator/c/src/qcgc/bag.c
--- a/rpython/translator/c/src/qcgc/bag.c
+++ b/rpython/translator/c/src/qcgc/bag.c
@@ -6,3 +6,4 @@
 DEFINE_BAG(linear_free_list, cell_t *);
 DEFINE_BAG(exp_free_list, struct exp_free_list_item_s);
 DEFINE_BAG(hbbucket, struct hbtable_entry_s);
+DEFINE_BAG(weakref_bag, struct weakref_bag_item_s);
diff --git a/rpython/translator/c/src/qcgc/bag.h b/rpython/translator/c/src/qcgc/bag.h
--- a/rpython/translator/c/src/qcgc/bag.h
+++ b/rpython/translator/c/src/qcgc/bag.h
@@ -92,7 +92,13 @@
 	bool mark_flag;
 };
 
+struct weakref_bag_item_s {
+	object_t *weakrefobj;
+	object_t **target;
+};
+
 DECLARE_BAG(arena_bag, arena_t *);
 DECLARE_BAG(linear_free_list, cell_t *);
 DECLARE_BAG(exp_free_list, struct exp_free_list_item_s);
 DECLARE_BAG(hbbucket, struct hbtable_entry_s);
+DECLARE_BAG(weakref_bag, struct weakref_bag_item_s);
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -1,15 +1,14 @@
 #pragma once
 
-#define CHECKED 0							// Enable runtime sanity checks
-											// warning: huge performance impact
-#define DEBUG_ZERO_ON_SWEEP 0				// Zero memory on sweep (debug only)
+#define CHECKED 1							// Enable runtime sanity checks
+#define DEBUG_ZERO_ON_SWEEP 1				// Zero memory on sweep (debug only)
 
 #define QCGC_INIT_ZERO 1					// Init new objects with zero bytes
 
 /**
  * Event logger
  */
-#define EVENT_LOG 0							// Enable event log
+#define EVENT_LOG 1							// Enable event log
 #define LOGFILE "./qcgc_events.log"			// Default logfile
 #define LOG_ALLOCATION 0					// Enable allocation log (warning:
 											// significant performance impact)
@@ -31,6 +30,12 @@
 #define QCGC_SMALL_FREE_LIST_INIT_SIZE 16	// Initial size for small free lists
 
 /**
+ * Auto Mark/Collect
+ */
+#define QCGC_MAJOR_COLLECTION_THRESHOLD (5 * (1<<QCGC_ARENA_SIZE_EXP))
+#define QCGC_INCMARK_THRESHOLD (1<<QCGC_ARENA_SIZE_EXP)
+
+/**
  * DO NOT MODIFY BELOW HERE
  */
 
diff --git a/rpython/translator/c/src/qcgc/gc_state.h b/rpython/translator/c/src/qcgc/gc_state.h
--- a/rpython/translator/c/src/qcgc/gc_state.h
+++ b/rpython/translator/c/src/qcgc/gc_state.h
@@ -2,6 +2,8 @@
 
 #include <stddef.h>
 
+#include "bag.h"
+#include "gray_stack.h"
 #include "shadow_stack.h"
 
 /**
@@ -25,7 +27,10 @@
 struct qcgc_state {
 	shadow_stack_t *shadow_stack;
 	shadow_stack_t *prebuilt_objects;
+	weakref_bag_t *weakrefs;
 	gray_stack_t *gp_gray_stack;
 	size_t gray_stack_size;
 	gc_phase_t phase;
+	size_t bytes_since_collection;
+	size_t bytes_since_incmark;
 } qcgc_state;
diff --git a/rpython/translator/c/src/qcgc/hugeblocktable.c b/rpython/translator/c/src/qcgc/hugeblocktable.c
--- a/rpython/translator/c/src/qcgc/hugeblocktable.c
+++ b/rpython/translator/c/src/qcgc/hugeblocktable.c
@@ -45,6 +45,17 @@
 	return false;
 }
 
+bool qcgc_hbtable_has(object_t *object) {
+	hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
+	size_t count = b->count;
+	for (size_t i = 0; i < count; i++) {
+		if (b->items[i].object == object) {
+			return true;
+		}
+	}
+	return false;
+}
+
 bool qcgc_hbtable_is_marked(object_t *object) {
 	hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
 	size_t count = b->count;
diff --git a/rpython/translator/c/src/qcgc/hugeblocktable.h b/rpython/translator/c/src/qcgc/hugeblocktable.h
--- a/rpython/translator/c/src/qcgc/hugeblocktable.h
+++ b/rpython/translator/c/src/qcgc/hugeblocktable.h
@@ -20,5 +20,6 @@
 void qcgc_hbtable_destroy(void);
 void qcgc_hbtable_insert(object_t *object);
 bool qcgc_hbtable_mark(object_t *object);
+bool qcgc_hbtable_has(object_t *object);
 bool qcgc_hbtable_is_marked(object_t *object);
 void qcgc_hbtable_sweep(void);
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -10,20 +10,41 @@
 #include "hugeblocktable.h"
 #include "event_logger.h"
 
+#define env_or_fallback(var, env_name, fallback) while(0) {		\
+	char *env_val = getenv(env_name);							\
+	if (env_val != NULL) {										\
+		if (1 != sscanf(env_val, "%zu", &var)) {				\
+			var = fallback;										\
+		}														\
+	}															\
+}
+
 void qcgc_mark(bool incremental);
 void qcgc_pop_object(object_t *object);
 void qcgc_push_object(object_t *object);
 void qcgc_sweep(void);
 
+static size_t major_collection_threshold = QCGC_MAJOR_COLLECTION_THRESHOLD;
+static size_t incmark_threshold = QCGC_INCMARK_THRESHOLD;
+
+QCGC_STATIC void update_weakrefs(void);
+
 void qcgc_initialize(void) {
 	qcgc_state.shadow_stack = qcgc_shadow_stack_create(QCGC_SHADOWSTACK_SIZE);
-	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); //XXX
+	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); // XXX
+	qcgc_state.weakrefs = qcgc_weakref_bag_create(16); // XXX
 	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
 	qcgc_state.gray_stack_size = 0;
 	qcgc_state.phase = GC_PAUSE;
+	qcgc_state.bytes_since_collection = 0;
+	qcgc_state.bytes_since_incmark = 0;
 	qcgc_allocator_initialize();
 	qcgc_hbtable_initialize();
 	qcgc_event_logger_initialize();
+
+	env_or_fallback(major_collection_threshold, "QCGC_MAJOR_COLLECTION",
+			QCGC_MAJOR_COLLECTION_THRESHOLD);
+	env_or_fallback(incmark_threshold, "QCGC_INCMARK", QCGC_INCMARK_THRESHOLD);
 }
 
 void qcgc_destroy(void) {
@@ -32,6 +53,7 @@
 	qcgc_allocator_destroy();
 	free(qcgc_state.shadow_stack);
 	free(qcgc_state.prebuilt_objects);
+	free(qcgc_state.weakrefs);
 	free(qcgc_state.gp_gray_stack);
 }
 
@@ -113,6 +135,14 @@
 			(uint8_t *) &size);
 #endif
 	object_t *result;
+
+	if (qcgc_state.bytes_since_collection > major_collection_threshold) {
+		qcgc_collect();
+	}
+	if (qcgc_state.bytes_since_incmark > incmark_threshold) {
+		qcgc_mark(true);
+	}
+
 	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
 		// Use bump / fit allocator
 		if (true) { // FIXME: Implement reasonable switch
@@ -130,6 +160,11 @@
 		result = qcgc_large_allocate(size);
 	}
 
+	// XXX: Should we use cells instead of bytes?
+	qcgc_state.bytes_since_collection += size;
+	qcgc_state.bytes_since_incmark += size;
+
+
 #if LOG_ALLOCATION
 	qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
 			(uint8_t *) &result);
@@ -165,11 +200,12 @@
 }
 
 void qcgc_mark(bool incremental) {
-#if CHECKED
-	assert(qcgc_state.phase == GC_PAUSE || qcgc_state.phase == GC_MARK);
-#endif
+	if (qcgc_state.phase == GC_COLLECT) {
+		return;	// Fast exit when there is nothing to mark
+	}
 	// FIXME: Log some more information
 	qcgc_event_logger_log(EVENT_MARK_START, 0, NULL);
+	qcgc_state.bytes_since_incmark = 0;
 
 	if (qcgc_state.phase == GC_PAUSE) {
 		qcgc_state.phase = GC_MARK;
@@ -329,9 +365,79 @@
 	qcgc_state.phase = GC_PAUSE;
 
 	qcgc_event_logger_log(EVENT_SWEEP_DONE, 0, NULL);
+	update_weakrefs();
 }
 
 void qcgc_collect(void) {
 	qcgc_mark(false);
 	qcgc_sweep();
+	qcgc_state.bytes_since_collection = 0;
 }
+
+void qcgc_register_weakref(object_t *weakrefobj, object_t **target) {
+#if CHECKED
+	assert((weakrefobj->flags & QCGC_PREBUILT_OBJECT) == 0);
+	assert((object_t *) qcgc_arena_addr((cell_t *) weakrefobj) != weakrefobj);
+#endif
+	// NOTE: At this point, the target must point to a pointer to a valid
+	// object. We don't register any weakrefs to prebuilt objects as they
+	// are always valid.
+	if (((*target)->flags & QCGC_PREBUILT_OBJECT) == 0) {
+		qcgc_state.weakrefs = qcgc_weakref_bag_add(qcgc_state.weakrefs,
+				(struct weakref_bag_item_s) {
+					.weakrefobj = weakrefobj,
+					.target = target});
+	}
+}
+
+QCGC_STATIC void update_weakrefs(void) {
+	size_t i = 0;
+	while (i < qcgc_state.weakrefs->count) {
+		struct weakref_bag_item_s item = qcgc_state.weakrefs->items[i];
+		// Check whether weakref object itself was collected
+		// We know the weakref object is a normal object
+		switch(qcgc_arena_get_blocktype((cell_t *) item.weakrefobj)) {
+			case BLOCK_EXTENT: // Fall through
+			case BLOCK_FREE:
+				// Weakref itself was collected, forget it
+				qcgc_state.weakrefs = qcgc_weakref_bag_remove_index(
+						qcgc_state.weakrefs, i);
+				continue;
+			case BLOCK_BLACK:
+			case BLOCK_WHITE:
+				// Weakref object is still valid, continue
+				break;
+		}
+
+		// Check whether the weakref target is still valid
+		object_t *points_to = *item.target;
+		if ((object_t *) qcgc_arena_addr((cell_t *) points_to) ==
+				points_to) {
+			// Huge object
+			if (qcgc_hbtable_has(points_to)) {
+				// Still valid
+				i++;
+			} else {
+				// Invalid
+				*(item.target) = NULL;
+				qcgc_state.weakrefs = qcgc_weakref_bag_remove_index(
+						qcgc_state.weakrefs, i);
+			}
+		} else {
+			// Normal object
+			switch(qcgc_arena_get_blocktype((cell_t *) points_to)) {
+				case BLOCK_BLACK: // Still valid
+				case BLOCK_WHITE:
+					i++;
+					break;
+				case BLOCK_EXTENT: // Fall through
+				case BLOCK_FREE:
+					// Invalid
+					*(item.target) = NULL;
+					qcgc_state.weakrefs = qcgc_weakref_bag_remove_index(
+							qcgc_state.weakrefs, i);
+					break;
+			}
+		}
+	}
+}
diff --git a/rpython/translator/c/src/qcgc/qcgc.h b/rpython/translator/c/src/qcgc/qcgc.h
--- a/rpython/translator/c/src/qcgc/qcgc.h
+++ b/rpython/translator/c/src/qcgc/qcgc.h
@@ -83,6 +83,15 @@
 object_t *qcgc_shadowstack_pop(void);
 
 /**
+ * Weakref registration
+ *
+ * @param	weakrefobj	Pointer to the weakref itself
+ * @param	target		Doublepointer to referenced object.
+ *						The referenced object must be a valid object.
+ */
+void qcgc_register_weakref(object_t *weakrefobj, object_t **target);
+
+/**
  * Tracing function.
  *
  * This function traces an object, i.e. calls visit on every object referenced
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1280,6 +1280,9 @@
     GC_CAN_MOVE = False
     GC_CAN_SHRINK_ARRAY = False
     removetypeptr = True
+    
+    def test_framework_static_roots(self):
+        py.test.skip("not implemented")
 
 
 

From pypy.commits at gmail.com  Mon Sep  5 16:03:20 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 05 Sep 2016 13:03:20 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Skip correct test,
 disable checks in qcgc and prepare weakref registration
Message-ID: <57cdcf88.581d1c0a.fa58a.4f36@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86893:a2b97ba105dc
Date: 2016-09-05 22:02 +0200
http://bitbucket.org/pypy/pypy/changeset/a2b97ba105dc/

Log:	Skip correct test, disable checks in qcgc and prepare weakref
	registration

diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -112,10 +112,15 @@
         v_instance, = op.args
         v_addr = hop.genop("cast_ptr_to_adr", [v_instance],
                            resulttype=llmemory.Address)
+        c_weakptr = rmodel.inputconst(lltype.Void, "weakptr")
         hop.genop("bare_setfield",
-                  [v_result, rmodel.inputconst(lltype.Void, "weakptr"), v_addr])
+                  [v_result, c_weakptr, v_addr])
         v_weakref = hop.genop("cast_ptr_to_weakrefptr", [v_result],
                               resulttype=llmemory.WeakRefPtr)
+        # Register weakref
+        v_fieldaddr = hop.genop("direct_fieldptr", [v_result, c_weakptr],
+                            resulttype=llmemory.Address)
+        #hop.genop("qcgc_register_weakref", [v_result, v_fieldaddr])
         hop.cast_result(v_weakref)
 
 class QcgcRootWalker(BaseRootWalker):
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#define CHECKED 1							// Enable runtime sanity checks
+#define CHECKED 0							// Enable runtime sanity checks
 #define DEBUG_ZERO_ON_SWEEP 1				// Zero memory on sweep (debug only)
 
 #define QCGC_INIT_ZERO 1					// Init new objects with zero bytes
@@ -8,7 +8,7 @@
 /**
  * Event logger
  */
-#define EVENT_LOG 1							// Enable event log
+#define EVENT_LOG 0							// Enable event log
 #define LOGFILE "./qcgc_events.log"			// Default logfile
 #define LOG_ALLOCATION 0					// Enable allocation log (warning:
 											// significant performance impact)
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1281,7 +1281,7 @@
     GC_CAN_SHRINK_ARRAY = False
     removetypeptr = True
     
-    def test_framework_static_roots(self):
+    def test_framework_nongc_static_root(self):
         py.test.skip("not implemented")
 
 

From pypy.commits at gmail.com  Mon Sep  5 16:15:42 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 05 Sep 2016 13:15:42 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Fix weakref test
Message-ID: <57cdd26e.411d1c0a.3eba2.cc39@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86895:b88d51ba77e6
Date: 2016-09-05 22:12 +0200
http://bitbucket.org/pypy/pypy/changeset/b88d51ba77e6/

Log:	Fix weakref test

diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1280,6 +1280,50 @@
     GC_CAN_MOVE = False
     GC_CAN_SHRINK_ARRAY = False
     removetypeptr = True
+
+    # Hoping I can do this here
+    def define_weakref(cls):
+        import weakref
+
+        class A:
+            pass
+
+        keepalive = []
+        def collect():
+            # Make sure the bump pointer leaves the current arena
+            for i in range(2**16):
+                a = A()
+            rgc.collect()
+
+        def fn():
+            n = 7000
+            weakrefs = []
+            a = None
+            for i in range(n):
+                if i & 1 == 0:
+                    a = A()
+                    a.index = i
+                assert a is not None
+                weakrefs.append(weakref.ref(a))
+                if i % 7 == 6:
+                    keepalive.append(a)
+            collect()
+            count_free = 0
+            for i in range(n):
+                a = weakrefs[i]()
+                if i % 7 == 6:
+                    assert a is not None
+                if a is not None:
+                    assert a.index == i & ~1
+                else:
+                    count_free += 1
+            return count_free
+        return fn
+
+    def test_weakref(self):
+        res = self.run('weakref')
+        # more than half of them should have been freed, ideally up to 6000
+        assert 3500 <= res <= 6000
     
     def test_framework_nongc_static_root(self):
         py.test.skip("not implemented")

From pypy.commits at gmail.com  Mon Sep  5 16:15:40 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 05 Sep 2016 13:15:40 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Register weakrefs (test failing
 due to not collected targets)
Message-ID: <57cdd26c.8f8e1c0a.41a0.c7a8@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86894:7c5c4f76938f
Date: 2016-09-05 22:08 +0200
http://bitbucket.org/pypy/pypy/changeset/7c5c4f76938f/

Log:	Register weakrefs (test failing due to not collected targets)

diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -120,7 +120,7 @@
         # Register weakref
         v_fieldaddr = hop.genop("direct_fieldptr", [v_result, c_weakptr],
                             resulttype=llmemory.Address)
-        #hop.genop("qcgc_register_weakref", [v_result, v_fieldaddr])
+        hop.genop("qcgc_register_weakref", [v_result, v_fieldaddr])
         hop.cast_result(v_weakref)
 
 class QcgcRootWalker(BaseRootWalker):
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -963,3 +963,8 @@
     def OP_QCGC_WRITE_BARRIER(self, op):
         obj = self.expr(op.args[0])
         return 'qcgc_write(%s);' % (obj,)
+
+    def OP_QCGC_REGISTER_WEAKREF(self, op):
+        weakref = self.expr(op.args[0])
+        fieldaddr = self.expr(op.args[1])
+        return 'qcgc_register_weakref(%s, %s);' % (weakref, fieldaddr)

From pypy.commits at gmail.com  Mon Sep  5 16:57:01 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 13:57:01 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: try to update documentation
Message-ID: <57cddc1d.6740c20a.d88a6.e49a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86896:59716a90a56a
Date: 2016-09-05 23:53 +0300
http://bitbucket.org/pypy/pypy/changeset/59716a90a56a/

Log:	try to update documentation

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -29,19 +29,17 @@
 ## Solution
 ## --------
 ##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
+## PyBytesObject contains two additional members: the ob_size and an array
+## char ob_sval which holds a \x0 terminated string.
 ##
 ## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   NULL buffer.  The first time PyString_AsString() is called, memory is
-##   allocated (with flavor='raw') and content is copied.
+##   buffer holding \x0.  The first time PyString_AsString() is called, the 
+##   PyStringObject is reallocated, and the string copied into the buffer. The
+##   ob_size reflects the length of the string.
 ##
 ## - A string allocated with PyString_FromStringAndSize(NULL, size) will
 ##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, but the reference won't be stored in the global map; there is no
-##   corresponding object in pypy.  When from_ref() or Py_INCREF() is called,
-##   the pypy string is created, and added to the global map of tracked
-##   objects.  The buffer is then supposed to be immutable.
+##   size+1, as part of the object. The buffer is then supposed to be immutable.
 ##
 ##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
 ##   there is no corresponding pypy object for the string

From pypy.commits at gmail.com  Mon Sep  5 16:57:03 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 13:57:03 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: create fixed arrays of
 Py_MAX_NDIMS, arbitrarily set to 32,
 in Py_buffer rather than malloc and leak memory
Message-ID: <57cddc1f.121a1c0a.10647.ce51@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86897:d765d56cd700
Date: 2016-09-05 23:56 +0300
http://bitbucket.org/pypy/pypy/changeset/d765d56cd700/

Log:	create fixed arrays of Py_MAX_NDIMS, arbitrarily set to 32, in
	Py_buffer rather than malloc and leak memory

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -122,7 +122,7 @@
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,6 +645,9 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.UCHAR),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -142,7 +142,8 @@
 typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
 typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -156,12 +157,14 @@
     char *format;
     Py_ssize_t *shape;
     Py_ssize_t *strides;
-    Py_ssize_t *suboffsets;
-
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format;
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
        mono-dimensional buffers. */
     /* Py_ssize_t smalltable[2]; */
-    void *internal;
+    void *internal; /* always NULL for app-level objects */
 } Py_buffer;
 
 
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,5 +1,5 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers, Py_ssize_tP)
+                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -24,6 +24,10 @@
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
     if not isinstance(w_obj, W_MemoryView):
         return view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
     try:
         view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
         view.c_obj = make_ref(space, w_obj)
@@ -36,11 +40,11 @@
         isstr = True
     view.c_len = w_obj.getlength()
     view.c_itemsize = w_obj.buf.getitemsize()
-    ndim = w_obj.buf.getndim()
     rffi.setintfield(view, 'c_ndim', ndim)
-    view.c_format = rffi.str2charp(w_obj.buf.getformat())
-    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
-    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
     shape = w_obj.buf.getshape()
     strides = w_obj.buf.getstrides()
     for i in range(ndim):
@@ -50,4 +54,3 @@
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
     return view
 
-

From pypy.commits at gmail.com  Mon Sep  5 17:08:43 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Sep 2016 14:08:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: add release note for 5.4.1
Message-ID: <57cddedb.81091c0a.d9c91.d67d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86898:0c20173f73ec
Date: 2016-09-06 00:08 +0300
http://bitbucket.org/pypy/pypy/changeset/0c20173f73ec/

Log:	add release note for 5.4.1

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -0,0 +1,61 @@
+==========
+PyPy 5.4.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.4.0, released last week,
+due to the following issues:
+
+  * Update list of contributors in documentation and LICENSE file, 
+    this was unfortunately left out of 5.4.0. My apoligies to the new
+    contributors
+
+  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+    dynamically loadable file
+
+  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+
+  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+
+  * Fix for an issue where os.access() accepted a float for mode
+
+  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+    the last byte of a unicode string sometimes
+
+  * Update built-in cffi_ to the soon-to-be-released 1.8.1 version
+
+  * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
+    translation with a message asking for help porting it 
+
+Thanks to those who reported the issues.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _cffi: https://cffi.readthedocs.io
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+

From pypy.commits at gmail.com  Mon Sep  5 18:00:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Sep 2016 15:00:20 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #283: initializer for nested
 anonymous structs inside unions
Message-ID: <57cdeaf4.a3a3c20a.2eacd.0f24@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2761:58bab5bcadd2
Date: 2016-09-05 23:55 +0200
http://bitbucket.org/cffi/cffi/changeset/58bab5bcadd2/

Log:	Issue #283: initializer for nested anonymous structs inside unions

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -187,10 +187,12 @@
     Py_ssize_t cf_offset;
     short cf_bitshift;   /* >= 0: bitshift; or BS_REGULAR or BS_EMPTY_ARRAY */
     short cf_bitsize;
+    unsigned char cf_flags;   /* BF_... */
     struct cfieldobject_s *cf_next;
 } CFieldObject;
 #define BS_REGULAR     (-1)      /* a regular field, not with bitshift */
 #define BS_EMPTY_ARRAY (-2)      /* a field which is an array 'type[0]' */
+#define BF_IGNORE_IN_CTOR  0x01  /* union field not in the first place */
 
 static PyTypeObject CTypeDescr_Type;
 static PyTypeObject CField_Type;
@@ -657,6 +659,7 @@
     {"offset", T_PYSSIZET, OFF(cf_offset), READONLY},
     {"bitshift", T_SHORT, OFF(cf_bitshift), READONLY},
     {"bitsize", T_SHORT, OFF(cf_bitsize), READONLY},
+    {"flags", T_UBYTE, OFF(cf_flags), READONLY},
     {NULL}      /* Sentinel */
 };
 #undef OFF
@@ -1274,24 +1277,14 @@
         return -1;
     }
 
-    if (ct->ct_flags & CT_UNION) {
-        Py_ssize_t n = PyObject_Size(init);
-        if (n < 0)
-            return -1;
-        if (n > 1) {
-            PyErr_Format(PyExc_ValueError,
-                         "initializer for '%s': %zd items given, but "
-                         "only one supported (use a dict if needed)",
-                         ct->ct_name, n);
-            return -1;
-        }
-    }
     if (PyList_Check(init) || PyTuple_Check(init)) {
         PyObject **items = PySequence_Fast_ITEMS(init);
         Py_ssize_t i, n = PySequence_Fast_GET_SIZE(init);
         CFieldObject *cf = (CFieldObject *)ct->ct_extra;
 
         for (i=0; i<n; i++) {
+            while (cf != NULL && (cf->cf_flags & BF_IGNORE_IN_CTOR))
+                cf = cf->cf_next;
             if (cf == NULL) {
                 PyErr_Format(PyExc_ValueError,
                              "too many initializers for '%s' (got %zd)",
@@ -4085,7 +4078,7 @@
 
 static CFieldObject *
 _add_field(PyObject *interned_fields, PyObject *fname, CTypeDescrObject *ftype,
-           Py_ssize_t offset, int bitshift, int fbitsize)
+           Py_ssize_t offset, int bitshift, int fbitsize, int flags)
 {
     int err;
     Py_ssize_t prev_size;
@@ -4098,6 +4091,7 @@
     cf->cf_offset = offset;
     cf->cf_bitshift = bitshift;
     cf->cf_bitsize = fbitsize;
+    cf->cf_flags = flags;
 
     Py_INCREF(fname);
     PyText_InternInPlace(&fname);
@@ -4184,7 +4178,7 @@
     int totalalignment = -1;
     CFieldObject **previous;
     int prev_bitfield_size, prev_bitfield_free;
-    int sflags = 0;
+    int sflags = 0, fflags;
 
     if (!PyArg_ParseTuple(args, "O!O!|Onii:complete_struct_or_union",
                           &CTypeDescr_Type, &ct,
@@ -4270,6 +4264,8 @@
         if (alignment < falign && do_align)
             alignment = falign;
 
+        fflags = (is_union && i > 0) ? BF_IGNORE_IN_CTOR : 0;
+
         if (fbitsize < 0) {
             /* not a bitfield: common case */
             int bs_flag;
@@ -4305,7 +4301,8 @@
                                            cfsrc->cf_type,
                                            boffset / 8 + cfsrc->cf_offset,
                                            cfsrc->cf_bitshift,
-                                           cfsrc->cf_bitsize);
+                                           cfsrc->cf_bitsize,
+                                           cfsrc->cf_flags | fflags);
                     if (*previous == NULL)
                         goto error;
                     previous = &(*previous)->cf_next;
@@ -4315,7 +4312,7 @@
             }
             else {
                 *previous = _add_field(interned_fields, fname, ftype,
-                                        boffset / 8, bs_flag, -1);
+                                        boffset / 8, bs_flag, -1, fflags);
                 if (*previous == NULL)
                     goto error;
                 previous = &(*previous)->cf_next;
@@ -4445,7 +4442,8 @@
                     bitshift = 8 * ftype->ct_size - fbitsize - bitshift;
 
                 *previous = _add_field(interned_fields, fname, ftype,
-                                       field_offset_bytes, bitshift, fbitsize);
+                                       field_offset_bytes, bitshift, fbitsize,
+                                       fflags);
                 if (*previous == NULL)
                     goto error;
                 previous = &(*previous)->cf_next;
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1414,6 +1414,7 @@
         assert p.b == 12
         assert p.c == 14
         assert p.d == 14
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [0, 0, 0, 0])
 
     def test_nested_field_offset_align(self):
         ffi = FFI(backend=self.Backend())
@@ -1453,14 +1454,42 @@
         assert p.b == 0
         assert p.c == 14
         assert p.d == 14
-        p = ffi.new("union foo_u *", {'b': 12})
-        assert p.a == 0
+        p = ffi.new("union foo_u *", {'a': -63, 'b': 12})
+        assert p.a == -63
         assert p.b == 12
-        assert p.c == 0
-        assert p.d == 0
-        # we cannot specify several items in the dict, even though
-        # in theory in this particular case it would make sense
-        # to give both 'a' and 'b'
+        assert p.c == -63
+        assert p.d == -63
+        p = ffi.new("union foo_u *", [123, 456])
+        assert p.a == 123
+        assert p.b == 456
+        assert p.c == 123
+        assert p.d == 123
+        py.test.raises(ValueError, ffi.new, "union foo_u *", [0, 0, 0])
+
+    def test_nested_anonymous_struct_2(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            struct foo_s {
+                int a;
+                union { int b; union { int c, d; }; };
+                int e;
+            };
+        """)
+        assert ffi.sizeof("struct foo_s") == 3 * SIZE_OF_INT
+        p = ffi.new("struct foo_s *", [11, 22, 33])
+        assert p.a == 11
+        assert p.b == p.c == p.d == 22
+        assert p.e == 33
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [11, 22, 33, 44])
+        FOO = ffi.typeof("struct foo_s")
+        fields = [(name, fld.offset, fld.flags) for (name, fld) in FOO.fields]
+        assert fields == [
+            ('a', 0 * SIZE_OF_INT, 0),
+            ('b', 1 * SIZE_OF_INT, 0),
+            ('c', 1 * SIZE_OF_INT, 1),
+            ('d', 1 * SIZE_OF_INT, 1),
+            ('e', 2 * SIZE_OF_INT, 0),
+        ]
 
     def test_cast_to_array_type(self):
         ffi = FFI(backend=self.Backend())

From pypy.commits at gmail.com  Mon Sep  5 22:08:24 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 05 Sep 2016 19:08:24 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Revert change to
 run_formatter() and fix unicode vs str issues.
Message-ID: <57ce2518.a710c20a.5f195.890b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86899:5dc79f574fb9
Date: 2016-09-06 02:20 +0100
http://bitbucket.org/pypy/pypy/changeset/5dc79f574fb9/

Log:	Revert change to run_formatter() and fix unicode vs str issues.

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -564,7 +564,7 @@
             return space.wrap(self._pad(string))
 
         def _get_locale(self, tp):
-            if tp == "n":
+            if tp == _lit("n"):
                 dec, thousands, grouping = rlocale.numeric_formatting()
             elif self._thousands_sep:
                 dec = "."
@@ -608,10 +608,10 @@
             spec.sign = "\0"
             spec.n_sign = 0
             sign = self._sign
-            if sign == "+":
+            if sign == _lit("+"):
                 spec.n_sign = 1
                 spec.sign = "-" if sign_char == "-" else "+"
-            elif sign == " ":
+            elif sign == _lit(" "):
                 spec.n_sign = 1
                 spec.sign = "-" if sign_char == "-" else " "
             elif sign_char == "-":
@@ -619,7 +619,7 @@
                 spec.sign = "-"
             extra_length = (spec.n_sign + spec.n_prefix + spec.n_decimal +
                             spec.n_remainder) # Not padding or digits
-            if self._fill_char == "0" and self._align == "=":
+            if self._fill_char ==_lit( "0") and self._align == _lit("="):
                 spec.n_min_width = self._width - extra_length
             if self._loc_thousands:
                 self._group_digits(spec, digits[to_number:])
@@ -629,14 +629,14 @@
             n_padding = self._width - (extra_length + n_grouped_digits)
             if n_padding > 0:
                 align = self._align
-                if align == "<":
+                if align == _lit("<"):
                     spec.n_rpadding = n_padding
-                elif align == ">":
+                elif align == _lit(">"):
                     spec.n_lpadding = n_padding
-                elif align == "^":
+                elif align == _lit("^"):
                     spec.n_lpadding = n_padding // 2
                     spec.n_rpadding = n_padding - spec.n_lpadding
-                elif align == "=":
+                elif align == _lit("="):
                     spec.n_spadding = n_padding
                 else:
                     raise AssertionError("shouldn't reach")
@@ -653,7 +653,7 @@
             for i in range(d_state - 1, d_state - n_chars - 1, -1):
                 buf.append(digits[i])
             for i in range(n_zeros):
-                buf.append("0")
+                buf.append(_lit("0"))
 
         def _group_digits(self, spec, digits):
             buf = []
@@ -702,7 +702,7 @@
             for c in s:
                 index = ord(c)
                 if ord("a") <= index <= ord("z"):
-                    c = chr(index - 32)
+                    c = _lit(chr(index - 32))
                 buf.append(c)
             return self.empty.join(buf)
 
@@ -791,7 +791,7 @@
                     result = self._long_to_base(base, space.bigint_w(w_num))
                 n_prefix = skip_leading if self._alternate else 0
                 to_prefix = 0
-                if result[0] == "-":
+                if result[0] == _lit("-"):
                     sign_char = "-"
                     skip_leading += 1
                     to_prefix += 1
@@ -874,21 +874,21 @@
                     return space.call_function(space.w_unicode, w_num)
                 return self.space.str(w_num)
             tp = self._type
-            if (tp == "b" or
-                tp == "c" or
-                tp == "d" or
-                tp == "o" or
-                tp == "x" or
-                tp == "X" or
-                tp == "n"):
+            if (tp == _lit("b") or
+                tp == _lit("c") or
+                tp == _lit("d") or
+                tp == _lit("o") or
+                tp == _lit("x") or
+                tp == _lit("X") or
+                tp == _lit("n")):
                 return self._format_int_or_long(w_num, kind)
-            elif (tp == "e" or
-                  tp == "E" or
-                  tp == "f" or
-                  tp == "F" or
-                  tp == "g" or
-                  tp == "G" or
-                  tp == "%"):
+            elif (tp == _lit("e") or
+                  tp == _lit("E") or
+                  tp == _lit("f") or
+                  tp == _lit("F") or
+                  tp == _lit("g") or
+                  tp == _lit("G") or
+                  tp == _lit("%")):
                 w_float = space.float(w_num)
                 return self._format_float(w_float)
             else:
@@ -932,7 +932,7 @@
                 add_pct = False
             if self._precision == -1:
                 self._precision = default_precision
-            result, special = rfloat.double_to_string(value, tp,
+            result, special = rfloat.double_to_string(value, str(tp)[0],
                                                       self._precision, flags)
             if add_pct:
                 result += "%"
@@ -963,15 +963,15 @@
                     return space.call_function(space.w_unicode, w_float)
                 return space.str(w_float)
             tp = self._type
-            if (tp == "\0" or
-                tp == "e" or
-                tp == "E" or
-                tp == "f" or
-                tp == "F" or
-                tp == "g" or
-                tp == "G" or
-                tp == "n" or
-                tp == "%"):
+            if (tp == _lit("\0") or
+                tp == _lit("e") or
+                tp == _lit("E") or
+                tp == _lit("f") or
+                tp == _lit("F") or
+                tp == _lit("g") or
+                tp == _lit("G") or
+                tp == _lit("n") or
+                tp == _lit("%")):
                 return self._format_float(w_float)
             self._unknown_presentation("float")
 
@@ -980,12 +980,12 @@
             tp = self._type
             self._get_locale(tp)
             default_precision = 6
-            if self._align == "=":
+            if self._align == _lit("="):
                 # '=' alignment is invalid
                 raise oefmt(space.w_ValueError,
                             "'=' alignment flag is not allowed in complex "
                             "format specifier")
-            if self._fill_char == "0":
+            if self._fill_char == _lit("0"):
                 # zero padding is invalid
                 raise oefmt(space.w_ValueError,
                             "Zero padding is not allowed in complex format "
@@ -1018,8 +1018,8 @@
 
             #might want to switch to double_to_string from formatd
             #in CPython it's named 're' - clashes with re module
-            re_num = formatd(w_complex.realval, tp, self._precision)
-            im_num = formatd(w_complex.imagval, tp, self._precision)
+            re_num = formatd(w_complex.realval, str(tp)[0], self._precision)
+            im_num = formatd(w_complex.imagval, str(tp)[0], self._precision)
             n_re_digits = len(re_num)
             n_im_digits = len(im_num)
 
@@ -1042,8 +1042,8 @@
             tmp_fill_char = self._fill_char
             tmp_align = self._align
             tmp_width = self._width
-            self._fill_char = "\0"
-            self._align = "<"
+            self._fill_char = _lit("\0")
+            self._align = _lit("<")
             self._width = -1
 
             #determine if we have remainder, might include dec or exponent or both
@@ -1125,14 +1125,14 @@
             if self._parse_spec(_lit("\0"), _lit(">")):
                 return space.str(w_complex)
             tp = self._type
-            if (tp == "\0" or
-                tp == "e" or
-                tp == "E" or
-                tp == "f" or
-                tp == "F" or
-                tp == "g" or
-                tp == "G" or
-                tp == "n"):
+            if (tp == _lit("\0") or
+                tp == _lit("e") or
+                tp == _lit("E") or
+                tp == _lit("f") or
+                tp == _lit("F") or
+                tp == _lit("g") or
+                tp == _lit("G") or
+                tp == _lit("n")):
                 return self._format_complex(w_complex)
             self._unknown_presentation("complex")
     return Formatter
@@ -1143,5 +1143,9 @@
 
 @specialize.arg(2)
 def run_formatter(space, w_format_spec, meth, *args):
-    formatter = str_formatter(space, space.str_w(w_format_spec))
-    return getattr(formatter, meth)(*args)
+    if space.isinstance_w(w_format_spec, space.w_unicode):
+        formatter = unicode_formatter(space, space.unicode_w(w_format_spec))
+        return getattr(formatter, meth)(*args)
+    else:
+        formatter = str_formatter(space, space.str_w(w_format_spec))
+        return getattr(formatter, meth)(*args)

From pypy.commits at gmail.com  Tue Sep  6 04:17:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Sep 2016 01:17:46 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update and document v1.8.2
Message-ID: <57ce7baa.898b1c0a.2cfd1.8161@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2762:48bb1b21620f
Date: 2016-09-06 09:49 +0200
http://bitbucket.org/cffi/cffi/changeset/48bb1b21620f/

Log:	Update and document v1.8.2

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.8.1"
+#define CFFI_VERSION  "1.8.2"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.1"
-__version_info__ = (1, 8, 1)
+__version__ = "1.8.2"
+__version_info__ = (1, 8, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.1"
+                               "\ncompiled with cffi version: 1.8.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.8'
 # The full version, including alpha/beta/rc tags.
-release = '1.8.1'
+release = '1.8.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,7 +51,7 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.1.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.2.tar.gz
 
    - MD5: ...
 
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,15 @@
 ======================
 
 
+v1.8.2
+======
+
+* Issue #283: fixed ``ffi.new()`` on structures/unions with nested
+  anonymous structures/unions, when there is at least one union in
+  the mix.  When initialized with a list or a dict, it should now
+  behave more closely like the ``{ }`` syntax does in GCC.
+
+
 v1.8.1
 ======
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.8.1',
+        version='1.8.2',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Tue Sep  6 04:17:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Sep 2016 01:17:48 -0700 (PDT)
Subject: [pypy-commit] cffi default: skip test
Message-ID: <57ce7bac.45c8c20a.4d79b.ecc7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2763:29ac108030f8
Date: 2016-09-06 09:57 +0200
http://bitbucket.org/cffi/cffi/changeset/29ac108030f8/

Log:	skip test

diff --git a/testing/cffi0/test_ctypes.py b/testing/cffi0/test_ctypes.py
--- a/testing/cffi0/test_ctypes.py
+++ b/testing/cffi0/test_ctypes.py
@@ -34,6 +34,9 @@
     def test_nested_anonymous_union(self):
         py.test.skip("ctypes backend: not supported: nested anonymous union")
 
+    def test_nested_anonymous_struct_2(self):
+        py.test.skip("ctypes backend: not supported: nested anonymous union")
+
     def test_CData_CType_2(self):
         if sys.version_info >= (3,):
             py.test.skip("ctypes backend: not supported in Python 3: CType")

From pypy.commits at gmail.com  Tue Sep  6 04:17:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Sep 2016 01:17:50 -0700 (PDT)
Subject: [pypy-commit] cffi default: A direct test
Message-ID: <57ce7bae.6740c20a.d88a6.91fb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2764:b518a1326d6c
Date: 2016-09-06 10:03 +0200
http://bitbucket.org/cffi/cffi/changeset/b518a1326d6c/

Log:	A direct test

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2536,6 +2536,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if

From pypy.commits at gmail.com  Tue Sep  6 04:22:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Sep 2016 01:22:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/b518a1326d6c
Message-ID: <57ce7cd8.05d71c0a.db689.843c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86900:39d096586b61
Date: 2016-09-06 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/39d096586b61/

Log:	update to cffi/b518a1326d6c

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.1
+Version: 1.8.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.1"
-__version_info__ = (1, 8, 1)
+__version__ = "1.8.2"
+__version_info__ = (1, 8, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.1"
+                               "\ncompiled with cffi version: 1.8.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.1"
+VERSION = "1.8.2"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
                 return True
         return False
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        pass
-
     def convert_from_object(self, cdata, w_ob):
         if not self._copy_from_same(cdata, w_ob):
             self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
     )
     def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self.force_lazy_struct()
-        self._check_only_one_argument_for_union(w_ob)
 
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
-            if len(lst_w) > len(self._fields_list):
-                raise oefmt(space.w_ValueError,
-                            "too many initializers for '%s' (got %d)",
-                            self.name, len(lst_w))
-            for i in range(len(lst_w)):
-                optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+            j = 0
+            for w_obj in lst_w:
+                try:
+                    while (self._fields_list[j].flags &
+                               W_CField.BF_IGNORE_IN_CTOR):
+                        j += 1
+                except IndexError:
+                    raise oefmt(space.w_ValueError,
+                                "too many initializers for '%s' (got %d)",
+                                self.name, len(lst_w))
+                optvarsize = self._fields_list[j].write_v(cdata, w_obj,
                                                           optvarsize)
+                j += 1
             return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
 class W_CTypeUnion(W_CTypeStructOrUnion):
     kind = "union"
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        space = self.space
-        n = space.int_w(space.len(w_ob))
-        if n > 1:
-            raise oefmt(space.w_ValueError,
-                        "initializer for '%s': %d items given, but only one "
-                        "supported (use a dict if needed)", self.name, n)
-
 
 class W_CField(W_Root):
     _immutable_ = True
@@ -200,18 +194,21 @@
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
 
-    def __init__(self, ctype, offset, bitshift, bitsize):
+    BF_IGNORE_IN_CTOR = 0x01
+
+    def __init__(self, ctype, offset, bitshift, bitsize, flags):
         self.ctype = ctype
         self.offset = offset
         self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
         self.bitsize = bitsize
+        self.flags = flags       # BF_xxx
 
     def is_bitfield(self):
         return self.bitshift >= 0
 
-    def make_shifted(self, offset):
+    def make_shifted(self, offset, fflags):
         return W_CField(self.ctype, offset + self.offset,
-                        self.bitshift, self.bitsize)
+                        self.bitshift, self.bitsize, self.flags | fflags)
 
     def read(self, cdata):
         cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
     offset = interp_attrproperty('offset', W_CField),
     bitshift = interp_attrproperty('bitshift', W_CField),
     bitsize = interp_attrproperty('bitsize', W_CField),
+    flags = interp_attrproperty('flags', W_CField),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
         if alignment < falign and do_align:
             alignment = falign
         #
+        if is_union and i > 0:
+            fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+        else:
+            fflags = 0
+        #
         if fbitsize < 0:
             # not a bitfield: common case
 
@@ -372,7 +377,7 @@
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8)
+                    fld = srcfld.make_shifted(boffset // 8, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                                           fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
@@ -489,7 +495,7 @@
                     bitshift = 8 * ftype.size - fbitsize- bitshift
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
-                                           bitshift, fbitsize)
+                                           bitshift, fbitsize, fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1415,6 +1415,7 @@
         assert p.b == 12
         assert p.c == 14
         assert p.d == 14
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [0, 0, 0, 0])
 
     def test_nested_field_offset_align(self):
         ffi = FFI(backend=self.Backend())
@@ -1454,14 +1455,42 @@
         assert p.b == 0
         assert p.c == 14
         assert p.d == 14
-        p = ffi.new("union foo_u *", {'b': 12})
-        assert p.a == 0
+        p = ffi.new("union foo_u *", {'a': -63, 'b': 12})
+        assert p.a == -63
         assert p.b == 12
-        assert p.c == 0
-        assert p.d == 0
-        # we cannot specify several items in the dict, even though
-        # in theory in this particular case it would make sense
-        # to give both 'a' and 'b'
+        assert p.c == -63
+        assert p.d == -63
+        p = ffi.new("union foo_u *", [123, 456])
+        assert p.a == 123
+        assert p.b == 456
+        assert p.c == 123
+        assert p.d == 123
+        py.test.raises(ValueError, ffi.new, "union foo_u *", [0, 0, 0])
+
+    def test_nested_anonymous_struct_2(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            struct foo_s {
+                int a;
+                union { int b; union { int c, d; }; };
+                int e;
+            };
+        """)
+        assert ffi.sizeof("struct foo_s") == 3 * SIZE_OF_INT
+        p = ffi.new("struct foo_s *", [11, 22, 33])
+        assert p.a == 11
+        assert p.b == p.c == p.d == 22
+        assert p.e == 33
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [11, 22, 33, 44])
+        FOO = ffi.typeof("struct foo_s")
+        fields = [(name, fld.offset, fld.flags) for (name, fld) in FOO.fields]
+        assert fields == [
+            ('a', 0 * SIZE_OF_INT, 0),
+            ('b', 1 * SIZE_OF_INT, 0),
+            ('c', 1 * SIZE_OF_INT, 1),
+            ('d', 1 * SIZE_OF_INT, 1),
+            ('e', 2 * SIZE_OF_INT, 0),
+        ]
 
     def test_cast_to_array_type(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
@@ -35,6 +35,9 @@
     def test_nested_anonymous_union(self):
         py.test.skip("ctypes backend: not supported: nested anonymous union")
 
+    def test_nested_anonymous_struct_2(self):
+        py.test.skip("ctypes backend: not supported: nested anonymous union")
+
     def test_CData_CType_2(self):
         if sys.version_info >= (3,):
             py.test.skip("ctypes backend: not supported in Python 3: CType")

From pypy.commits at gmail.com  Tue Sep  6 04:27:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Sep 2016 01:27:46 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.8: Release branch,
 starting at 1.8.2 for CPython
Message-ID: <57ce7e02.c3f0c20a.54624.ed90@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.8
Changeset: r2765:f745eccd83e3
Date: 2016-09-06 10:21 +0200
http://bitbucket.org/cffi/cffi/changeset/f745eccd83e3/

Log:	Release branch, starting at 1.8.2 for CPython


From pypy.commits at gmail.com  Tue Sep  6 04:27:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Sep 2016 01:27:47 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.8: md5/sha
Message-ID: <57ce7e03.436ec20a.434cf.df7b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.8
Changeset: r2766:ac64324fb262
Date: 2016-09-06 10:26 +0200
http://bitbucket.org/cffi/cffi/changeset/ac64324fb262/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.8.2.tar.gz
 
-   - MD5: ...
+   - MD5: 538f307b6c5169bba41fbfda2b070762
 
-   - SHA: ...
+   - SHA: 9d2722ba9241b232b980bb9243e12451513a8000
 
-   - SHA256: ...
+   - SHA256: 2b636db1a179439d73ae0a090479e179a43df5d4eddc7e4c4067f960d4038530
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Tue Sep  6 04:27:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Sep 2016 01:27:49 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge release-1.8
Message-ID: <57ce7e05.e97ac20a.1eabd.eb2b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2767:477e1350143e
Date: 2016-09-06 10:26 +0200
http://bitbucket.org/cffi/cffi/changeset/477e1350143e/

Log:	hg merge release-1.8

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.8.2.tar.gz
 
-   - MD5: ...
+   - MD5: 538f307b6c5169bba41fbfda2b070762
 
-   - SHA: ...
+   - SHA: 9d2722ba9241b232b980bb9243e12451513a8000
 
-   - SHA256: ...
+   - SHA256: 2b636db1a179439d73ae0a090479e179a43df5d4eddc7e4c4067f960d4038530
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Tue Sep  6 06:59:08 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 03:59:08 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: test,
 fix issue #2395 - do not force pyobj
Message-ID: <57cea17c.c310c20a.ee2ea.2b92@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86902:0e2d9a73f5a1
Date: 2016-09-06 13:46 +0300
http://bitbucket.org/pypy/pypy/changeset/0e2d9a73f5a1/

Log:	test, fix issue #2395 - do not force pyobj

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -156,9 +156,6 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not pyobj_has_w_obj(ref):
-        # XXX Force the ref?
-        bytes_realize(space, ref)
     return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -183,8 +183,27 @@
                  Py_INCREF(Py_None);
                  return Py_None;
              """),
+            ("c_only", "METH_NOARGS",
+            """
+                int ret;
+                char * buf2;
+                PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+                if (!obj)
+                    return NULL;
+                buf2 = PyBytes_AsString(obj);
+                if (!buf2)
+                    return NULL;
+                /* buf should not have been forced, issue #2395 */
+                ret = _PyBytes_Resize(&obj, 512);
+                if (ret < 0)
+                    return NULL;
+                 Py_DECREF(obj);
+                 Py_INCREF(Py_None);
+                 return Py_None;
+            """),
             ])
         module.getbytes()
+        module.c_only()
 
     def test_py_string_as_string_Unicode(self):
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Tue Sep  6 06:59:10 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 03:59:10 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Moved tag release-pypy2.7-v5.4.1 to
 changeset 0e2d9a73f5a1 (from changeset 050d84dd7899)
Message-ID: <57cea17e.94071c0a.8c4bb.c3d0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-5.x
Changeset: r86903:c95650101a99
Date: 2016-09-06 13:47 +0300
http://bitbucket.org/pypy/pypy/changeset/c95650101a99/

Log:	Moved tag release-pypy2.7-v5.4.1 to changeset 0e2d9a73f5a1 (from
	changeset 050d84dd7899)

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -30,3 +30,5 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1

From pypy.commits at gmail.com  Tue Sep  6 06:59:11 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 03:59:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: Moved tag release-pypy2.7-v5.4.1 to
 changeset 0e2d9a73f5a1 (from changeset 050d84dd7899)
Message-ID: <57cea17f.c19d1c0a.c520.cb9f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86904:6c7485fc8b69
Date: 2016-09-06 13:47 +0300
http://bitbucket.org/pypy/pypy/changeset/6c7485fc8b69/

Log:	Moved tag release-pypy2.7-v5.4.1 to changeset 0e2d9a73f5a1 (from
	changeset 050d84dd7899)

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -31,3 +31,5 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1

From pypy.commits at gmail.com  Tue Sep  6 06:59:06 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 03:59:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: test,
 fix issue #2395 - do not force pyobj
Message-ID: <57cea17a.a710c20a.5f195.3803@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86901:9eb6ddd96e0e
Date: 2016-09-06 13:46 +0300
http://bitbucket.org/pypy/pypy/changeset/9eb6ddd96e0e/

Log:	test, fix issue #2395 - do not force pyobj

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -156,9 +156,6 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not pyobj_has_w_obj(ref):
-        # XXX Force the ref?
-        bytes_realize(space, ref)
     return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -183,8 +183,27 @@
                  Py_INCREF(Py_None);
                  return Py_None;
              """),
+            ("c_only", "METH_NOARGS",
+            """
+                int ret;
+                char * buf2;
+                PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+                if (!obj)
+                    return NULL;
+                buf2 = PyBytes_AsString(obj);
+                if (!buf2)
+                    return NULL;
+                /* buf should not have been forced, issue #2395 */
+                ret = _PyBytes_Resize(&obj, 512);
+                if (ret < 0)
+                    return NULL;
+                 Py_DECREF(obj);
+                 Py_INCREF(Py_None);
+                 return Py_None;
+            """),
             ])
         module.getbytes()
+        module.c_only()
 
     def test_py_string_as_string_Unicode(self):
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Tue Sep  6 06:59:13 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 03:59:13 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface: close branch to be merged
Message-ID: <57cea181.436ec20a.434cf.22c5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface
Changeset: r86905:b33e2ca4e535
Date: 2016-09-06 13:50 +0300
http://bitbucket.org/pypy/pypy/changeset/b33e2ca4e535/

Log:	close branch to be merged


From pypy.commits at gmail.com  Tue Sep  6 06:59:16 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 03:59:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge buffer-interface,
 which implements buffer-protocol parts of cpyext and numpypy
Message-ID: <57cea184.861b1c0a.6b21.c1a0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86906:03fc4cb79e37
Date: 2016-09-06 13:52 +0300
http://bitbucket.org/pypy/pypy/changeset/03fc4cb79e37/

Log:	merge buffer-interface, which implements buffer-protocol parts of
	cpyext and numpypy

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1428,6 +1428,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -597,6 +597,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -122,7 +122,7 @@
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,6 +645,9 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.UCHAR),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,8 +1,9 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
-from pypy.module.cpyext.pyobject import PyObject
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -33,13 +34,82 @@
     raise an error if the object can't support a simpler view of its memory.
 
     0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    view.c_len = buf.getlength()
+    view.c_obj = make_ref(space, w_obj)
+    ndim = buf.getndim()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_readonly', int(buf.readonly))
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.str2charp(buf.getformat())
+    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+        
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
+def PyBuffer_IsContiguous(space, view, fort):
     """Return 1 if the memory defined by the view is C-style (fortran is
     'C') or Fortran-style (fortran is 'F') contiguous or either one
     (fortran is 'A').  Return 0 otherwise."""
-    # PyPy only supports contiguous Py_buffers for now.
-    return 1
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
+    
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -29,19 +29,17 @@
 ## Solution
 ## --------
 ##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
+## PyBytesObject contains two additional members: the ob_size and an array
+## char ob_sval which holds a \x0 terminated string.
 ##
 ## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   NULL buffer.  The first time PyString_AsString() is called, memory is
-##   allocated (with flavor='raw') and content is copied.
+##   buffer holding \x0.  The first time PyString_AsString() is called, the 
+##   PyStringObject is reallocated, and the string copied into the buffer. The
+##   ob_size reflects the length of the string.
 ##
 ## - A string allocated with PyString_FromStringAndSize(NULL, size) will
 ##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, but the reference won't be stored in the global map; there is no
-##   corresponding object in pypy.  When from_ref() or Py_INCREF() is called,
-##   the pypy string is created, and added to the global map of tracked
-##   objects.  The buffer is then supposed to be immutable.
+##   size+1, as part of the object. The buffer is then supposed to be immutable.
 ##
 ##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
 ##   there is no corresponding pypy object for the string
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -142,7 +142,8 @@
 typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
 typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -156,12 +157,14 @@
     char *format;
     Py_ssize_t *shape;
     Py_ssize_t *strides;
-    Py_ssize_t *suboffsets;
-
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format;
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
        mono-dimensional buffers. */
     /* Py_ssize_t smalltable[2]; */
-    void *internal;
+    void *internal; /* always NULL for app-level objects */
 } Py_buffer;
 
 
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,7 +1,8 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
@@ -12,6 +13,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -20,21 +22,35 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-    # TODO - fill in fields
-    '''
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if not isinstance(w_obj, W_MemoryView):
+        return view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_obj)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        isstr = False
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        rffi.setintfield(view, 'c_readonly', 1)
+        isstr = True
+    view.c_len = w_obj.getlength()
+    view.c_itemsize = w_obj.buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    shape = w_obj.buf.getshape()
+    strides = w_obj.buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    ''' 
     return view
 
-
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -508,10 +508,9 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Releases a Py_buffer obtained from getbuffer ParseTuple's s*.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
+    Release the buffer view. This should be called when the buffer is 
+    no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
     view.c_obj = lltype.nullptr(PyObject.TO)
+    # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
     def getshape(self):
         return self.shape
 
+    def getstrides(self):
+        return self.strides
+
     def getitemsize(self):
         return self.itemsize
 
+    def getndim(self):
+        return self.ndim
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
   PyMyArray* self = (PyMyArray*)obj;
-  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
-    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
-    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
@@ -188,7 +185,131 @@
     (initproc)PyMyArray_init,     /* tp_init */
 };
 
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+    Py_buffer* view = NULL;
+    PyObject* obj = PyTuple_GetItem(args, 0);
+    PyObject* memoryview = PyMemoryView_FromObject(obj);
+    if (memoryview == NULL)
+        return PyInt_FromLong(-1);
+    view = PyMemoryView_GET_BUFFER(memoryview);
+    Py_DECREF(memoryview);
+    return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG)                                        \
+    buf_flag = PyUnicode_FromString(#FLAG);                         \
+    flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ);  \
+    Py_DECREF(buf_flag);                                            \
+    if (flag_matches == 1) {                                        \
+        Py_DECREF(tmp);                                             \
+        flags |= PyBUF_##FLAG;                                      \
+        continue;                                                   \
+    }                                                               \
+    else if (flag_matches == -1) {                                  \
+        Py_DECREF(tmp);                                             \
+        return NULL;                                                \
+    }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+    PyObject *buffer_obj, *pyflags;
+    PyObject *tmp, *buf_flag;
+    Py_buffer buffer;
+    PyObject *shape, *strides;
+    Py_ssize_t i, n;
+    int flag_matches;
+    int flags = 0;
+
+    if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+        return NULL;
+    }
+
+    n = PySequence_Length(pyflags);
+    if (n < 0) {
+        return NULL;
+    }
+
+    for (i=0; i < n; i++) {
+        tmp = PySequence_GetItem(pyflags, i);
+        if (tmp == NULL) {
+            return NULL;
+        }
+
+        GET_PYBUF_FLAG(SIMPLE);
+        GET_PYBUF_FLAG(WRITABLE);
+        GET_PYBUF_FLAG(STRIDES);
+        GET_PYBUF_FLAG(ND);
+        GET_PYBUF_FLAG(C_CONTIGUOUS);
+        GET_PYBUF_FLAG(F_CONTIGUOUS);
+        GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+        GET_PYBUF_FLAG(INDIRECT);
+        GET_PYBUF_FLAG(FORMAT);
+        GET_PYBUF_FLAG(STRIDED);
+        GET_PYBUF_FLAG(STRIDED_RO);
+        GET_PYBUF_FLAG(RECORDS);
+        GET_PYBUF_FLAG(RECORDS_RO);
+        GET_PYBUF_FLAG(FULL);
+        GET_PYBUF_FLAG(FULL_RO);
+        GET_PYBUF_FLAG(CONTIG);
+        GET_PYBUF_FLAG(CONTIG_RO);
+
+        Py_DECREF(tmp);
+
+        /* One of the flags must match */
+        PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+        return NULL;
+    }
+
+    if (buffer.shape == NULL) {
+        Py_INCREF(Py_None);
+        shape = Py_None;
+    }
+    else {
+        shape = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+        }
+    }
+
+    if (buffer.strides == NULL) {
+        Py_INCREF(Py_None);
+        strides = Py_None;
+    }
+    else {
+        strides = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+        }
+    }
+
+    PyBuffer_Release(&buffer);
+    return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
 static PyMethodDef buffer_functions[] = {
+    {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
+    {"get_buffer_info",   (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
@@ -12,6 +12,12 @@
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
+    def test_frombuffer(self, space, api):
+        w_buf = space.newbuffer(StringBuffer("hello"))
+        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        ndim = w_view.c_ndim
+        assert ndim == 1
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):
@@ -21,6 +27,25 @@
         y = memoryview(arr)
         assert y.format == 'i'
         assert y.shape == (10,)
+        assert len(y) == 10
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
+        viewlen = module.test_buffer(arr)
+        assert viewlen == y.itemsize * len(y)
+
+    def test_buffer_info(self):
+        from _numpypy import multiarray as np
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        # test_export_flags from numpy test_multiarray
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        # test_relaxed_strides from numpy test_multiarray
+        arr = np.zeros((1, 10))
+        if arr.flags.f_contiguous:
+            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+            assert strides[0] == 8
+            arr = np.ones((10, 1), order='F')
+            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+            assert strides[-1] == 8
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
                     STRUCT_TYPE = PySequenceMethods
+                elif slot_names[0] == 'c_tp_as_buffer':
+                    STRUCT_TYPE = PyBufferProcs
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
     def getdictvalue(self, space, key):
         return self.items[key]
 
+    def descr_memoryview(self, space, buf):
+        raise oefmt(space.w_TypeError, "error")
+
 class IterDictObject(W_Root):
     def __init__(self, space, w_dict):
         self.space = space
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -377,7 +377,25 @@
     def __exit__(self, typ, value, traceback):
         keepalive_until_here(self)
 
-    def get_buffer(self, space, readonly):
+    def get_buffer(self, space, flags):
+        errtype = space.w_ValueError # should be BufferError, numpy does this instead
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_F_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not Fortran contiguous")
+        if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                     self.flags & NPY.ARRAY_C_CONTIGUOUS)):
+           raise oefmt(errtype, "ndarray is not contiguous")
+        if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and
+            not self.flags & NPY.ARRAY_WRITEABLE):
+           raise oefmt(errtype, "buffer source array is read-only")
+        readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
         return ArrayBuffer(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
@@ -695,6 +713,7 @@
                  index + self.impl.start)
 
     def setitem(self, index, v):
+        # XXX what if self.readonly?
         raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstring import strip_spaces
@@ -42,7 +43,7 @@
         raise oefmt(space.w_ValueError,
                     "object __array__ method not producing an array")
 
-def try_interface_method(space, w_object):
+def try_interface_method(space, w_object, copy):
     try:
         w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
         if w_interface is None:
@@ -81,17 +82,20 @@
             raise oefmt(space.w_ValueError,
                     "__array_interface__ could not decode dtype %R", w_dtype
                     )
-        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
+        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
+                                   space.isinstance_w(w_data, space.w_list)):
             data_w = space.listview(w_data)
-            data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
-            read_only = True # XXX why not space.is_true(data_w[1])
+            w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+            read_only = space.is_true(data_w[1]) or copy
             offset = 0
-            return W_NDimArray.from_shape_and_storage(space, shape, data, 
-                                    dtype, strides=strides, start=offset), read_only
+            w_base = w_object
+            if read_only:
+                w_base = None
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+                                dtype, w_base=w_base, strides=strides,
+                                start=offset), read_only
         if w_data is None:
-            data = w_object
-        else:
-            data = w_data
+            w_data = w_object
         w_offset = space.finditem(w_interface, space.wrap('offset'))
         if w_offset is None:
             offset = 0
@@ -101,7 +105,7 @@
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
                    "__array_interface__ strides not fully supported yet") 
-        arr = frombuffer(space, data, dtype, support.product(shape), offset)
+        arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
         
@@ -110,6 +114,78 @@
             return None, False
         raise
 
+def _descriptor_from_pep3118_format(space, c_format):
+    descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+    if descr:
+        return descr
+    msg = "invalid PEP 3118 format string: '%s'" % c_format
+    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+    return None 
+
+def _array_from_buffer_3118(space, w_object, dtype):
+    try:
+        w_buf = space.call_method(space.builtin, "memoryview", w_object)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            # object does not have buffer interface
+            return w_object
+        raise
+    format = space.getattr(w_buf,space.newbytes('format'))
+    if format:
+        descr = _descriptor_from_pep3118_format(space, space.str_w(format))
+        if not descr:
+            return w_object
+        if dtype and descr:
+            raise oefmt(space.w_NotImplementedError,
+                "creating an array from a memoryview while specifying dtype "
+                "not supported")
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
+            msg = ("Item size computed from the PEP 3118 buffer format "
+                  "string does not match the actual item size.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+        dtype = descr 
+    elif not dtype:
+        dtype = descriptor.get_dtype_cache(space).w_stringdtype
+        dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
+    nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim')))
+    shape = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('shape')))]
+    strides = []
+    buflen = space.len_w(w_buf) * dtype.elsize
+    if shape:
+        strides = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('strides')))]
+        if not strides:
+            d = buflen
+            strides = [0] * nd
+            for k in range(nd):
+                if shape[k] > 0:
+                    d /= shape[k]
+                    strides[k] = d
+    else:
+        if nd == 1:
+            shape = [buflen / dtype.elsize, ]
+            strides = [dtype.elsize, ]
+        elif nd > 1:
+            msg = ("ndim computed from the PEP 3118 buffer format "
+                   "is greater than 1, but shape is NULL.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+    try:
+        w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
+    except OperationError as e:
+        if e.match(space, space.w_ValueError):
+            return w_object
+        else:
+            raise e
+    writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
+    w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
+               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
+               writable=writable, strides=strides)
+    if w_ret:
+        return w_ret
+    return w_object
 
 @unwrap_spec(ndmin=int, copy=bool, subok=bool)
 def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -127,6 +203,7 @@
 
 def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
 
+    from pypy.module.micronumpy.boxes import W_GenericBox
     # numpy testing calls array(type(array([]))) and expects a ValueError
     if space.isinstance_w(w_object, space.w_type):
         raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
@@ -134,13 +211,19 @@
     dtype = descriptor.decode_w_dtype(space, w_dtype)
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
-        if w_array is not None:
+        if w_array is None:
+            if (    not space.isinstance_w(w_object, space.w_str) and 
+                    not space.isinstance_w(w_object, space.w_unicode) and
+                    not isinstance(w_object, W_GenericBox)):
+                # use buffer interface
+                w_object = _array_from_buffer_3118(space, w_object, dtype)
+        else:
             # continue with w_array, but do further operations in place
             w_object = w_array
             copy = False
             dtype = w_object.get_dtype()
     if not isinstance(w_object, W_NDimArray):
-        w_array, _copy = try_interface_method(space, w_object)
+        w_array, _copy = try_interface_method(space, w_object, copy)
         if w_array is not None:
             w_object = w_array
             copy = _copy
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -805,19 +805,19 @@
         return w_result
 
     def buffer_w(self, space, flags):
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, flags)
 
     def readbuf_w(self, space):
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO)
 
     def writebuf_w(self, space):
-        return self.implementation.get_buffer(space, False)
+        return self.implementation.get_buffer(space, space.BUF_FULL)
 
     def charbuf_w(self, space):
-        return self.implementation.get_buffer(space, True).as_str()
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
 
     def descr_get_data(self, space):
-        return space.newbuffer(self.implementation.get_buffer(space, False))
+        return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3215,7 +3215,9 @@
         raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)}))
 
         a = array([1, 2, 3])
-        b = array(Dummy(a.__array_interface__))
+        d = Dummy(a.__array_interface__)
+        b = array(d)
+        assert b.base is None
         b[1] = 200
         assert a[1] == 2 # upstream compatibility, is this a bug?
         interface_a = a.__array_interface__
@@ -3226,6 +3228,8 @@
         interface_b.pop('data')
         interface_a.pop('data')
         assert interface_a == interface_b
+        b = array(d, copy=False)
+        assert b.base is d
 
         b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
                          'data': 'a'*100}))
@@ -3594,6 +3598,7 @@
         cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
         cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
         cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+        cls.w_one = cls.space.wrap(struct.pack('i', 1))
 
     def test_frombuffer(self):
         import numpy as np
@@ -3645,8 +3650,6 @@
         else:
             EMPTY = None
         x = np.array([1, 2, 3, 4, 5], dtype='i')
-        y = memoryview('abc')
-        assert y.format == 'B'
         y = memoryview(x)
         assert y.format == 'i'
         assert y.shape == (5,)
@@ -3654,6 +3657,16 @@
         assert y.strides == (4,)
         assert y.suboffsets == EMPTY
         assert y.itemsize == 4
+        assert isinstance(y, memoryview)
+        assert y[0] == self.one
+        assert (np.array(y) == x).all()
+
+        x = np.array([0, 0, 0, 0], dtype='O')
+        y = memoryview(x)
+        # handles conversion of address to pinned object?
+        z = np.array(y)
+        assert z.dtype == 'O'
+        assert (z == x).all()
 
     def test_fromstring(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -702,3 +702,32 @@
         ret = obj.sum()
         print type(ret)
         assert ret.info == 'spam'
+
+    def test_ndarray_subclass_assigns_base(self):
+        import numpy as np
+        init_called = []
+        class _DummyArray(object):
+            """ Dummy object that just exists to hang __array_interface__ dictionaries
+            and possibly keep alive a reference to a base array.
+            """
+            def __init__(self, interface, base=None):
+                self.__array_interface__ = interface
+                init_called.append(1)
+                self.base = base
+
+        x = np.zeros(10)
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 1
+        assert y.base is d
+
+        x = np.zeros((0,), dtype='float32')
+        intf = x.__array_interface__.copy()
+        intf["strides"] = x.strides
+        x.__array_interface__["strides"] = x.strides
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 2
+        assert y.base is d
+
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1851,7 +1851,7 @@
                     arr.gcstruct)
 
     def read(self, arr, i, offset, dtype):
-        if arr.gcstruct is V_OBJECTSTORE:
+        if arr.gcstruct is V_OBJECTSTORE and not arr.base():
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot read object from array with no gc hook")
         return self.box(self._read(arr.storage, i, offset))
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -14,6 +14,7 @@
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
+    _attrs_ = ['buf']
 
     def __init__(self, buf):
         assert isinstance(buf, Buffer)
@@ -115,7 +116,7 @@
             self.buf.setslice(start, value.as_str())
 
     def descr_len(self, space):
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.buf.getlength() / self.buf.getitemsize())
 
     def w_get_format(self, space):
         return space.wrap(self.buf.getformat())

From pypy.commits at gmail.com  Tue Sep  6 06:59:17 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 03:59:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: document branches and update release
 5.4.1
Message-ID: <57cea185.449a1c0a.2f51c.d679@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86907:2ee81d5bf769
Date: 2016-09-06 13:57 +0300
http://bitbucket.org/pypy/pypy/changeset/2ee81d5bf769/

Log:	document branches and update release 5.4.1

diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
--- a/pypy/doc/release-pypy2.7-v5.4.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -5,8 +5,8 @@
 We have released a bugfix for PyPy2.7-v5.4.0, released last week,
 due to the following issues:
 
-  * Update list of contributors in documentation and LICENSE file, 
-    this was unfortunately left out of 5.4.0. My apoligies to the new
+  * Update list of contributors in documentation and LICENSE file,
+    this was unfortunately left out of 5.4.0. My apologies to the new
     contributors
 
   * Allow tests run with `-A` to find `libm.so` even if it is a script not a
@@ -21,10 +21,13 @@
   * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
     the last byte of a unicode string sometimes
 
-  * Update built-in cffi_ to the soon-to-be-released 1.8.1 version
+  * Update built-in cffi_ to the soon-to-be-released 1.8.2 version
 
   * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
-    translation with a message asking for help porting it 
+    translation with a message asking for help porting it
+
+  * Fix a regression where a PyBytesObject was forced (converted to a RPython
+    object) when not required, reported as issue #2395
 
 Thanks to those who reported the issues.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,9 @@
 
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+

From pypy.commits at gmail.com  Tue Sep  6 10:39:10 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 06 Sep 2016 07:39:10 -0700 (PDT)
Subject: [pypy-commit] pypy union-side-effects: Use more general strategy in
 2 tests
Message-ID: <57ced50e.c41f1c0a.59642.189a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects
Changeset: r86908:35a154a7faf2
Date: 2016-09-06 15:38 +0100
http://bitbucket.org/pypy/pypy/changeset/35a154a7faf2/

Log:	Use more general strategy in 2 tests

diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -170,12 +170,12 @@
     lambda st_ann: valid_unions(st_ann) | st.builds(SomeTuple, st.lists(st_ann)),
     max_leaves=3)
 
- at given(s=st_numeric)
+ at given(s=st_annotation)
 def test_union_unary(s):
     assert union(s, s) == s
     assert union(s_ImpossibleValue, s) == s
 
- at given(s1=st_numeric, s2=st_numeric)
+ at given(s1=st_annotation, s2=st_annotation)
 def test_commutativity_of_union_compatibility(s1, s2):
     assert compatible(s1, s2) == compatible(s2, s1)
 

From pypy.commits at gmail.com  Tue Sep  6 10:51:13 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 06 Sep 2016 07:51:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: improve the trace of import statements
 to not contain calls
Message-ID: <57ced7e1.94071c0a.8c4bb.234e@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86909:aab338e52409
Date: 2016-09-06 16:50 +0200
http://bitbucket.org/pypy/pypy/changeset/aab338e52409/

Log:	improve the trace of import statements to not contain calls

	(needed slightly awkward code)

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -259,10 +259,8 @@
         raise oefmt(space.w_ValueError, "Empty module name")
     w = space.wrap
 
-    if w_fromlist is not None and space.is_true(w_fromlist):
-        fromlist_w = space.fixedview(w_fromlist)
-    else:
-        fromlist_w = None
+    if w_fromlist is not None and not space.is_true(w_fromlist):
+        w_fromlist = None
 
     rel_modulename = None
     if (level != 0 and w_globals is not None and
@@ -284,19 +282,19 @@
                     w_mod = None
                 else:
                     w_mod = absolute_import(space, rel_modulename, rel_level,
-                                            fromlist_w, tentative=True)
+                                            w_fromlist, tentative=True)
             else:
                 w_mod = absolute_import(space, rel_modulename, rel_level,
-                                        fromlist_w, tentative=False)
+                                        w_fromlist, tentative=False)
             if w_mod is not None:
                 return w_mod
 
-    w_mod = absolute_import(space, modulename, 0, fromlist_w, tentative=0)
+    w_mod = absolute_import(space, modulename, 0, w_fromlist, tentative=0)
     if rel_modulename is not None:
         space.setitem(space.sys.get('modules'), w(rel_modulename), space.w_None)
     return w_mod
 
-def absolute_import(space, modulename, baselevel, fromlist_w, tentative):
+def absolute_import(space, modulename, baselevel, w_fromlist, tentative):
     # Short path: check in sys.modules, but only if there is no conflict
     # on the import lock.  In the situation of 'import' statements
     # inside tight loops, this should be true, and absolute_import_try()
@@ -304,25 +302,25 @@
     # if the import lock is currently held by another thread, then we
     # have to wait, and so shouldn't use the fast path.
     if not getimportlock(space).lock_held_by_someone_else():
-        w_mod = absolute_import_try(space, modulename, baselevel, fromlist_w)
+        w_mod = absolute_import_try(space, modulename, baselevel, w_fromlist)
         if w_mod is not None and not space.is_w(w_mod, space.w_None):
             return w_mod
     return absolute_import_with_lock(space, modulename, baselevel,
-                                     fromlist_w, tentative)
+                                     w_fromlist, tentative)
 
 @jit.dont_look_inside
 def absolute_import_with_lock(space, modulename, baselevel,
-                              fromlist_w, tentative):
+                              w_fromlist, tentative):
     lock = getimportlock(space)
     lock.acquire_lock()
     try:
         return _absolute_import(space, modulename, baselevel,
-                                fromlist_w, tentative)
+                                w_fromlist, tentative)
     finally:
         lock.release_lock(silent_after_fork=True)
 
 @jit.unroll_safe
-def absolute_import_try(space, modulename, baselevel, fromlist_w):
+def absolute_import_try(space, modulename, baselevel, w_fromlist):
     """ Only look up sys.modules, not actually try to load anything
     """
     w_path = None
@@ -330,7 +328,7 @@
     if '.' not in modulename:
         w_mod = check_sys_modules_w(space, modulename)
         first = w_mod
-        if fromlist_w is not None and w_mod is not None:
+        if w_fromlist is not None and w_mod is not None:
             w_path = try_getattr(space, w_mod, space.wrap('__path__'))
     else:
         level = 0
@@ -345,28 +343,36 @@
                 return None
             if level == baselevel:
                 first = w_mod
-            if fromlist_w is not None:
+            if w_fromlist is not None:
                 w_path = try_getattr(space, w_mod, space.wrap('__path__'))
             level += 1
-    if fromlist_w is not None:
+    if w_fromlist is not None:
+        # bit artificial code but important to not just unwrap w_fromlist
+        # to get a better trace. if it is unwrapped, the immutability of the
+        # tuple is lost
         if w_path is not None:
-            if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],
-                                                   space.wrap('*')):
+            length = space.len_w(w_fromlist)
+            if length == 1 and space.eq_w(
+                    space.getitem(w_fromlist, space.wrap(0)),
+                    space.wrap('*')):
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
-                    fromlist_w = space.fixedview(w_all)
+                    w_fromlist = w_all
                 else:
-                    fromlist_w = []
+                    w_fromlist = None
                     # "from x import *" with x already imported and no x.__all__
                     # always succeeds without doing more imports.  It will
                     # just copy everything from x.__dict__ as it is now.
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    return None
+
+            if w_fromlist is not None:
+                for i in range(length):
+                    w_name = space.getitem(w_fromlist, space.wrap(i))
+                    if try_getattr(space, w_mod, w_name) is None:
+                        return None
         return w_mod
     return first
 
-def _absolute_import(space, modulename, baselevel, fromlist_w, tentative):
+def _absolute_import(space, modulename, baselevel, w_fromlist, tentative):
     w = space.wrap
 
     if '/' in modulename or '\\' in modulename:
@@ -394,18 +400,23 @@
         w_path = try_getattr(space, w_mod, w('__path__'))
         level += 1
 
-    if fromlist_w is not None:
+    if w_fromlist is not None:
         if w_path is not None:
-            if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],w('*')):
+            length = space.len_w(w_fromlist)
+            if length == 1 and space.eq_w(
+                    space.getitem(w_fromlist, space.wrap(0)),
+                    space.wrap('*')):
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
-                    fromlist_w = space.fixedview(w_all)
+                    w_fromlist = w_all
                 else:
-                    fromlist_w = []
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    load_part(space, w_path, prefix, space.str0_w(w_name),
-                              w_mod, tentative=1)
+                    w_fromlist = None
+            if w_fromlist is not None:
+                for i in range(length):
+                    w_name = space.getitem(w_fromlist, space.wrap(i))
+                    if try_getattr(space, w_mod, w_name) is None:
+                        load_part(space, w_path, prefix, space.str0_w(w_name),
+                                  w_mod, tentative=1)
         return w_mod
     else:
         return first
diff --git a/pypy/module/pypyjit/test_pypy_c/test_import.py b/pypy/module/pypyjit/test_pypy_c/test_import.py
--- a/pypy/module/pypyjit/test_pypy_c/test_import.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_import.py
@@ -38,3 +38,27 @@
         # call_may_force(absolute_import_with_lock).
         for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")):
             assert 'call' not in opname    # no call-like opcode
+
+    def test_import_fast_path(self, tmpdir):
+        print tmpdir
+        pkg = tmpdir.join('mypkg').ensure(dir=True)
+        subdir = pkg.join("sub").ensure(dir=True)
+        pkg.join('__init__.py').write("")
+        subdir.join('__init__.py').write("")
+        subdir.join('mod.py').write(str(py.code.Source("""
+            def do_the_import():
+                import sys
+        """)))
+        def main(path, n):
+            def do_the_import():
+                from mypkg.sub import mod
+            import sys
+            sys.path.append(path)
+            for i in range(n):
+                do_the_import()
+        #
+        log = self.run(main, [str(tmpdir), 300])
+        loop, = log.loops_by_filename(self.filepath)
+        # check that no string compares and other calls are there
+        for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")):
+            assert 'call' not in opname    # no call-like opcode
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -64,6 +64,8 @@
 
     def setitem_str(self, w_dict, key, w_value):
         cell = self.getdictvalue_no_unwrapping(w_dict, key)
+        #if (key == '__package__' or key == "__path__") and cell is not None and w_value is not cell:
+        #    print "WARNING", key, w_value, cell, self
         return self._setitem_str_cell_known(cell, w_dict, key, w_value)
 
     def _setitem_str_cell_known(self, cell, w_dict, key, w_value):

From pypy.commits at gmail.com  Tue Sep  6 14:36:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Sep 2016 11:36:45 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the checksums
Message-ID: <57cf0cbd.436ec20a.434cf.da57@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r786:42216d0bec0f
Date: 2016-09-06 20:36 +0200
http://bitbucket.org/pypy/pypy.org/changeset/42216d0bec0f/

Log:	update the checksums

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -450,18 +450,18 @@
 
 pypy2.7-v5.4.1 md5::
 
-    70958c05af6628a66db5072ef1c72522  pypy2-v5.4.1-linux-armel.tar.bz2
-    5246b7b963689ec5b70291c2b104476a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-    106cfa49756df7ae3bf531ce6659d0ed  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-    f561bedb338fa09f011eaf1edac0faf1  pypy2-v5.4.1-linux32.tar.bz2
-    df1327fc3cd41a1ff860e90b5a901899  pypy2-v5.4.1-linux64.tar.bz2
-    b24ebe9f4825fc05afd76a9e8f47018c  pypy2-v5.4.1-osx64.tar.bz2
-    b8e90edf11639d1757413c8bf5a11d49  pypy2-v5.4.1-ppc64.tar.bz2
-    55f6747988d981699e52a7c60aed8a7f  pypy2-v5.4.1-ppc64le.tar.bz2
-    082cca9bb948c8b1389c35db7174396a  pypy2-v5.4.1-s390x.tar.bz2
-    129e730c84f55133b9694bc48e2d4812  pypy2-v5.4.1-src.tar.bz2
-    f20a420d90475b72c6ef8b4ab90377f0  pypy2-v5.4.1-src.zip
-    125874d61b4ac4e2fd7d0b7c2db3b041  pypy2-v5.4.1-win32.zip
+    425ffedf0db4dd737d450aa064ae0e7a  pypy2-v5.4.1-linux-armel.tar.bz2
+    15f41409cbadbde3ef22ee60ded8579a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+    5940ea0c1077e3bc2ba461bf55800abb  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+    23cbad540abff48ea67f5b75368344be  pypy2-v5.4.1-linux32.tar.bz2
+    2f4a82ae306f3d30d486fb945ca9c6be  pypy2-v5.4.1-linux64.tar.bz2
+    a18cbd990c97f75a08235a35e546a637  pypy2-v5.4.1-osx64.tar.bz2
+    90b0650dda1b5bf33f6119f20ba9c3b6  pypy2-v5.4.1-ppc64.tar.bz2
+    9a2c1b3124151b79e68f36eb2d4891d1  pypy2-v5.4.1-ppc64le.tar.bz2
+    826d4b48e43c84a50dc7e2adc2cb69d4  pypy2-v5.4.1-s390x.tar.bz2
+    d1d197d16331aa23a7fd4b5d4c3c1717  pypy2-v5.4.1-src.tar.bz2
+    1aab9fe6e7c03e959cde466819034bab  pypy2-v5.4.1-src.zip
+    b04aad943aac92862a73b1fd90157a00  pypy2-v5.4.1-win32.zip
 
 pypy3.3-v5.2-alpha md5::
 
@@ -483,18 +483,18 @@
 
 pypy2.7-5.4.1 sha1::
 
-    74fea5a7a0a3d8c899404ddc8c0296d4bf4ca3d0  pypy2-v5.4.1-linux-armel.tar.bz2
-    4c72d94325567d2079ca5021da3cba6cbc835744  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-    18486da20d2513c083be308f8222f83f80c74671  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-    c38c40dfcbe9cba712a508fed0ea3dc6da5e2b7c  pypy2-v5.4.1-linux32.tar.bz2
-    2c9a45c1bf67d8f2fac9d09e082f059ce892f291  pypy2-v5.4.1-linux64.tar.bz2
-    765bed9e45fa58a6f65ae20ff28b5e39beb56793  pypy2-v5.4.1-osx64.tar.bz2
-    6e73fa972ca01d58e95b36f133319bff0cc66876  pypy2-v5.4.1-ppc64.tar.bz2
-    4410c7514328d936084b00ac9d01af1aecdc7289  pypy2-v5.4.1-ppc64le.tar.bz2
-    8726412fcbaea859f21fb4b1c21fd5832e4c56d7  pypy2-v5.4.1-s390x.tar.bz2
-    0d865c16a3779f492b7f4687cd46c21bbfc05609  pypy2-v5.4.1-src.tar.bz2
-    8f898a052786d3b60e9effe162c15fa572a5f52d  pypy2-v5.4.1-src.zip
-    21958b782dc727a0be3bbc248e0ca9af18305654  pypy2-v5.4.1-win32.zip
+    a54b2a8b6def85663b10fd956d51fbd052954b83  pypy2-v5.4.1-linux-armel.tar.bz2
+    61c9a5269d6d414c4a1d7c41bbc6a45da318f138  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+    ea48ffe40887e25adcf1969a6b0e25dbe42a2457  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+    bf0ac4668323abead04dd0948a2e541a158e4a00  pypy2-v5.4.1-linux32.tar.bz2
+    d16601b04987381c4922a19dacb0ca2591a3b566  pypy2-v5.4.1-linux64.tar.bz2
+    4084b7db8ee16a52e7ecb89d10765d961864c67c  pypy2-v5.4.1-osx64.tar.bz2
+    98eac6f32f4fe8fce6eb0f0594163f3e01dccced  pypy2-v5.4.1-ppc64.tar.bz2
+    fc676af00b356ae48602b531675774f2badfc4fd  pypy2-v5.4.1-ppc64le.tar.bz2
+    70b736f1fdb92ae9916630b7cc8954ed92490f64  pypy2-v5.4.1-s390x.tar.bz2
+    1143948f50b1b95b55465b246639b4c48251e38e  pypy2-v5.4.1-src.tar.bz2
+    a6a9ce0defb401d8777d8af5949d23393416a390  pypy2-v5.4.1-src.zip
+    49c2ad75f1531730f7ee466d833d318333915ce0  pypy2-v5.4.1-win32.zip
 
 pypy3.3-v5.2-alpha sha1::
 
@@ -510,18 +510,18 @@
 
 pypy2.7-5.4.1 sha256::
 
-    8925b76fe9ca6f960d8f914ed67f7a3c52ce2b4c65fa71a5ef7d4b285c2c3a36  pypy2-v5.4.1-linux-armel.tar.bz2
-    0213b0d948ae0afea8b4cb93f08e55b0562522b3ab8f2706c4e22ffe8cd86f84  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-    2daee13ec1836c1041c89c18d9514134ff606dc3648fc6304611eb1ec0819289  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-    85bccf8679f908c08850115fe74325474fe1b2e1e793c147d1fa484b56472b12  pypy2-v5.4.1-linux32.tar.bz2
-    06c29d59565d9fdb618ed8aa730e05cf975da21158955591dff38d9e305af074  pypy2-v5.4.1-linux64.tar.bz2
-    507c81af9ca302c67a582255306529f88fba56760d353e17d667a114eee1f7e2  pypy2-v5.4.1-osx64.tar.bz2
-    c11b37f5e97b647003426987e223d75a0dc0da1ecc35675ddad0af8a9add972d  pypy2-v5.4.1-ppc64.tar.bz2
-    652d97fbd574d0349f7aa8b37c8c5a1238ed0cd3d6b68cf2ea8280b7ead4c7ad  pypy2-v5.4.1-ppc64le.tar.bz2
-    987b3354dcbed5fd3f0d8d9d1484a259f0dff97da5d11a84b354c6e61a4af891  pypy2-v5.4.1-s390x.tar.bz2
-    92af82664ace96d721c66dbe8726d4f39c7d01f568d9df56c11149be2960238f  pypy2-v5.4.1-src.tar.bz2
-    08148d1157dd16f402c7844fc0cdfde9e7d187c7fd1549a93e888e2fd13828bf  pypy2-v5.4.1-src.zip
-    b703224af4e99243d090783a7b685063da7ba01ef28bb99a89cacfce2fb0dfc2  pypy2-v5.4.1-win32.zip
+    a1eb5f672aae62606176305e52a51b060ba974b6181ebefcd2c555ecf5f8614f  pypy2-v5.4.1-linux-armel.tar.bz2
+    2c4befc4517adec874155a8b6fa0b9d18388943d4ffe778002072db7783e417a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+    b38646519ee1a888c68f8f4713c122867b4b36693c8acabb38eb827a9d2d51f9  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+    6d1e2386ec1e05dffed493aa2d5e6db5cf5de18d7350d44b85f2e45aa5c9a774  pypy2-v5.4.1-linux32.tar.bz2
+    9c85319778224d7fb0c348f55fe3fada15bb579c5f3870a13ad63b42a737dd72  pypy2-v5.4.1-linux64.tar.bz2
+    ae9329c8f0a6df431c6224c27c634f998688ac803e8d100cee9a774e6bba38b5  pypy2-v5.4.1-osx64.tar.bz2
+    ff118f2dc2393c821595fa7872e4d7bdae2a9a16a854759e103608df383d765a  pypy2-v5.4.1-ppc64.tar.bz2
+    684e862cdb281a22c95d73012f7d2693be9fd9cdd171784256da1514ae1c8164  pypy2-v5.4.1-ppc64le.tar.bz2
+    4004c86bf2dfdd1d92120f514d296af7602e0c5e2c6526a14ff8b5508c0fe8f7  pypy2-v5.4.1-s390x.tar.bz2
+    45dbc50c81498f6f1067201b8fc887074b43b84ee32cc47f15e7db17571e9352  pypy2-v5.4.1-src.tar.bz2
+    54b23c11a92dd6328a58787c3d719189f0aa24c872b479acf50094dfb4be0a46  pypy2-v5.4.1-src.zip
+    ec729218a820bc2aa2cf1fcacf9de0fee9e04144fe138596198a6b4615505e03  pypy2-v5.4.1-win32.zip
 
 pypy3.3-v5.2-alpha sha256::
 

From pypy.commits at gmail.com  Tue Sep  6 14:47:56 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 11:47:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove one more pyobj force,
 rework documentation
Message-ID: <57cf0f5c.02d31c0a.f7eda.09d5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86910:abbb7cdf30ab
Date: 2016-09-06 21:46 +0300
http://bitbucket.org/pypy/pypy/changeset/abbb7cdf30ab/

Log:	remove one more pyobj force, rework documentation

diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
--- a/pypy/doc/release-pypy2.7-v5.4.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -21,7 +21,7 @@
   * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
     the last byte of a unicode string sometimes
 
-  * Update built-in cffi_ to the soon-to-be-released 1.8.2 version
+  * Update built-in cffi_ to version 1.8.1
 
   * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
     translation with a message asking for help porting it
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -14,43 +14,31 @@
 ## Implementation of PyBytesObject
 ## ================================
 ##
-## The problem
-## -----------
+## PyBytesObject has its own ob_sval buffer, so we have two copies of a string;
+## one in the PyBytesObject returned from various C-API functions and another
+## in the corresponding RPython object.
 ##
-## PyString_AsString() must return a (non-movable) pointer to the underlying
-## ob_sval, whereas pypy strings are movable.  C code may temporarily store
-## this address and use it, as long as it owns a reference to the PyObject.
-## There is no "release" function to specify that the pointer is not needed
-## any more.
+## The following calls can create a PyBytesObject without a correspoinding
+## RPython object:
+
+## In any of the above PyBytesObject, the ob_sval buffer may be modified as
+## long as the freshly allocated PyBytesObject is not "forced" via a call
+## to any of the more sophisticated C-API functions. 
 ##
-## Also, the pointer may be used to fill the initial value of string. This is
-## valid only when the string was just allocated, and is not used elsewhere.
-##
-## Solution
-## --------
-##
-## PyBytesObject contains two additional members: the ob_size and an array
-## char ob_sval which holds a \x0 terminated string.
-##
-## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   buffer holding \x0.  The first time PyString_AsString() is called, the 
-##   PyStringObject is reallocated, and the string copied into the buffer. The
-##   ob_size reflects the length of the string.
-##
-## - A string allocated with PyString_FromStringAndSize(NULL, size) will
-##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, as part of the object. The buffer is then supposed to be immutable.
-##
-##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
-##   there is no corresponding pypy object for the string
-##
-## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
-##   similar object.
-##
-## - PyString_Size() doesn't need to force the object.
+## Care has been taken in implementing the functions below, so that
+## if they are called with a non-forced PyBytesObject, they will not 
+## unintentionally force the creation of a RPython object. As long as only these
+## are used, the ob_sval buffer is still modifiable:
+## 
+## PyBytes_AsString / PyString_AsString 
+## PyBytes_AS_STRING / PyString_AS_STRING
+## PyBytes_AsStringAndSize / PyString_AsStringAndSize
+## PyBytes_Size / PyString_Size
+## PyBytes_Resize / PyString_Resize
+## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string.
+##   corresponds to the pypy gc-managed string, 
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -177,9 +165,6 @@
             raise oefmt(space.w_TypeError,
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
-    if not pyobj_has_w_obj(ref):
-        # force the ref
-        bytes_realize(space, ref)
     ref_str = rffi.cast(PyBytesObject, ref)
     data[0] = ref_str.c_ob_sval
     if length:

From pypy.commits at gmail.com  Tue Sep  6 15:02:14 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 06 Sep 2016 12:02:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: documentation
Message-ID: <57cf12b6.a3a3c20a.2eacd.afda@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86911:c1a85c2f66cf
Date: 2016-09-06 22:00 +0300
http://bitbucket.org/pypy/pypy/changeset/c1a85c2f66cf/

Log:	documentation

diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -20,8 +20,10 @@
 ##
 ## The following calls can create a PyBytesObject without a correspoinding
 ## RPython object:
-
-## In any of the above PyBytesObject, the ob_sval buffer may be modified as
+##
+## PyBytes_FromStringAndSize(NULL, n) / PyString_FromStringAndSize(NULL, n)
+##
+## In the PyBytesObject returned, the ob_sval buffer may be modified as
 ## long as the freshly allocated PyBytesObject is not "forced" via a call
 ## to any of the more sophisticated C-API functions. 
 ##

From pypy.commits at gmail.com  Tue Sep  6 16:43:34 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 06 Sep 2016 13:43:34 -0700 (PDT)
Subject: [pypy-commit] pypy remove-getarrayitem-pure: Sanity checks
Message-ID: <57cf2a76.c4f6c20a.c752e.51c1@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: remove-getarrayitem-pure
Changeset: r86913:424b9e80e6a4
Date: 2016-04-05 21:54 -0400
http://bitbucket.org/pypy/pypy/changeset/424b9e80e6a4/

Log:	Sanity checks

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -515,9 +515,24 @@
         return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_F, arraybox,
                                             indexbox, arraydescr)
 
-    opimpl_getarrayitem_gc_i_pure = opimpl_getarrayitem_gc_i
-    opimpl_getarrayitem_gc_r_pure = opimpl_getarrayitem_gc_r
-    opimpl_getarrayitem_gc_f_pure = opimpl_getarrayitem_gc_f
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_gc_i_pure(self, arraybox, indexbox, arraydescr):
+        assert arraydescr.is_always_pure()
+        return self.opimpl_getarrayitem_gc_i(arraybox, indexbox, arraydescr)
+
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_gc_r_pure(self, arraybox, indexbox, arraydescr):
+        assert arraydescr.is_always_pure()
+        return self.opimpl_getarrayitem_gc_r(arraybox, indexbox, arraydescr)
+
+    @arguments("box", "box", "descr")
+    def opimpl_getarrayitem_gc_f_pure(self, arraybox, indexbox, arraydescr):
+        assert arraydescr.is_always_pure()
+        return self.opimpl_getarrayitem_gc_f(arraybox, indexbox, arraydescr)
+
+    # opimpl_getarrayitem_gc_i_pure = opimpl_getarrayitem_gc_i
+    # opimpl_getarrayitem_gc_r_pure = opimpl_getarrayitem_gc_r
+    # opimpl_getarrayitem_gc_f_pure = opimpl_getarrayitem_gc_f
 
     @arguments("box", "box", "descr")
     def opimpl_getarrayitem_raw_i(self, arraybox, indexbox, arraydescr):

From pypy.commits at gmail.com  Tue Sep  6 16:43:32 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 06 Sep 2016 13:43:32 -0700 (PDT)
Subject: [pypy-commit] pypy remove-getarrayitem-pure: Move code
Message-ID: <57cf2a74.c50e1c0a.2dbfe.38f7@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: remove-getarrayitem-pure
Changeset: r86912:2fb300020852
Date: 2016-04-03 13:16 -0400
http://bitbucket.org/pypy/pypy/changeset/2fb300020852/

Log:	Move code

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -447,6 +447,8 @@
                 pass
             self.force_lazy_set(fielddescr, can_cache=False)
         for arraydescr in effectinfo.write_descrs_arrays:
+            if arraydescr.is_always_pure():
+                continue
             self.force_lazy_setarrayitem(arraydescr, can_cache=False)
             if arraydescr in self.corresponding_array_descrs:
                 dictdescr = self.corresponding_array_descrs.pop(arraydescr)
@@ -552,10 +554,7 @@
         cf.do_setfield(self, op)
 
     def optimize_GETARRAYITEM_GC_I(self, op):
-        arrayinfo = self.ensure_ptr_info_arg0(op)
-        indexb = self.getintbound(op.getarg(1))
         arraydescr = op.getdescr()
-
         if (arraydescr.is_always_pure() and
             self.get_constant_box(op.getarg(0)) is not None and
             self.get_constant_box(op.getarg(1)) is not None):
@@ -563,6 +562,8 @@
             self.optimizer.make_constant(op, resbox)
             return
 
+        arrayinfo = self.ensure_ptr_info_arg0(op)
+        indexb = self.getintbound(op.getarg(1))
         cf = None
         if indexb.is_constant():
             index = indexb.getint()

From pypy.commits at gmail.com  Tue Sep  6 16:43:36 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 06 Sep 2016 13:43:36 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Emit proper instruction
 sequence for checking raw pointers
Message-ID: <57cf2a78.919a1c0a.eb019.3add@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86914:9c278b23ff9f
Date: 2016-09-06 16:40 -0400
http://bitbucket.org/pypy/pypy/changeset/9c278b23ff9f/

Log:	Emit proper instruction sequence for checking raw pointers

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -365,6 +365,13 @@
     def visitor_dispatch_virtual_type(self, visitor):
         raise NotImplementedError("abstract")
 
+    def make_guards(self, op, short, optimizer):
+        from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
+        op = ResOperation(rop.INT_EQ, [op, CONST_0])
+        short.append(op)
+        op = ResOperation(rop.GUARD_FALSE, [op])
+        short.append(op)
+
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
 

From pypy.commits at gmail.com  Tue Sep  6 18:02:08 2016
From: pypy.commits at gmail.com (stefanor)
Date: Tue, 06 Sep 2016 15:02:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: If the expected exception was raised,
 the SSLContext can't be shut down yet
Message-ID: <57cf3ce0.04141c0a.63e28.531f@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r86915:0dcc383e61e3
Date: 2016-09-06 15:01 -0700
http://bitbucket.org/pypy/pypy/changeset/0dcc383e61e3/

Log:	If the expected exception was raised, the SSLContext can't be shut
	down yet

diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -450,7 +450,12 @@
                 # For compatibility
                 assert exc.value.errno == _ssl.SSL_ERROR_WANT_READ
             finally:
-                c.shutdown()
+                try:
+                    c.shutdown()
+                except _ssl.SSLError:
+                    # If the expected exception was raised, the SSLContext
+                    # can't be shut down yet
+                    pass
         finally:
             s.close()
 

From pypy.commits at gmail.com  Wed Sep  7 00:16:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 06 Sep 2016 21:16:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57cf9481.0575c20a.329f0.3186@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86916:37c0e5dcf737
Date: 2016-09-07 05:15 +0100
http://bitbucket.org/pypy/pypy/changeset/37c0e5dcf737/

Log:	hg merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -30,3 +30,6 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.1
+Version: 1.8.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.1"
-__version_info__ = (1, 8, 1)
+__version__ = "1.8.2"
+__version_info__ = (1, 8, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.1"
+                               "\ncompiled with cffi version: 1.8.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1009,6 +1009,12 @@
                         myref is not None and myref is other())
                 def __ne__(self, other):
                     return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
             self._weakref_cache_ref = {}, MyRef
         weak_cache, MyRef = self._weakref_cache_ref
 
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -0,0 +1,64 @@
+==========
+PyPy 5.4.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.4.0, released last week,
+due to the following issues:
+
+  * Update list of contributors in documentation and LICENSE file,
+    this was unfortunately left out of 5.4.0. My apologies to the new
+    contributors
+
+  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+    dynamically loadable file
+
+  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+
+  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+
+  * Fix for an issue where os.access() accepted a float for mode
+
+  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+    the last byte of a unicode string sometimes
+
+  * Update built-in cffi_ to version 1.8.1
+
+  * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
+    translation with a message asking for help porting it
+
+  * Fix a regression where a PyBytesObject was forced (converted to a RPython
+    object) when not required, reported as issue #2395
+
+Thanks to those who reported the issues.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _cffi: https://cffi.readthedocs.io
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,9 @@
 
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1432,6 +1432,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.1"
+VERSION = "1.8.2"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
                 return True
         return False
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        pass
-
     def convert_from_object(self, cdata, w_ob):
         if not self._copy_from_same(cdata, w_ob):
             self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
     )
     def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self.force_lazy_struct()
-        self._check_only_one_argument_for_union(w_ob)
 
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
-            if len(lst_w) > len(self._fields_list):
-                raise oefmt(space.w_ValueError,
-                            "too many initializers for '%s' (got %d)",
-                            self.name, len(lst_w))
-            for i in range(len(lst_w)):
-                optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+            j = 0
+            for w_obj in lst_w:
+                try:
+                    while (self._fields_list[j].flags &
+                               W_CField.BF_IGNORE_IN_CTOR):
+                        j += 1
+                except IndexError:
+                    raise oefmt(space.w_ValueError,
+                                "too many initializers for '%s' (got %d)",
+                                self.name, len(lst_w))
+                optvarsize = self._fields_list[j].write_v(cdata, w_obj,
                                                           optvarsize)
+                j += 1
             return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
 class W_CTypeUnion(W_CTypeStructOrUnion):
     kind = "union"
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        space = self.space
-        n = space.int_w(space.len(w_ob))
-        if n > 1:
-            raise oefmt(space.w_ValueError,
-                        "initializer for '%s': %d items given, but only one "
-                        "supported (use a dict if needed)", self.name, n)
-
 
 class W_CField(W_Root):
     _immutable_ = True
@@ -200,18 +194,21 @@
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
 
-    def __init__(self, ctype, offset, bitshift, bitsize):
+    BF_IGNORE_IN_CTOR = 0x01
+
+    def __init__(self, ctype, offset, bitshift, bitsize, flags):
         self.ctype = ctype
         self.offset = offset
         self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
         self.bitsize = bitsize
+        self.flags = flags       # BF_xxx
 
     def is_bitfield(self):
         return self.bitshift >= 0
 
-    def make_shifted(self, offset):
+    def make_shifted(self, offset, fflags):
         return W_CField(self.ctype, offset + self.offset,
-                        self.bitshift, self.bitsize)
+                        self.bitshift, self.bitsize, self.flags | fflags)
 
     def read(self, cdata):
         cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
     offset = interp_attrproperty('offset', W_CField),
     bitshift = interp_attrproperty('bitshift', W_CField),
     bitsize = interp_attrproperty('bitsize', W_CField),
+    flags = interp_attrproperty('flags', W_CField),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
         if alignment < falign and do_align:
             alignment = falign
         #
+        if is_union and i > 0:
+            fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+        else:
+            fflags = 0
+        #
         if fbitsize < 0:
             # not a bitfield: common case
 
@@ -372,7 +377,7 @@
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8)
+                    fld = srcfld.make_shifted(boffset // 8, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                                           fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
@@ -489,7 +495,7 @@
                     bitshift = 8 * ftype.size - fbitsize- bitshift
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
-                                           bitshift, fbitsize)
+                                           bitshift, fbitsize, fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -121,7 +121,7 @@
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
 """.split()
 for name in constant_names:
@@ -647,6 +647,9 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.UCHAR),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,20 +1,66 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import buffer
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
 from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
-# PyObject_GetBuffer has been removed, it is defined in abstract.c
-# PyObject_CheckBuffer is also already defined
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+        
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
+def PyBuffer_IsContiguous(space, view, fort):
     """Return 1 if the memory defined by the view is C-style (fortran is
     'C') or Fortran-style (fortran is 'F') contiguous or either one
     (fortran is 'A').  Return 0 otherwise."""
-    # PyPy only supports contiguous Py_buffers for now.
-    return 1
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
+    
 
 class CBuffer(buffer.Buffer):
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -14,45 +14,33 @@
 ## Implementation of PyBytesObject
 ## ================================
 ##
-## The problem
-## -----------
+## PyBytesObject has its own ob_sval buffer, so we have two copies of a string;
+## one in the PyBytesObject returned from various C-API functions and another
+## in the corresponding RPython object.
 ##
-## PyBytes_AsString() must return a (non-movable) pointer to the underlying
-## ob_sval, whereas pypy strings are movable.  C code may temporarily store
-## this address and use it, as long as it owns a reference to the PyObject.
-## There is no "release" function to specify that the pointer is not needed
-## any more.
+## The following calls can create a PyBytesObject without a correspoinding
+## RPython object:
 ##
-## Also, the pointer may be used to fill the initial value of string. This is
-## valid only when the string was just allocated, and is not used elsewhere.
+## PyBytes_FromStringAndSize(NULL, n) / PyString_FromStringAndSize(NULL, n)
 ##
-## Solution
-## --------
+## In the PyBytesObject returned, the ob_sval buffer may be modified as
+## long as the freshly allocated PyBytesObject is not "forced" via a call
+## to any of the more sophisticated C-API functions. 
 ##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
-##
-## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   NULL buffer.  The first time PyBytes_AsString() is called, memory is
-##   allocated (with flavor='raw') and content is copied.
-##
-## - A string allocated with PyBytes_FromStringAndSize(NULL, size) will
-##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, but the reference won't be stored in the global map; there is no
-##   corresponding object in pypy.  When from_ref() or Py_INCREF() is called,
-##   the pypy string is created, and added to the global map of tracked
-##   objects.  The buffer is then supposed to be immutable.
-##
-##-  A buffer obtained from PyBytes_AS_STRING() could be mutable iff
-##   there is no corresponding pypy object for the string
-##
-## - _PyBytes_Resize() works only on not-yet-pypy'd strings, and returns a
-##   similar object.
-##
-## - PyBytes_Size() doesn't need to force the object.
+## Care has been taken in implementing the functions below, so that
+## if they are called with a non-forced PyBytesObject, they will not 
+## unintentionally force the creation of a RPython object. As long as only these
+## are used, the ob_sval buffer is still modifiable:
+## 
+## PyBytes_AsString / PyString_AsString 
+## PyBytes_AS_STRING / PyString_AS_STRING
+## PyBytes_AsStringAndSize / PyString_AsStringAndSize
+## PyBytes_Size / PyString_Size
+## PyBytes_Resize / PyString_Resize
+## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string.
+##   corresponds to the pypy gc-managed string, 
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -150,9 +138,6 @@
         raise oefmt(space.w_TypeError,
             "expected bytes, %T found", from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not pyobj_has_w_obj(ref):
-        # XXX Force the ref?
-        bytes_realize(space, ref)
     return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
@@ -170,9 +155,6 @@
     if not PyBytes_Check(space, ref):
         raise oefmt(space.w_TypeError,
             "expected bytes, %T found", from_ref(space, ref))
-    if not pyobj_has_w_obj(ref):
-        # force the ref
-        bytes_realize(space, ref)
     ref_str = rffi.cast(PyBytesObject, ref)
     data[0] = ref_str.c_ob_sval
     if length:
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -131,7 +131,8 @@
 typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
 
 
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -145,12 +146,14 @@
     char *format;
     Py_ssize_t *shape;
     Py_ssize_t *strides;
-    Py_ssize_t *suboffsets;
-
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format;
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
        mono-dimensional buffers. */
     /* Py_ssize_t smalltable[2]; */
-    void *internal;
+    void *internal; /* always NULL for app-level objects */
 } Py_buffer;
 
 
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.1-alpha0"
-#define PYPY_VERSION_NUM  0x05040100
+#define PYPY_VERSION "5.5.0-alpha0"
+#define PYPY_VERSION_NUM  0x05050000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,7 +1,8 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.pyobject import PyObject, from_ref
@@ -16,6 +17,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -24,24 +26,38 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-    # TODO - fill in fields
-    '''
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if not isinstance(w_obj, W_MemoryView):
+        return view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_obj)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        isstr = False
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        rffi.setintfield(view, 'c_readonly', 1)
+        isstr = True
+    view.c_len = w_obj.getlength()
+    view.c_itemsize = w_obj.buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    shape = w_obj.buf.getshape()
+    strides = w_obj.buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    ''' 
     return view
 
-
 @cpython_api([lltype.Ptr(Py_buffer)], PyObject)
 def PyMemoryView_FromBuffer(space, view):
     """Create a memoryview object wrapping the given buffer structure view.
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
     def getshape(self):
         return self.shape
 
+    def getstrides(self):
+        return self.strides
+
     def getitemsize(self):
         return self.itemsize
 
+    def getndim(self):
+        return self.ndim
+
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
   PyMyArray* self = (PyMyArray*)obj;
-  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
-    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
-    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
@@ -188,7 +185,131 @@
     (initproc)PyMyArray_init,     /* tp_init */
 };
 
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+    Py_buffer* view = NULL;
+    PyObject* obj = PyTuple_GetItem(args, 0);
+    PyObject* memoryview = PyMemoryView_FromObject(obj);
+    if (memoryview == NULL)
+        return PyInt_FromLong(-1);
+    view = PyMemoryView_GET_BUFFER(memoryview);
+    Py_DECREF(memoryview);
+    return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG)                                        \
+    buf_flag = PyUnicode_FromString(#FLAG);                         \
+    flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ);  \
+    Py_DECREF(buf_flag);                                            \
+    if (flag_matches == 1) {                                        \
+        Py_DECREF(tmp);                                             \
+        flags |= PyBUF_##FLAG;                                      \
+        continue;                                                   \
+    }                                                               \
+    else if (flag_matches == -1) {                                  \
+        Py_DECREF(tmp);                                             \
+        return NULL;                                                \
+    }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+    PyObject *buffer_obj, *pyflags;
+    PyObject *tmp, *buf_flag;
+    Py_buffer buffer;
+    PyObject *shape, *strides;
+    Py_ssize_t i, n;
+    int flag_matches;
+    int flags = 0;
+
+    if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+        return NULL;
+    }
+
+    n = PySequence_Length(pyflags);
+    if (n < 0) {
+        return NULL;
+    }
+
+    for (i=0; i < n; i++) {
+        tmp = PySequence_GetItem(pyflags, i);
+        if (tmp == NULL) {
+            return NULL;
+        }
+
+        GET_PYBUF_FLAG(SIMPLE);
+        GET_PYBUF_FLAG(WRITABLE);
+        GET_PYBUF_FLAG(STRIDES);
+        GET_PYBUF_FLAG(ND);
+        GET_PYBUF_FLAG(C_CONTIGUOUS);
+        GET_PYBUF_FLAG(F_CONTIGUOUS);
+        GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+        GET_PYBUF_FLAG(INDIRECT);
+        GET_PYBUF_FLAG(FORMAT);
+        GET_PYBUF_FLAG(STRIDED);
+        GET_PYBUF_FLAG(STRIDED_RO);
+        GET_PYBUF_FLAG(RECORDS);
+        GET_PYBUF_FLAG(RECORDS_RO);
+        GET_PYBUF_FLAG(FULL);
+        GET_PYBUF_FLAG(FULL_RO);
+        GET_PYBUF_FLAG(CONTIG);
+        GET_PYBUF_FLAG(CONTIG_RO);
+
+        Py_DECREF(tmp);
+
+        /* One of the flags must match */
+        PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+        return NULL;
+    }
+
+    if (buffer.shape == NULL) {
+        Py_INCREF(Py_None);
+        shape = Py_None;
+    }
+    else {
+        shape = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+        }
+    }
+
+    if (buffer.strides == NULL) {
+        Py_INCREF(Py_None);
+        strides = Py_None;
+    }
+    else {
+        strides = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+        }
+    }
+
+    PyBuffer_Release(&buffer);
+    return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
 static PyMethodDef buffer_functions[] = {
+    {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
+    {"get_buffer_info",   (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -179,8 +179,27 @@
                  Py_INCREF(Py_None);
                  return Py_None;
              """),
+            ("c_only", "METH_NOARGS",
+            """
+                int ret;
+                char * buf2;
+                PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+                if (!obj)
+                    return NULL;
+                buf2 = PyBytes_AsString(obj);
+                if (!buf2)
+                    return NULL;
+                /* buf should not have been forced, issue #2395 */
+                ret = _PyBytes_Resize(&obj, 512);
+                if (ret < 0)
+                    return NULL;
+                 Py_DECREF(obj);
+                 Py_INCREF(Py_None);
+                 return Py_None;
+            """),
             ])
         module.getbytes()
+        module.c_only()
 
 
 class TestBytes(BaseApiTest):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
@@ -11,6 +11,13 @@
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
+    def test_frombuffer(self, space, api):
+        w_buf = space.newbuffer(StringBuffer("hello"))
+        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        ndim = w_view.c_ndim
+        assert ndim == 1
+
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     def test_fillWithObject(self):
         module = self.import_extension('foo', [
@@ -62,6 +69,25 @@
         y = memoryview(arr)
         assert y.format == 'i'
         assert y.shape == (10,)
+        assert len(y) == 10
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
+        viewlen = module.test_buffer(arr)
+        assert viewlen == y.itemsize * len(y)
+
+    def test_buffer_info(self):
+        from _numpypy import multiarray as np
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        # test_export_flags from numpy test_multiarray
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        # test_relaxed_strides from numpy test_multiarray
+        arr = np.zeros((1, 10))
+        if arr.flags.f_contiguous:
+            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+            assert strides[0] == 8
+            arr = np.ones((10, 1), order='F')
+            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+            assert strides[-1] == 8
+
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -41,9 +41,11 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
                     STRUCT_TYPE = PySequenceMethods
+                elif slot_names[0] == 'c_tp_as_buffer':
+                    STRUCT_TYPE = PyBufferProcs
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
     def getdictvalue(self, space, key):
         return self.items[key]
 
+    def descr_memoryview(self, space, buf):
+        raise oefmt(space.w_TypeError, "error")
+
 class IterDictObject(W_Root):
     def __init__(self, space, w_dict):
         self.space = space
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -377,7 +377,25 @@
     def __exit__(self, typ, value, traceback):
         keepalive_until_here(self)
 
-    def get_buffer(self, space, readonly):
+    def get_buffer(self, space, flags):
+        errtype = space.w_ValueError # should be BufferError, numpy does this instead
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_F_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not Fortran contiguous")
+        if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                     self.flags & NPY.ARRAY_C_CONTIGUOUS)):
+           raise oefmt(errtype, "ndarray is not contiguous")
+        if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and
+            not self.flags & NPY.ARRAY_WRITEABLE):
+           raise oefmt(errtype, "buffer source array is read-only")
+        readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
         return ArrayBuffer(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
@@ -695,6 +713,7 @@
                  index + self.impl.start)
 
     def setitem(self, index, v):
+        # XXX what if self.readonly?
         raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstring import strip_spaces
@@ -42,7 +43,7 @@
         raise oefmt(space.w_ValueError,
                     "object __array__ method not producing an array")
 
-def try_interface_method(space, w_object):
+def try_interface_method(space, w_object, copy):
     try:
         w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
         if w_interface is None:
@@ -81,17 +82,20 @@
             raise oefmt(space.w_ValueError,
                     "__array_interface__ could not decode dtype %R", w_dtype
                     )
-        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
+        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
+                                   space.isinstance_w(w_data, space.w_list)):
             data_w = space.listview(w_data)
-            data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
-            read_only = True # XXX why not space.is_true(data_w[1])
+            w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+            read_only = space.is_true(data_w[1]) or copy
             offset = 0
-            return W_NDimArray.from_shape_and_storage(space, shape, data, 
-                                    dtype, strides=strides, start=offset), read_only
+            w_base = w_object
+            if read_only:
+                w_base = None
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+                                dtype, w_base=w_base, strides=strides,
+                                start=offset), read_only
         if w_data is None:
-            data = w_object
-        else:
-            data = w_data
+            w_data = w_object
         w_offset = space.finditem(w_interface, space.wrap('offset'))
         if w_offset is None:
             offset = 0
@@ -101,7 +105,7 @@
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
                    "__array_interface__ strides not fully supported yet") 
-        arr = frombuffer(space, data, dtype, support.product(shape), offset)
+        arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
         
@@ -110,6 +114,78 @@
             return None, False
         raise
 
+def _descriptor_from_pep3118_format(space, c_format):
+    descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+    if descr:
+        return descr
+    msg = "invalid PEP 3118 format string: '%s'" % c_format
+    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+    return None 
+
+def _array_from_buffer_3118(space, w_object, dtype):
+    try:
+        w_buf = space.call_method(space.builtin, "memoryview", w_object)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            # object does not have buffer interface
+            return w_object
+        raise
+    format = space.getattr(w_buf,space.newbytes('format'))
+    if format:
+        descr = _descriptor_from_pep3118_format(space, space.str_w(format))
+        if not descr:
+            return w_object
+        if dtype and descr:
+            raise oefmt(space.w_NotImplementedError,
+                "creating an array from a memoryview while specifying dtype "
+                "not supported")
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
+            msg = ("Item size computed from the PEP 3118 buffer format "
+                  "string does not match the actual item size.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+        dtype = descr 
+    elif not dtype:
+        dtype = descriptor.get_dtype_cache(space).w_stringdtype
+        dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
+    nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim')))
+    shape = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('shape')))]
+    strides = []
+    buflen = space.len_w(w_buf) * dtype.elsize
+    if shape:
+        strides = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('strides')))]
+        if not strides:
+            d = buflen
+            strides = [0] * nd
+            for k in range(nd):
+                if shape[k] > 0:
+                    d /= shape[k]
+                    strides[k] = d
+    else:
+        if nd == 1:
+            shape = [buflen / dtype.elsize, ]
+            strides = [dtype.elsize, ]
+        elif nd > 1:
+            msg = ("ndim computed from the PEP 3118 buffer format "
+                   "is greater than 1, but shape is NULL.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+    try:
+        w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
+    except OperationError as e:
+        if e.match(space, space.w_ValueError):
+            return w_object
+        else:
+            raise e
+    writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
+    w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
+               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
+               writable=writable, strides=strides)
+    if w_ret:
+        return w_ret
+    return w_object
 
 @unwrap_spec(ndmin=int, copy=bool, subok=bool)
 def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -127,6 +203,7 @@
 
 def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
 
+    from pypy.module.micronumpy.boxes import W_GenericBox
     # numpy testing calls array(type(array([]))) and expects a ValueError
     if space.isinstance_w(w_object, space.w_type):
         raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
@@ -134,13 +211,19 @@
     dtype = descriptor.decode_w_dtype(space, w_dtype)
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
-        if w_array is not None:
+        if w_array is None:
+            if (    not space.isinstance_w(w_object, space.w_str) and 
+                    not space.isinstance_w(w_object, space.w_unicode) and
+                    not isinstance(w_object, W_GenericBox)):
+                # use buffer interface
+                w_object = _array_from_buffer_3118(space, w_object, dtype)
+        else:
             # continue with w_array, but do further operations in place
             w_object = w_array
             copy = False
             dtype = w_object.get_dtype()
     if not isinstance(w_object, W_NDimArray):
-        w_array, _copy = try_interface_method(space, w_object)
+        w_array, _copy = try_interface_method(space, w_object, copy)
         if w_array is not None:
             w_object = w_array
             copy = _copy
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -806,10 +806,10 @@
 
     def buffer_w(self, space, flags):
         # XXX format isn't always 'B' probably
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, flags)
 
     def descr_get_data(self, space):
-        return space.newbuffer(self.implementation.get_buffer(space, False))
+        return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3206,7 +3206,9 @@
         raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)}))
 
         a = array([1, 2, 3])
-        b = array(Dummy(a.__array_interface__))
+        d = Dummy(a.__array_interface__)
+        b = array(d)
+        assert b.base is None
         b[1] = 200
         assert a[1] == 2 # upstream compatibility, is this a bug?
         interface_a = a.__array_interface__
@@ -3217,6 +3219,8 @@
         interface_b.pop('data')
         interface_a.pop('data')
         assert interface_a == interface_b
+        b = array(d, copy=False)
+        assert b.base is d
 
         b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
                          'data': 'a'*100}))
@@ -3585,6 +3589,7 @@
         cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
         cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
         cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+        cls.w_one = cls.space.wrap(struct.pack('i', 1))
 
     def test_frombuffer(self):
         import numpy as np
@@ -3636,8 +3641,6 @@
         else:
             EMPTY = None
         x = np.array([1, 2, 3, 4, 5], dtype='i')
-        y = memoryview('abc')
-        assert y.format == 'B'
         y = memoryview(x)
         assert y.format == 'i'
         assert y.shape == (5,)
@@ -3645,6 +3648,16 @@
         assert y.strides == (4,)
         assert y.suboffsets == EMPTY
         assert y.itemsize == 4
+        assert isinstance(y, memoryview)
+        assert y[0] == self.one
+        assert (np.array(y) == x).all()
+
+        x = np.array([0, 0, 0, 0], dtype='O')
+        y = memoryview(x)
+        # handles conversion of address to pinned object?
+        z = np.array(y)
+        assert z.dtype == 'O'
+        assert (z == x).all()
 
     def test_fromstring(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -702,3 +702,32 @@
         ret = obj.sum()
         print type(ret)
         assert ret.info == 'spam'
+
+    def test_ndarray_subclass_assigns_base(self):
+        import numpy as np
+        init_called = []
+        class _DummyArray(object):
+            """ Dummy object that just exists to hang __array_interface__ dictionaries
+            and possibly keep alive a reference to a base array.
+            """
+            def __init__(self, interface, base=None):
+                self.__array_interface__ = interface
+                init_called.append(1)
+                self.base = base
+
+        x = np.zeros(10)
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 1
+        assert y.base is d
+
+        x = np.zeros((0,), dtype='float32')
+        intf = x.__array_interface__.copy()
+        intf["strides"] = x.strides
+        x.__array_interface__["strides"] = x.strides
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 2
+        assert y.base is d
+
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1851,7 +1851,7 @@
                     arr.gcstruct)
 
     def read(self, arr, i, offset, dtype):
-        if arr.gcstruct is V_OBJECTSTORE:
+        if arr.gcstruct is V_OBJECTSTORE and not arr.base():
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot read object from array with no gc hook")
         return self.box(self._read(arr.storage, i, offset))
diff --git a/pypy/module/pypyjit/test_pypy_c/test_import.py b/pypy/module/pypyjit/test_pypy_c/test_import.py
--- a/pypy/module/pypyjit/test_pypy_c/test_import.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_import.py
@@ -38,3 +38,27 @@
         # call_may_force(absolute_import_with_lock).
         for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")):
             assert 'call' not in opname    # no call-like opcode
+
+    def test_import_fast_path(self, tmpdir):
+        print tmpdir
+        pkg = tmpdir.join('mypkg').ensure(dir=True)
+        subdir = pkg.join("sub").ensure(dir=True)
+        pkg.join('__init__.py').write("")
+        subdir.join('__init__.py').write("")
+        subdir.join('mod.py').write(str(py.code.Source("""
+            def do_the_import():
+                import sys
+        """)))
+        def main(path, n):
+            def do_the_import():
+                from mypkg.sub import mod
+            import sys
+            sys.path.append(path)
+            for i in range(n):
+                do_the_import()
+        #
+        log = self.run(main, [str(tmpdir), 300])
+        loop, = log.loops_by_filename(self.filepath)
+        # check that no string compares and other calls are there
+        for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")):
+            assert 'call' not in opname    # no call-like opcode
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 5, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1415,6 +1415,7 @@
         assert p.b == 12
         assert p.c == 14
         assert p.d == 14
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [0, 0, 0, 0])
 
     def test_nested_field_offset_align(self):
         ffi = FFI(backend=self.Backend())
@@ -1454,14 +1455,42 @@
         assert p.b == 0
         assert p.c == 14
         assert p.d == 14
-        p = ffi.new("union foo_u *", {'b': 12})
-        assert p.a == 0
+        p = ffi.new("union foo_u *", {'a': -63, 'b': 12})
+        assert p.a == -63
         assert p.b == 12
-        assert p.c == 0
-        assert p.d == 0
-        # we cannot specify several items in the dict, even though
-        # in theory in this particular case it would make sense
-        # to give both 'a' and 'b'
+        assert p.c == -63
+        assert p.d == -63
+        p = ffi.new("union foo_u *", [123, 456])
+        assert p.a == 123
+        assert p.b == 456
+        assert p.c == 123
+        assert p.d == 123
+        py.test.raises(ValueError, ffi.new, "union foo_u *", [0, 0, 0])
+
+    def test_nested_anonymous_struct_2(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            struct foo_s {
+                int a;
+                union { int b; union { int c, d; }; };
+                int e;
+            };
+        """)
+        assert ffi.sizeof("struct foo_s") == 3 * SIZE_OF_INT
+        p = ffi.new("struct foo_s *", [11, 22, 33])
+        assert p.a == 11
+        assert p.b == p.c == p.d == 22
+        assert p.e == 33
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [11, 22, 33, 44])
+        FOO = ffi.typeof("struct foo_s")
+        fields = [(name, fld.offset, fld.flags) for (name, fld) in FOO.fields]
+        assert fields == [
+            ('a', 0 * SIZE_OF_INT, 0),
+            ('b', 1 * SIZE_OF_INT, 0),
+            ('c', 1 * SIZE_OF_INT, 1),
+            ('d', 1 * SIZE_OF_INT, 1),
+            ('e', 2 * SIZE_OF_INT, 0),
+        ]
 
     def test_cast_to_array_type(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
@@ -35,6 +35,9 @@
     def test_nested_anonymous_union(self):
         py.test.skip("ctypes backend: not supported: nested anonymous union")
 
+    def test_nested_anonymous_struct_2(self):
+        py.test.skip("ctypes backend: not supported: nested anonymous union")
+
     def test_CData_CType_2(self):
         if sys.version_info >= (3,):
             py.test.skip("ctypes backend: not supported in Python 3: CType")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -154,7 +154,10 @@
         from cffi.setuptools_ext import _set_py_limited_api
         try:
             import setuptools
-            orig_version = setuptools.__version__
+        except ImportError as e:
+            py.test.skip(str(e))
+        orig_version = setuptools.__version__
+        try:
             setuptools.__version__ = '26.0.0'
             from setuptools import Extension
 
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -64,6 +64,8 @@
 
     def setitem_str(self, w_dict, key, w_value):
         cell = self.getdictvalue_no_unwrapping(w_dict, key)
+        #if (key == '__package__' or key == "__path__") and cell is not None and w_value is not cell:
+        #    print "WARNING", key, w_value, cell, self
         return self._setitem_str_cell_known(cell, w_dict, key, w_value)
 
     def _setitem_str_cell_known(self, cell, w_dict, key, w_value):
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -26,6 +26,7 @@
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
+    _attrs_ = ['buf']
 
     def __init__(self, buf, format=None, itemsize=1, ndim=-1,
                  shape=None, strides=None, suboffsets=None):
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
 min=4
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,6 +97,21 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
+if OPENSSL_VERSION_NUMBER >= 0x10100000:
+    eci.pre_include_bits = ()
+    eci.post_include_bits = ()
+    raise Exception("""OpenSSL version >= 1.1 not supported yet.
+
+    This program requires OpenSSL version 1.0.x, and may also
+    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
+    some work to update to; contributions are welcome.  Sorry,
+    you need to install an older version of OpenSSL for now.
+    Make sure this older version is the one picked up by this
+    program when it runs the compiler.
+    
+    This is the configuration used: %r""" % (eci,))
+
+
 class CConfig:
     _compilation_info_ = eci
 
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1,5 +1,5 @@
 import sys
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, enforceargs
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
@@ -145,19 +145,21 @@
 _invalid_byte_3_of_4 = _invalid_cont_byte
 _invalid_byte_4_of_4 = _invalid_cont_byte
 
- at specialize.arg(2)
+ at enforceargs(allow_surrogates=bool)
 def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xe0 and ordch2 < 0xa0)
             # surrogates shouldn't be valid UTF-8!
-            or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f))
+            or (ordch1 == 0xed and ordch2 > 0x9f and not allow_surrogates))
 
 def _invalid_byte_2_of_4(ordch1, ordch2):
     return (ordch2>>6 != 0x2 or    # 0b10
             (ordch1 == 0xf0 and ordch2 < 0x90) or
             (ordch1 == 0xf4 and ordch2 > 0x8f))
 
- at specialize.arg(5)
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(4)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:
@@ -330,6 +332,9 @@
     return unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                                      allow_surrogates=allow_surrogates)
 
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(3)
 def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
                               allow_surrogates=False):
     assert(size >= 0)
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -55,7 +55,7 @@
                 s = s.encode(encoding)
         except LookupError as e:
             py.test.skip(e)
-        result, consumed = decoder(s, len(s), True)
+        result, consumed = decoder(s, len(s), 'strict', final=True)
         assert consumed == len(s)
         self.typeequals(trueresult, result)
 
@@ -69,7 +69,7 @@
                 s = s.decode(encoding)
         except LookupError as e:
             py.test.skip(e)
-        result = encoder(s, len(s), True)
+        result = encoder(s, len(s), 'strict')
         self.typeequals(trueresult, result)
 
     def checkencodeerror(self, s, encoding, start, stop):
@@ -823,9 +823,15 @@
         def f(x):
 
             s1 = "".join(["\xd7\x90\xd6\x96\xeb\x96\x95\xf0\x90\x91\x93"] * x)
-            u, consumed = runicode.str_decode_utf_8(s1, len(s1), True)
-            s2 = runicode.unicode_encode_utf_8(u, len(u), True)
-            return s1 == s2
+            u, consumed = runicode.str_decode_utf_8(s1, len(s1), 'strict',
+                                                    allow_surrogates=True)
+            s2 = runicode.unicode_encode_utf_8(u, len(u), 'strict',
+                                                    allow_surrogates=True)
+            u3, consumed3 = runicode.str_decode_utf_8(s1, len(s1), 'strict',
+                                                    allow_surrogates=False)
+            s3 = runicode.unicode_encode_utf_8(u3, len(u3), 'strict',
+                                                    allow_surrogates=False)
+            return s1 == s2 == s3
         res = interpret(f, [2])
         assert res
 
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -30,12 +30,13 @@
         assert value is not None
         result = UnicodeBuilder(len(value))
         self.rstr_decode_utf_8(
-            value, len(value), 'strict', final=False,
+            value, len(value), 'strict', final=True,
             errorhandler=self.ll_raise_unicode_exception_decode,
             allow_surrogates=False, result=result)
         return self.ll.llunicode(result.build())
 
-    def ll_raise_unicode_exception_decode(self, errors, encoding, msg, s,
+    @staticmethod
+    def ll_raise_unicode_exception_decode(errors, encoding, msg, s,
                                        startingpos, endingpos):
         raise UnicodeDecodeError(encoding, s, startingpos, endingpos, msg)
 
@@ -411,7 +412,8 @@
             allow_surrogates=False)
         return self.ll.llstr(bytes)
 
-    def ll_raise_unicode_exception_encode(self, errors, encoding, msg, u,
+    @staticmethod
+    def ll_raise_unicode_exception_encode(errors, encoding, msg, u,
                                           startingpos, endingpos):
         raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py
--- a/rpython/rtyper/test/test_runicode.py
+++ b/rpython/rtyper/test/test_runicode.py
@@ -162,6 +162,18 @@
 
         assert self.ll_to_string(self.interpret(f, [0])) == f(0)
 
+    def test_unicode_decode_final(self):
+        strings = ['\xc3', '']
+        def f(n):
+            try:
+                strings[n].decode('utf-8')
+            except UnicodeDecodeError:
+                return True
+            return False
+
+        assert f(0)
+        assert self.interpret(f, [0])
+
     def test_utf_8_decoding_annotation(self):
         from rpython.rlib.runicode import str_decode_utf_8
         def errorhandler(errors, encoding, msg, s,

From pypy.commits at gmail.com  Wed Sep  7 03:00:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 00:00:01 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57cfbaf1.c398c20a.9932f.bb14@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r787:e957cbe8c298
Date: 2016-09-07 08:59 +0200
http://bitbucket.org/pypy/pypy.org/changeset/e957cbe8c298/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 61.9
+      value: 62.0
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $64959 of $105000 (61.9%)
+   $65097 of $105000 (62.0%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5426 left
+    we have $5552 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $58959 of $80000 (73.7%)
+   $58969 of $80000 (73.7%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $48187 left
+    we have $48195 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Sep  7 03:25:40 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Sep 2016 00:25:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix formatting
Message-ID: <57cfc0f4.121a1c0a.c5d01.d3e4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86917:053c990217cb
Date: 2016-09-07 10:21 +0300
http://bitbucket.org/pypy/pypy/changeset/053c990217cb/

Log:	fix formatting

diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
--- a/pypy/doc/release-pypy2.7-v5.4.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -9,16 +9,16 @@
     this was unfortunately left out of 5.4.0. My apologies to the new
     contributors
 
-  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+  * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a
     dynamically loadable file
 
-  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+  * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython
 
-  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+  * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
 
   * Fix for an issue where os.access() accepted a float for mode
 
-  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+  * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
     the last byte of a unicode string sometimes
 
   * Update built-in cffi_ to version 1.8.1

From pypy.commits at gmail.com  Wed Sep  7 03:28:41 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Sep 2016 00:28:41 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update for 5.4.1
Message-ID: <57cfc1a9.88711c0a.4e105.d5d9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r788:34944c992b22
Date: 2016-09-07 10:26 +0300
http://bitbucket.org/pypy/pypy.org/changeset/34944c992b22/

Log:	update for 5.4.1

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,7 +74,8 @@
 performance improvements.</p>
 <p>We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:</p>
 <ul class="simple">
-<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.4.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.0.html">what's new in PyPy2.7?</a> )</li>
+<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.4.1</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.0.html">what's new in PyPy2.7?</a>
+and <a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.1.html">bugfix release</a>)</li>
 <li>the Python3.3 compatible release &#8212; <strong>PyPy3.3 v5.2-alpha</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy3.3-v5.2-alpha1.html">what's new in PyPy3.3?</a>).</li>
 <li>the Python2.7 Software Transactional Memory special release &#8212; <strong>PyPy-STM 2.5.1</strong> (Linux x86-64 only)</li>
 </ul>
@@ -114,22 +115,22 @@
 <li>or <a class="reference internal" href="#translate">translate</a> your own PyPy.</li>
 </ul>
 </div>
-<div class="section" id="python2-7-compatible-pypy-5-4-0">
-<h1>Python2.7 compatible PyPy 5.4.0</h1>
+<div class="section" id="python2-7-compatible-pypy-5-4-1">
+<h1>Python2.7 compatible PyPy 5.4.1</h1>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
 <li>FreeBSD 9.2 x86 64 bit <em>(hopefully availabe soon)</em> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
 installer <a class="reference external" href="http://www.microsoft.com/en-us/download/details.aspx?id=5582">vcredist_x86.exe</a>.)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-s390x.tar.bz2">s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-s390x.tar.bz2">s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
 <a class="reference external" href="http://buildbot.pypy.org/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
 </ul>
@@ -198,7 +199,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in <tt class="docutils literal">/opt</tt>, and
 if you want, put a symlink from somewhere like
-<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy2-5.4.0/bin/pypy</span></tt>.  Do
+<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy2-5.4.1/bin/pypy</span></tt>.  Do
 not move or copy the executable <tt class="docutils literal">pypy</tt> outside the tree &ndash; put
 a symlink to it, otherwise it will not find its libraries.</p>
 </div>
@@ -263,7 +264,7 @@
 <li><p class="first">Get the source code.  The following packages contain the source at
 the same revision as the above binaries:</p>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.tar.bz2">pypy2-v5.4.0-src.tar.bz2</a> (sources)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.tar.bz2">pypy2-v5.4.1-src.tar.bz2</a> (sources)</li>
 </ul>
 <p>Or you can checkout the current trunk using <a class="reference external" href="https://www.mercurial-scm.org/">Mercurial</a> (the trunk
 usually works and is of course more up-to-date):</p>
@@ -403,20 +404,20 @@
 <div class="section" id="checksums">
 <h1>Checksums</h1>
 <p>Here are the checksums for each of the downloads</p>
-<p>pypy2.7-v5.4.0 md5:</p>
+<p>pypy2.7-v5.4.1 md5:</p>
 <pre class="literal-block">
-50ea504e66f4d9297f5228d7a3b026ec  pypy2-v5.4.0-linux-armel.tar.bz2
-e838ba554bc53c793f23c378a898fa0f  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
-b1b9b755631ef85d400d7690ece50210  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
-df7180d5070ac19a234fc6c39b88f420  pypy2-v5.4.0-linux32.tar.bz2
-5e228ba05b6eaa0af37321fd3f425891  pypy2-v5.4.0-linux64.tar.bz2
-b32d4c97275901665945f1f2813b6f26  pypy2-v5.4.0-osx64.tar.bz2
-1d32ef8036a9fe718f397813bd070be8  pypy2-v5.4.0-ppc64.tar.bz2
-d8abb09416b4370ea40c51a710d12b18  pypy2-v5.4.0-ppc64le.tar.bz2
-b560c2811a3089f22b21db9beea7f273  pypy2-v5.4.0-s390x.tar.bz2
-c806bea7ecbb999fffeea3a06e6462e8  pypy2-v5.4.0-src.tar.bz2
-26c2ab1c891651eb620dbde499088c1f  pypy2-v5.4.0-src.zip
-bd25b15c0d6c0f7c7f6fa75f1da35014  pypy2-v5.4.0-win32.zip
+425ffedf0db4dd737d450aa064ae0e7a  pypy2-v5.4.1-linux-armel.tar.bz2
+15f41409cbadbde3ef22ee60ded8579a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+5940ea0c1077e3bc2ba461bf55800abb  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+23cbad540abff48ea67f5b75368344be  pypy2-v5.4.1-linux32.tar.bz2
+2f4a82ae306f3d30d486fb945ca9c6be  pypy2-v5.4.1-linux64.tar.bz2
+a18cbd990c97f75a08235a35e546a637  pypy2-v5.4.1-osx64.tar.bz2
+90b0650dda1b5bf33f6119f20ba9c3b6  pypy2-v5.4.1-ppc64.tar.bz2
+9a2c1b3124151b79e68f36eb2d4891d1  pypy2-v5.4.1-ppc64le.tar.bz2
+826d4b48e43c84a50dc7e2adc2cb69d4  pypy2-v5.4.1-s390x.tar.bz2
+d1d197d16331aa23a7fd4b5d4c3c1717  pypy2-v5.4.1-src.tar.bz2
+1aab9fe6e7c03e959cde466819034bab  pypy2-v5.4.1-src.zip
+b04aad943aac92862a73b1fd90157a00  pypy2-v5.4.1-win32.zip
 </pre>
 <p>pypy3.3-v5.2-alpha md5:</p>
 <pre class="literal-block">
@@ -435,20 +436,20 @@
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 </pre>
-<p>pypy2.7-5.4.0 sha1:</p>
+<p>pypy2.7-5.4.1 sha1:</p>
 <pre class="literal-block">
-c50062a83e4bb9fc59b76901c92e7bf1ecd0351f  pypy2-v5.4.0-linux-armel.tar.bz2
-f4ebad7a9a31dfa55b35cc01b0533ef8e31ab7c4  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
-c0becdcb7f44e09947afab9df759313ec94563ef  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
-63be7254bdecd4f3272bcc47f0da7f5db82435a0  pypy2-v5.4.0-linux32.tar.bz2
-b0e0405ca8f3b143e16122767eb5605d3388af0c  pypy2-v5.4.0-linux64.tar.bz2
-9c97f54d492886fcaae8611733bcc40a625c8245  pypy2-v5.4.0-osx64.tar.bz2
-4a263167bbc89447e5adc2ed687ed44798bbca08  pypy2-v5.4.0-ppc64.tar.bz2
-b3554db74a826fd8e86f1132e9c2cb2e49caac1c  pypy2-v5.4.0-ppc64le.tar.bz2
-165920a2d0eeda83e8808e7fce93f2a9db7f736a  pypy2-v5.4.0-s390x.tar.bz2
-95163f8f3c8e9e52e126fc1807d8d94e3d224aec  pypy2-v5.4.0-src.tar.bz2
-b26546821836cb4bfda0160d37d4dd31fd3aace8  pypy2-v5.4.0-src.zip
-5ec0ca235cc68b557770b8cf5e1e49bd7b1a0aad  pypy2-v5.4.0-win32.zip
+a54b2a8b6def85663b10fd956d51fbd052954b83  pypy2-v5.4.1-linux-armel.tar.bz2
+61c9a5269d6d414c4a1d7c41bbc6a45da318f138  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+ea48ffe40887e25adcf1969a6b0e25dbe42a2457  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+bf0ac4668323abead04dd0948a2e541a158e4a00  pypy2-v5.4.1-linux32.tar.bz2
+d16601b04987381c4922a19dacb0ca2591a3b566  pypy2-v5.4.1-linux64.tar.bz2
+4084b7db8ee16a52e7ecb89d10765d961864c67c  pypy2-v5.4.1-osx64.tar.bz2
+98eac6f32f4fe8fce6eb0f0594163f3e01dccced  pypy2-v5.4.1-ppc64.tar.bz2
+fc676af00b356ae48602b531675774f2badfc4fd  pypy2-v5.4.1-ppc64le.tar.bz2
+70b736f1fdb92ae9916630b7cc8954ed92490f64  pypy2-v5.4.1-s390x.tar.bz2
+1143948f50b1b95b55465b246639b4c48251e38e  pypy2-v5.4.1-src.tar.bz2
+a6a9ce0defb401d8777d8af5949d23393416a390  pypy2-v5.4.1-src.zip
+49c2ad75f1531730f7ee466d833d318333915ce0  pypy2-v5.4.1-win32.zip
 </pre>
 <p>pypy3.3-v5.2-alpha sha1:</p>
 <pre class="literal-block">
@@ -462,20 +463,20 @@
 4b31ab492716ea375dd090bbacdf3d7c2d483059  pypy3.3-v5.2.0-alpha1-src.tar.bz2
 d9f5b64f144ebec1a200156809fbbe04fdf7eb7e  pypy3.3-v5.2.0-alpha1-src.zip
 </pre>
-<p>pypy2.7-5.4.0 sha256:</p>
+<p>pypy2.7-5.4.1 sha256:</p>
 <pre class="literal-block">
-04509044f21bb41ee6d3fafcf637fc0c586c248d4cdae6ac3357606a7b660fdb  pypy2-v5.4.0-linux-armel.tar.bz2
-95c690bcae6771ebce6cf06c7c2842e0662e007e35162afc963337aa597b471a  pypy2-v5.4.0-linux-armhf-raring.tar.bz2
-839b08db89b7e20cb670b8cf02596e033ea0b76fb8336af7bedfbb04b6b502da  pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
-ce581270464b14cdecd13dedb9bd7bf98232f767ac4ac282229a405d8e807af1  pypy2-v5.4.0-linux32.tar.bz2
-bdfea513d59dcd580970cb6f79f3a250d00191fd46b68133d5327e924ca845f8  pypy2-v5.4.0-linux64.tar.bz2
-3adf21c2bf3432759c99123f21240d71a72aba81d73129e48ef912c34631b723  pypy2-v5.4.0-osx64.tar.bz2
-dc09a057264dafb7e4bceca57b6a6ba3b0a5273e125a9b29da32b8439f980270  pypy2-v5.4.0-ppc64.tar.bz2
-4feb0711e7c235b247f8ea0b22e8a676f89e8831488b7a4e9c7f3a6943d07052  pypy2-v5.4.0-ppc64le.tar.bz2
-6bceb2760b1c7d6105d20207102862160ddddfd9b1a2707b3a8d866ac29e08d3  pypy2-v5.4.0-s390x.tar.bz2
-d9568ebe9a14d0eaefde887d78f3cba63d665e95c0d234bb583932341f55a655  pypy2-v5.4.0-src.tar.bz2
-3c165676be8df3b482727438836a9a240ea641392ddd60593f825e1d50029022  pypy2-v5.4.0-src.zip
-442c0a917781b6155bf78d2648f1ccd9a36c321926a043f83efcea22a99960b4  pypy2-v5.4.0-win32.zip
+a1eb5f672aae62606176305e52a51b060ba974b6181ebefcd2c555ecf5f8614f  pypy2-v5.4.1-linux-armel.tar.bz2
+2c4befc4517adec874155a8b6fa0b9d18388943d4ffe778002072db7783e417a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+b38646519ee1a888c68f8f4713c122867b4b36693c8acabb38eb827a9d2d51f9  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+6d1e2386ec1e05dffed493aa2d5e6db5cf5de18d7350d44b85f2e45aa5c9a774  pypy2-v5.4.1-linux32.tar.bz2
+9c85319778224d7fb0c348f55fe3fada15bb579c5f3870a13ad63b42a737dd72  pypy2-v5.4.1-linux64.tar.bz2
+ae9329c8f0a6df431c6224c27c634f998688ac803e8d100cee9a774e6bba38b5  pypy2-v5.4.1-osx64.tar.bz2
+ff118f2dc2393c821595fa7872e4d7bdae2a9a16a854759e103608df383d765a  pypy2-v5.4.1-ppc64.tar.bz2
+684e862cdb281a22c95d73012f7d2693be9fd9cdd171784256da1514ae1c8164  pypy2-v5.4.1-ppc64le.tar.bz2
+4004c86bf2dfdd1d92120f514d296af7602e0c5e2c6526a14ff8b5508c0fe8f7  pypy2-v5.4.1-s390x.tar.bz2
+45dbc50c81498f6f1067201b8fc887074b43b84ee32cc47f15e7db17571e9352  pypy2-v5.4.1-src.tar.bz2
+54b23c11a92dd6328a58787c3d719189f0aa24c872b479acf50094dfb4be0a46  pypy2-v5.4.1-src.zip
+ec729218a820bc2aa2cf1fcacf9de0fee9e04144fe138596198a6b4615505e03  pypy2-v5.4.1-win32.zip
 </pre>
 <p>pypy3.3-v5.2-alpha sha256:</p>
 <pre class="literal-block">
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,12 +14,14 @@
 
 We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release &#8212; **PyPy2.7 v5.4.0** &#8212; (`what's new in PyPy2.7?`_ )
+* the Python2.7 compatible release &#8212; **PyPy2.7 v5.4.1** &#8212; (`what's new in PyPy2.7?`_
+  and `bugfix release`_)
 * the Python3.3 compatible release &#8212; **PyPy3.3 v5.2-alpha** &#8212; (`what's new in PyPy3.3?`_).
 
 * the Python2.7 Software Transactional Memory special release &#8212; **PyPy-STM 2.5.1** (Linux x86-64 only)
 
 .. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.0.html
+.. _bugfix release: http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.1.html
 .. _what's new in PyPy3.3?: http://doc.pypy.org/en/latest/release-pypy3.3-v5.2-alpha1.html
 
 
@@ -75,7 +77,7 @@
 .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
 
 
-Python2.7 compatible PyPy 5.4.0
+Python2.7 compatible PyPy 5.4.1
 -----------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -94,18 +96,18 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux32.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64le.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
@@ -205,7 +207,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.4.0/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.4.1/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -291,9 +293,9 @@
 1. Get the source code.  The following packages contain the source at
    the same revision as the above binaries:
 
-   * `pypy2-v5.4.0-src.tar.bz2`__ (sources)
+   * `pypy2-v5.4.1-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.0-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.tar.bz2
 
    Or you can checkout the current trunk using Mercurial_ (the trunk
    usually works and is of course more up-to-date)::

From pypy.commits at gmail.com  Wed Sep  7 03:33:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 00:33:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: os.access() and float arguments is a
 translation crash with sandboxing
Message-ID: <57cfc2c5.04e21c0a.5fe1.d1f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86918:2f7389e85d58
Date: 2016-09-07 09:32 +0200
http://bitbucket.org/pypy/pypy/changeset/2f7389e85d58/

Log:	os.access() and float arguments is a translation crash with
	sandboxing

diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
--- a/pypy/doc/release-pypy2.7-v5.4.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -16,7 +16,7 @@
 
   * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
 
-  * Fix for an issue where os.access() accepted a float for mode
+  * Fix for an issue for translating the sandbox
 
   * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
     the last byte of a unicode string sometimes

From pypy.commits at gmail.com  Wed Sep  7 04:01:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 01:01:09 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: A branch to implement
 rawrefcount for Boehm (needed by revdb)
Message-ID: <57cfc945.c19d1c0a.89e5a.da1f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86919:97b95319d48d
Date: 2016-09-07 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/97b95319d48d/

Log:	A branch to implement rawrefcount for Boehm (needed by revdb)


From pypy.commits at gmail.com  Wed Sep  7 05:00:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 02:00:48 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: Add missing operation
Message-ID: <57cfd740.4177c20a.ff41e.3f1a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86920:8910c2dbe6a0
Date: 2016-09-07 10:01 +0200
http://bitbucket.org/pypy/pypy/changeset/8910c2dbe6a0/

Log:	Add missing operation

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -491,6 +491,7 @@
     'gc_rawrefcount_create_link_pyobj': LLOp(),
     'gc_rawrefcount_from_obj':          LLOp(sideeffects=False),
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
+    'gc_rawrefcount_next_dead':         LLOp(),
 
     # ------- JIT & GC interaction, only for some GCs ----------
 

From pypy.commits at gmail.com  Wed Sep  7 05:00:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 02:00:50 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: Import the C code from
 https://bitbucket.org/arigo/arigo/src/default/hack/pypy-hack/boehm-rawrefcount/
Message-ID: <57cfd742.88711c0a.4e105.fab3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86921:5706f541d3da
Date: 2016-09-07 11:00 +0200
http://bitbucket.org/pypy/pypy/changeset/5706f541d3da/

Log:	Import the C code from
	https://bitbucket.org/arigo/arigo/src/default/hack/pypy-hack/boehm-
	rawrefcount/

diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -0,0 +1,258 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <assert.h>
+#include <limits.h>
+#include <gc/gc.h>
+#include <gc/gc_mark.h>
+
+
+#define REFCNT_FROM_PYPY  (LONG_MAX / 4 + 1)
+
+typedef struct pypy_header0 gcobj_t;    /* opaque here */
+
+#ifndef _WIN32
+typedef intptr_t Py_ssize_t;
+#else
+typedef long Py_ssize_t;
+#endif
+
+/* this is the first two words of the PyObject structure used in
+   pypy/module/cpyext */
+typedef struct {
+    Py_ssize_t ob_refcnt;
+    Py_ssize_t ob_pypy_link;
+} pyobj_t;
+
+struct link_s {
+    pyobj_t *pyobj;    /* NULL if entry unused */
+    uintptr_t gcenc;
+    struct link_s *next_in_bucket;
+};
+
+#define MARKER_LIST_START  ((pyobj_t *)-1)
+
+static struct link_s **hash_buckets, *hash_list, *hash_free_list;
+static uintptr_t hash_mask_bucket;
+static intptr_t hash_list_walk_next = -1;
+
+static uintptr_t hash_get_hash(gcobj_t *gcobj)
+{
+    assert(gcobj != NULL);
+    uintptr_t h = (uintptr_t)gcobj;
+    assert((h & 1) == 0);
+    h -= (h >> 6);
+    return h & hash_mask_bucket;
+}
+
+static gcobj_t *decode_gcenc(uintptr_t gcenc)
+{
+    if (gcenc & 1)
+        gcenc = ~gcenc;
+    return (gcobj_t *)gcenc;
+}
+
+static void hash_link(struct link_s *lnk)
+{
+    uintptr_t h = hash_get_hash(decode_gcenc(lnk->gcenc));
+    lnk->next_in_bucket = hash_buckets[h];
+    hash_buckets[h] = lnk;
+}
+
+static void boehm_is_about_to_collect(void);
+
+static void hash_grow_table(void)
+{
+    static int rec = 0;
+    assert(!rec);   /* recursive hash_grow_table() */
+    rec = 1;
+
+    if (hash_buckets == NULL)
+        GC_set_start_callback(boehm_is_about_to_collect);
+
+    uintptr_t i, num_buckets = (hash_mask_bucket + 1) * 2;
+    if (num_buckets < 16) num_buckets = 16;
+    assert((num_buckets & (num_buckets - 1)) == 0);  /* power of two */
+
+    /* The new hash_buckets: an array of pointers to struct link_s, of
+       length a power of two, used as a dictionary hash table.  It is
+       not allocated with Boehm because there is no point in Boehm looking
+       in it.
+     */
+    struct link_s **new_buckets = calloc(num_buckets, sizeof(struct link_s *));
+    assert(new_buckets);
+
+    /* The new hash_list: the array of all struct link_s.  Their order
+       is irrelevant.  There is a GC_register_finalizer() on the 'gcenc'
+       field, so we don't move the array; instead we allocate a new array
+       to use in addition to the old one.  There are a total of 2 to 4
+       times as many 'struct link_s' as the length of 'buckets'.
+     */
+    uintptr_t num_list = num_buckets * 2;
+    struct link_s *new_list = GC_MALLOC(num_list * sizeof(struct link_s));
+    for (i = num_list; i-- > 1; ) {
+        new_list[i].next_in_bucket = hash_free_list;
+        hash_free_list = &new_list[i];
+    }
+    /* list[0] is abused to store a pointer to the previous list and
+       the length of the current list */
+    struct link_s *old_list = hash_list;
+    new_list[0].next_in_bucket = old_list;
+    new_list[0].gcenc = num_list;
+    new_list[0].pyobj = MARKER_LIST_START;
+
+    hash_list = new_list;
+    free(hash_buckets);
+    hash_buckets = new_buckets;
+    hash_mask_bucket = num_buckets - 1;
+    hash_list_walk_next = hash_mask_bucket;
+
+    /* re-add all old 'struct link_s' to the hash_buckets */
+    struct link_s *plist = old_list;
+    while (plist != NULL) {
+        uintptr_t count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc != 0)
+                hash_link(&plist[i]);
+        }
+        plist = plist[0].next_in_bucket;
+    }
+    GC_reachable_here(old_list);
+
+    rec = 0;
+}
+
+static void hash_add_entry(gcobj_t *gcobj, pyobj_t *pyobj)
+{
+    if (hash_free_list == NULL) {
+        hash_grow_table();
+    }
+    assert(pyobj->ob_pypy_link == 0);
+
+    struct link_s *lnk = hash_free_list;
+    hash_free_list = lnk->next_in_bucket;
+    lnk->pyobj = pyobj;
+    lnk->gcenc = (uintptr_t)gcobj;
+    pyobj->ob_pypy_link = (Py_ssize_t)lnk;
+
+    hash_link(lnk);
+
+    int j = GC_general_register_disappearing_link((void **)&lnk->gcenc, gcobj);
+    assert(j == GC_SUCCESS);
+}
+
+static pyobj_t *hash_get_entry(gcobj_t *gcobj)
+{
+    if (hash_buckets == NULL)
+        return NULL;
+    uintptr_t h = hash_get_hash(gcobj);
+    struct link_s *lnk = hash_buckets[h];
+    while (lnk != NULL) {
+        assert(lnk->pyobj != NULL);
+        if (decode_gcenc(lnk->gcenc) == gcobj)
+            return lnk->pyobj;
+        lnk = lnk->next_in_bucket;
+    }
+    return NULL;
+}
+
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_next_dead(void)
+{
+    while (hash_list_walk_next >= 0) {
+        struct link_s *p, **pp = &hash_buckets[hash_list_walk_next];
+        while (1) {
+            p = *pp;
+            if (p == NULL)
+                break;
+            assert(p->pyobj != NULL);
+            if (p->gcenc == 0) {
+                /* quadratic time on the number of links from the same
+                   bucket chain, but it should be small with very high
+                   probability */
+                pyobj_t *result = p->pyobj;
+                printf("next_dead: %p\n", result);
+                assert(result->ob_refcnt == REFCNT_FROM_PYPY);
+                p->pyobj = NULL;
+                *pp = p->next_in_bucket;
+                p->next_in_bucket = hash_free_list;
+                hash_free_list = p;
+                return result;
+            }
+            else {
+                assert(p->gcenc != ~(uintptr_t)0);
+                pp = &p->next_in_bucket;
+            }
+        }
+        hash_list_walk_next--;
+    }
+    return NULL;
+}
+
+RPY_EXTERN
+void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                     /*pyobj_t*/void *pyobj)
+{
+    gcobj_t *gcobj1 = (gcobj_t *)gcobj;
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    assert(pyobj1->ob_pypy_link == 0);
+    assert(pyobj1->ob_refcnt >= REFCNT_FROM_PYPY);
+
+    hash_add_entry(gcobj1, pyobj1);
+}
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj)
+{
+    return hash_get_entry((gcobj_t *)gcobj);
+}
+
+RPY_EXTERN
+/*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj)
+{
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    if (pyobj1->ob_pypy_link == 0)
+        return NULL;
+
+    struct link_s *lnk = (struct link_s *)pyobj1->ob_pypy_link;
+    assert(lnk->pyobj == pyobj1);
+    
+    gcobj_t *g = decode_gcenc(lnk->gcenc);
+    assert(g != NULL);
+    return g;
+}
+
+static void boehm_is_about_to_collect(void)
+{
+    struct link_s *plist = hash_list;
+    while (plist != NULL) {
+        uintptr_t i, count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc == 0)
+                continue;
+
+            pyobj_t *p = plist[i].pyobj;
+            assert(p != NULL);
+            assert(p->ob_refcnt >= REFCNT_FROM_PYPY);
+
+            printf("plist[%d].gcenc: %p ", (int)i, plist[i].gcenc);
+
+            if ((plist[i].gcenc & 1) ^ (p->ob_refcnt == REFCNT_FROM_PYPY)) {
+                /* ob_refcnt > FROM_PYPY: non-zero regular refcnt, 
+                   the gc obj must stay alive.  decode gcenc.
+                   ---OR---
+                   ob_refcnt == FROM_PYPY: no refs from C code, the
+                   gc obj must not (necessarily) stay alive.  encode gcenc.
+                */
+                plist[i].gcenc = ~plist[i].gcenc;
+            }
+            printf("-> %p\n", plist[i].gcenc);
+        }
+        plist = plist[0].next_in_bucket;
+    }
+    if (hash_mask_bucket > 0)
+        hash_list_walk_next = hash_mask_bucket;
+}
diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -0,0 +1,22 @@
+#include "common_header.h"
+
+#define OP_GC_RAWREFCOUNT_INIT(callback, r)   /* nothing */
+
+#define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r)   \
+    gc_rawrefcount_create_link_pypy(gcobj, pyobj)
+
+#define OP_GC_RAWREFCOUNT_FROM_OBJ(gcobj, r)   \
+    r = gc_rawrefcount_from_obj(gcobj)
+
+#define OP_GC_RAWREFCOUNT_TO_OBJ(pyobj, r)   \
+    r = gc_rawrefcount_to_obj(pyobj)
+
+#define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
+    r = gc_rawrefcount_next()
+
+
+RPY_EXTERN void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                                /*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj);
+RPY_EXTERN /*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_next_dead(void);
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -0,0 +1,230 @@
+import itertools, os, subprocess
+from hypothesis import given, strategies
+from rpython.tool.udir import udir
+
+
+TEST_CODE = r"""
+#define RPY_EXTERN  /* nothing */
+#include "boehm-rawrefcount.c"
+
+static gcobj_t *alloc_gcobj(void)   /* for tests */
+{
+    gcobj_t *g = GC_MALLOC(1000);
+    printf("gc obj: %p\n", g);
+    return g;
+}
+
+static pyobj_t *alloc_pyobj(void)   /* for tests */
+{
+    pyobj_t *p = malloc(1000);
+    p->ob_refcnt = 1;
+    p->ob_pypy_link = 0;
+    printf("py obj: %p\n", p);
+    return p;
+}
+
+static void decref(pyobj_t *p)      /* for tests */
+{
+    p->ob_refcnt--;
+    if (p->ob_refcnt == 0) {
+        printf("decref to zero: %p\n", p);
+        free(p);
+    }
+    assert(p->ob_refcnt >= REFCNT_FROM_PYPY ||
+           p->ob_refcnt < REFCNT_FROM_PYPY * 0.99);
+}
+
+void run_test(void);     /* forward declaration, produced by the test */
+
+int main(void)
+{
+    run_test();
+    while (gc_rawrefcount_next_dead() != NULL)
+        ;
+    return 0;
+}
+"""
+
+
+operations = strategies.sampled_from([
+    'new_pyobj',
+    'new_gcobj',
+    'create_link',
+    'from_obj',
+    'to_obj',
+    'forget_pyobj',
+    'forget_gcobj',
+    'collect',
+    'dead',
+    ])
+
+
+ at strategies.composite
+def make_code(draw):
+    code = []
+    pyobjs = []
+    gcobjs = []
+    num_gcobj = itertools.count()
+    num_pyobj = itertools.count()
+    links_g2p = {}
+    links_p2g = {}
+
+    def new_gcobj():
+        varname = 'g%d' % next(num_gcobj)
+        code.append('gcobj_t *volatile %s = alloc_gcobj();' % varname)
+        gcobjs.append(varname)
+        return varname
+
+    def new_pyobj():
+        varname = 'p%d' % next(num_pyobj)
+        code.append('pyobj_t *%s = alloc_pyobj();' % varname)
+        pyobjs.append(varname)
+        return varname
+
+    for op in draw(strategies.lists(operations, average_size=250)):
+        if op == 'new_gcobj':
+            new_gcobj()
+        elif op == 'new_pyobj':
+            new_pyobj()
+        elif op == 'create_link':
+            gvars = [varname for varname in gcobjs if varname not in links_g2p]
+            if gvars == []:
+                gvars.append(new_gcobj())
+            pvars = [varname for varname in pyobjs if varname not in links_p2g]
+            if pvars == []:
+                pvars.append(new_pyobj())
+            gvar = draw(strategies.sampled_from(gvars))
+            pvar = draw(strategies.sampled_from(pvars))
+            code.append(r'printf("create_link %%p-%%p\n", %s, %s); '
+                            % (gvar, pvar) +
+                        "%s->ob_refcnt += REFCNT_FROM_PYPY; " % pvar +
+                        "gc_rawrefcount_create_link_pypy(%s, %s);"
+                            % (gvar, pvar))
+            links_g2p[gvar] = pvar
+            links_p2g[pvar] = gvar
+        elif op == 'from_obj':
+            if gcobjs:
+                prnt = False
+                gvar = draw(strategies.sampled_from(gcobjs))
+                if gvar not in links_g2p:
+                    check = "== NULL"
+                elif links_g2p[gvar] in pyobjs:
+                    check = "== %s" % (links_g2p[gvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_from_obj(%s) %s);"
+                            % (gvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", %s, '
+                        'gc_rawrefcount_from_obj(%s));' % (gvar, gvar))
+        elif op == 'to_obj':
+            if pyobjs:
+                prnt = False
+                pvar = draw(strategies.sampled_from(pyobjs))
+                if pvar not in links_p2g:
+                    check = "== NULL"
+                elif links_p2g[pvar] in gcobjs:
+                    check = "== %s" % (links_p2g[pvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_to_obj(%s) %s);"
+                            % (pvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", '
+                        'gc_rawrefcount_to_obj(%s), %s);' % (pvar, pvar))
+        elif op == 'forget_pyobj':
+            if pyobjs:
+                index = draw(strategies.sampled_from(range(len(pyobjs))))
+                pvar = pyobjs.pop(index)
+                code.append(r'printf("-p%%p\n", %s); ' % pvar +
+                            "decref(%s); %s = NULL;" % (pvar, pvar))
+        elif op == 'forget_gcobj':
+            if gcobjs:
+                index = draw(strategies.sampled_from(range(len(gcobjs))))
+                gvar = gcobjs.pop(index)
+                code.append(r'printf("-g%%p\n", %s); ' % gvar +
+                            "%s = NULL;" % (gvar,))
+        elif op == 'collect':
+            code.append("GC_gcollect();")
+        elif op == 'dead':
+            code.append('gc_rawrefcount_next_dead();')
+        else:
+            assert False, op
+
+    return '\n'.join(code)
+
+
+ at given(make_code())
+def test_random(code):
+    filename = str(udir.join("test-rawrefcount-boehm.c"))
+    with open(filename, "w") as f:
+        print >> f, TEST_CODE
+        print >> f, 'void run_test(void) {'
+        print >> f, code
+        print >> f, '}'
+
+    srcdir = os.path.dirname(os.path.dirname(
+        os.path.abspath(os.path.join(__file__))))
+    srcdir = os.path.join(srcdir, 'src')
+
+    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o test-rawrefcount-boehm"
+                    " test-rawrefcount-boehm.c" % (udir, srcdir))
+    assert err == 0
+    p = subprocess.Popen("./test-rawrefcount-boehm", stdout=subprocess.PIPE,
+                         cwd=str(udir))
+    stdout, _ = p.communicate()
+    assert p.wait() == 0
+
+    gcobjs = {}
+    pyobjs = {}
+    links_p2g = {}
+    links_g2p = {}
+    for line in stdout.splitlines():
+        if line.startswith('py obj: '):
+            p = line[8:]
+            assert not pyobjs.get(p)
+            pyobjs[p] = True
+            assert p not in links_p2g
+        elif line.startswith('gc obj: '):
+            g = line[8:]
+            assert not gcobjs.get(g)
+            gcobjs[g] = True
+            if g in links_g2p: del links_g2p[g]
+        elif line.startswith('-p'):
+            p = line[2:]
+            assert pyobjs[p] == True
+            pyobjs[p] = False
+        elif line.startswith('-g'):
+            g = line[2:]
+            assert gcobjs[g] == True
+            gcobjs[g] = False
+        elif line.startswith('decref to zero: '):
+            p = line[16:]
+            assert pyobjs[p] == False
+            assert p not in links_p2g
+            del pyobjs[p]
+        elif line.startswith('create_link '):
+            g, p = line[12:].split('-')
+            assert g in gcobjs
+            assert p in pyobjs
+            assert g not in links_g2p
+            assert p not in links_p2g
+            links_g2p[g] = p
+            links_p2g[p] = g
+        elif line.startswith('link '):
+            g, p = line[5:].split('-')
+            assert g in gcobjs
+            assert p in pyobjs
+            assert links_g2p[g] == p
+            assert links_p2g[p] == g
+        elif line.startswith('plist['):
+            pass
+        elif line.startswith('next_dead: '):
+            p = line[11:]
+            assert pyobjs[p] == False
+            del pyobjs[p]
+            del links_p2g[p]
+        else:
+            assert False, repr(line)

From pypy.commits at gmail.com  Wed Sep  7 05:28:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 02:28:31 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: Integration in-progress
Message-ID: <57cfddbf.0cce1c0a.30a6.fd8e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86922:a6e4b73b6de0
Date: 2016-09-07 11:27 +0200
http://bitbucket.org/pypy/pypy/changeset/a6e4b73b6de0/

Log:	Integration in-progress

diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -4,10 +4,11 @@
 #  This is meant for pypy's cpyext module, but is a generally
 #  useful interface over our GC.  XXX "pypy" should be removed here
 #
-import sys, weakref
-from rpython.rtyper.lltypesystem import lltype, llmemory
+import sys, weakref, py
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib import rgc
 
 
@@ -229,6 +230,11 @@
         v_p, v_ob = hop.inputargs(*hop.args_r)
         hop.exception_cannot_occur()
         hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
+        #
+        if hop.rtyper.annotator.translator.config.translation.gc == "boehm":
+            c_func = hop.inputconst(lltype.typeOf(func_boehm_eci),
+                                    func_boehm_eci)
+            hop.genop('direct_call', [c_func])
 
 
 class Entry(ExtRegistryEntry):
@@ -281,3 +287,10 @@
         v_ob = hop.genop('gc_rawrefcount_next_dead', [],
                          resulttype = llmemory.Address)
         return _spec_ob(hop, v_ob)
+
+src_dir = py.path.local(__file__).dirpath() / 'src'
+boehm_eci = ExternalCompilationInfo(
+    post_include_bits     = [(src_dir / 'boehm-rawrefcount.h').read()],
+    separate_module_files = [(src_dir / 'boehm-rawrefcount.c')],
+)
+func_boehm_eci = rffi.llexternal_use_eci(boehm_eci)
diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
--- a/rpython/rlib/src/boehm-rawrefcount.c
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -6,6 +6,12 @@
 #include <gc/gc.h>
 #include <gc/gc_mark.h>
 
+#ifdef TEST_BOEHM_RAWREFCOUNT
+#  define RPY_EXTERN  /* nothing */
+#else
+#  include "common_header.h"
+#endif
+
 
 #define REFCNT_FROM_PYPY  (LONG_MAX / 4 + 1)
 
@@ -172,7 +178,9 @@
                    bucket chain, but it should be small with very high
                    probability */
                 pyobj_t *result = p->pyobj;
+#ifdef TEST_BOEHM_RAWREFCOUNT
                 printf("next_dead: %p\n", result);
+#endif
                 assert(result->ob_refcnt == REFCNT_FROM_PYPY);
                 p->pyobj = NULL;
                 *pp = p->next_in_bucket;
@@ -238,7 +246,9 @@
             assert(p != NULL);
             assert(p->ob_refcnt >= REFCNT_FROM_PYPY);
 
+#ifdef TEST_BOEHM_RAWREFCOUNT
             printf("plist[%d].gcenc: %p ", (int)i, plist[i].gcenc);
+#endif
 
             if ((plist[i].gcenc & 1) ^ (p->ob_refcnt == REFCNT_FROM_PYPY)) {
                 /* ob_refcnt > FROM_PYPY: non-zero regular refcnt, 
@@ -249,8 +259,10 @@
                 */
                 plist[i].gcenc = ~plist[i].gcenc;
             }
+#ifdef TEST_BOEHM_RAWREFCOUNT
             printf("-> %p\n", plist[i].gcenc);
-        }
+#endif
+    }
         plist = plist[0].next_in_bucket;
     }
     if (hash_mask_bucket > 0)
diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
--- a/rpython/rlib/src/boehm-rawrefcount.h
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -1,6 +1,8 @@
-#include "common_header.h"
 
-#define OP_GC_RAWREFCOUNT_INIT(callback, r)   /* nothing */
+/* Missing:
+   OP_GC_RAWREFCOUNT_INIT(callback, r): the callback is not supported here
+   OP_GC_RAWREFCOUNT_CREATE_LINK_PYOBJ(): not implemented, maybe not needed
+*/
 
 #define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r)   \
     gc_rawrefcount_create_link_pypy(gcobj, pyobj)
@@ -12,7 +14,7 @@
     r = gc_rawrefcount_to_obj(pyobj)
 
 #define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
-    r = gc_rawrefcount_next()
+    r = gc_rawrefcount_next_dead()
 
 
 RPY_EXTERN void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -214,6 +214,7 @@
 
 
 class TestTranslated(StandaloneTests):
+    _GC = 'incminimark'
 
     def test_full_translation(self):
         class State:
@@ -226,29 +227,37 @@
         def make_p():
             p = W_Root(42)
             ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+            ob.c_ob_refcnt += REFCNT_FROM_PYPY
             rawrefcount.create_link_pypy(p, ob)
-            ob.c_ob_refcnt += REFCNT_FROM_PYPY
             assert rawrefcount.from_obj(PyObject, p) == ob
             assert rawrefcount.to_obj(W_Root, ob) == p
             return ob, p
 
         FTYPE = rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER
+        has_callback = (self._GC != "boehm")
 
         def entry_point(argv):
-            ll_dealloc_trigger_callback = llhelper(FTYPE, dealloc_trigger)
-            rawrefcount.init(ll_dealloc_trigger_callback)
+            if has_callback:
+                ll_dealloc_trigger_callback = llhelper(FTYPE, dealloc_trigger)
+                rawrefcount.init(ll_dealloc_trigger_callback)
             ob, p = make_p()
             if state.seen != []:
                 print "OB COLLECTED REALLY TOO SOON"
                 return 1
+            if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
+                print "got a next_dead() really too soon"
+                return 1
             rgc.collect()
             if state.seen != []:
                 print "OB COLLECTED TOO SOON"
                 return 1
+            if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
+                print "got a next_dead() too soon"
+                return 1
             objectmodel.keepalive_until_here(p)
             p = None
             rgc.collect()
-            if state.seen != [1]:
+            if has_callback and state.seen != [1]:
                 print "OB NOT COLLECTED"
                 return 1
             if rawrefcount.next_dead(PyObject) != ob:
@@ -262,7 +271,11 @@
             return 0
 
         self.config = get_combined_translation_config(translating=True)
-        self.config.translation.gc = "incminimark"
+        self.config.translation.gc = self._GC
         t, cbuilder = self.compile(entry_point)
         data = cbuilder.cmdexec('hi there')
         assert data.startswith('OK!\n')
+
+
+class TestBoehm(TestTranslated):
+    _GC = "boehm"
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
--- a/rpython/rlib/test/test_rawrefcount_boehm.py
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -4,7 +4,7 @@
 
 
 TEST_CODE = r"""
-#define RPY_EXTERN  /* nothing */
+#define TEST_BOEHM_RAWREFCOUNT
 #include "boehm-rawrefcount.c"
 
 static gcobj_t *alloc_gcobj(void)   /* for tests */

From pypy.commits at gmail.com  Wed Sep  7 05:43:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 02:43:20 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: Stop trying to use the same
 test, and wrote a boehm-specific test
Message-ID: <57cfe138.ec02c20a.bd1d.87bc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86923:c8bdf1398ada
Date: 2016-09-07 11:42 +0200
http://bitbucket.org/pypy/pypy/changeset/c8bdf1398ada/

Log:	Stop trying to use the same test, and wrote a boehm-specific test

diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
--- a/rpython/rlib/src/boehm-rawrefcount.c
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -206,7 +206,8 @@
     pyobj_t *pyobj1 = (pyobj_t *)pyobj;
 
     assert(pyobj1->ob_pypy_link == 0);
-    assert(pyobj1->ob_refcnt >= REFCNT_FROM_PYPY);
+    /*assert(pyobj1->ob_refcnt >= REFCNT_FROM_PYPY);*/
+    /*^^^ could also be fixed just after the call to create_link_pypy()*/
 
     hash_add_entry(gcobj1, pyobj1);
 }
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -214,7 +214,6 @@
 
 
 class TestTranslated(StandaloneTests):
-    _GC = 'incminimark'
 
     def test_full_translation(self):
         class State:
@@ -227,37 +226,29 @@
         def make_p():
             p = W_Root(42)
             ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+            rawrefcount.create_link_pypy(p, ob)
             ob.c_ob_refcnt += REFCNT_FROM_PYPY
-            rawrefcount.create_link_pypy(p, ob)
             assert rawrefcount.from_obj(PyObject, p) == ob
             assert rawrefcount.to_obj(W_Root, ob) == p
             return ob, p
 
         FTYPE = rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER
-        has_callback = (self._GC != "boehm")
 
         def entry_point(argv):
-            if has_callback:
-                ll_dealloc_trigger_callback = llhelper(FTYPE, dealloc_trigger)
-                rawrefcount.init(ll_dealloc_trigger_callback)
+            ll_dealloc_trigger_callback = llhelper(FTYPE, dealloc_trigger)
+            rawrefcount.init(ll_dealloc_trigger_callback)
             ob, p = make_p()
             if state.seen != []:
                 print "OB COLLECTED REALLY TOO SOON"
                 return 1
-            if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
-                print "got a next_dead() really too soon"
-                return 1
             rgc.collect()
             if state.seen != []:
                 print "OB COLLECTED TOO SOON"
                 return 1
-            if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
-                print "got a next_dead() too soon"
-                return 1
             objectmodel.keepalive_until_here(p)
             p = None
             rgc.collect()
-            if has_callback and state.seen != [1]:
+            if state.seen != [1]:
                 print "OB NOT COLLECTED"
                 return 1
             if rawrefcount.next_dead(PyObject) != ob:
@@ -271,11 +262,50 @@
             return 0
 
         self.config = get_combined_translation_config(translating=True)
-        self.config.translation.gc = self._GC
+        self.config.translation.gc = "incminimark"
         t, cbuilder = self.compile(entry_point)
         data = cbuilder.cmdexec('hi there')
         assert data.startswith('OK!\n')
 
 
-class TestBoehm(TestTranslated):
-    _GC = "boehm"
+class TestBoehmTranslated(StandaloneTests):
+
+    def test_full_translation(self):
+
+        def make_ob():
+            p = W_Root(42)
+            ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+            rawrefcount.create_link_pypy(p, ob)
+            ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            assert rawrefcount.from_obj(PyObject, p) == ob
+            assert rawrefcount.to_obj(W_Root, ob) == p
+            return ob
+
+        def entry_point(argv):
+            oblist = [make_ob() for i in range(50)]
+            rgc.collect()
+            deadlist = []
+            while True:
+                ob = rawrefcount.next_dead(PyObject)
+                if not ob: break
+                deadlist.append(ob)
+            if len(deadlist) == 0:
+                print "no dead object"
+                return 1
+            if len(deadlist) < 30:
+                print "not enough dead objects"
+                return 1
+            for ob in deadlist:
+                if ob not in oblist:
+                    print "unexpected value for dead pointer"
+                    return 1
+                oblist.remove(ob)
+            print "OK!"
+            lltype.free(ob, flavor='raw')
+            return 0
+
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.gc = "boehm"
+        t, cbuilder = self.compile(entry_point)
+        data = cbuilder.cmdexec('hi there')
+        assert data.startswith('OK!\n')

From pypy.commits at gmail.com  Wed Sep  7 07:05:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 04:05:07 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: For Boehm,
 we can't pass the callback to rawrefcount.init() and we need
Message-ID: <57cff463.93841c0a.48e69.302a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86924:c4bc3be86d15
Date: 2016-09-07 13:04 +0200
http://bitbucket.org/pypy/pypy/changeset/c4bc3be86d15/

Log:	For Boehm, we can't pass the callback to rawrefcount.init() and we
	need instead to regularly check ourselves for dead objects.

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -1,7 +1,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.executioncontext import AsyncAction
+from pypy.interpreter import executioncontext
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
@@ -14,8 +14,9 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        pyobj_dealloc_action = PyObjDeallocAction(space)
-        self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        if space.config.translation.gc != "boehm":
+            pyobj_dealloc_action = PyObjDeallocAction(space)
+            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -67,6 +68,11 @@
             state.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
+            #
+            if self.space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(self.space)
+                self.space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,8 +90,10 @@
         from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
-            rawrefcount.init(llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
-                                      self.dealloc_trigger))
+            if space.config.translation.gc != "boehm":
+                rawrefcount.init(
+                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
+                    self.dealloc_trigger))
             init_static_data_translated(space)
 
         setup_new_method_def(space)
@@ -143,15 +151,23 @@
         self.extensions[path] = w_copy
 
 
-class PyObjDeallocAction(AsyncAction):
+def _rawrefcount_perform(space):
+    from pypy.module.cpyext.pyobject import PyObject, decref
+    while True:
+        py_obj = rawrefcount.next_dead(PyObject)
+        if not py_obj:
+            break
+        decref(space, py_obj)
+
+class PyObjDeallocAction(executioncontext.AsyncAction):
     """An action that invokes _Py_Dealloc() on the dying PyObjects.
     """
+    def perform(self, executioncontext, frame):
+        _rawrefcount_perform(self.space)
 
+class BoehmPyObjDeallocAction(executioncontext.PeriodicAsyncAction):
+    # This variant is used with Boehm, which doesn't have the explicit
+    # callback.  Instead we must periodically check ourselves.
     def perform(self, executioncontext, frame):
-        from pypy.module.cpyext.pyobject import PyObject, decref
-
-        while True:
-            py_obj = rawrefcount.next_dead(PyObject)
-            if not py_obj:
-                break
-            decref(self.space, py_obj)
+        if we_are_translated():
+            _rawrefcount_perform(self.space)

From pypy.commits at gmail.com  Wed Sep  7 08:23:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 05:23:29 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: tweak
Message-ID: <57d006c1.82ddc20a.52249.28d8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86925:9a75c25f25c3
Date: 2016-09-07 14:22 +0200
http://bitbucket.org/pypy/pypy/changeset/9a75c25f25c3/

Log:	tweak

diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
--- a/rpython/rlib/src/boehm-rawrefcount.c
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -237,6 +237,7 @@
 static void boehm_is_about_to_collect(void)
 {
     struct link_s *plist = hash_list;
+    uintptr_t gcenc_union = 0;
     while (plist != NULL) {
         uintptr_t i, count = plist[0].gcenc;
         for (i = 1; i < count; i++) {
@@ -260,12 +261,13 @@
                 */
                 plist[i].gcenc = ~plist[i].gcenc;
             }
+            gcenc_union |= plist[i].gcenc;
 #ifdef TEST_BOEHM_RAWREFCOUNT
             printf("-> %p\n", plist[i].gcenc);
 #endif
     }
         plist = plist[0].next_in_bucket;
     }
-    if (hash_mask_bucket > 0)
+    if (gcenc_union & 1)   /* if there is at least one item potentially dead */
         hash_list_walk_next = hash_mask_bucket;
 }

From pypy.commits at gmail.com  Wed Sep  7 08:23:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 05:23:31 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: Test and fix
Message-ID: <57d006c3.a6a5c20a.90c9b.2be8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86926:43fec2a94ecc
Date: 2016-09-07 14:22 +0200
http://bitbucket.org/pypy/pypy/changeset/43fec2a94ecc/

Log:	Test and fix

diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
--- a/rpython/rlib/src/boehm-rawrefcount.c
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -143,8 +143,16 @@
 
     hash_link(lnk);
 
-    int j = GC_general_register_disappearing_link((void **)&lnk->gcenc, gcobj);
-    assert(j == GC_SUCCESS);
+    if (GC_base(gcobj) == NULL) {
+        /* 'gcobj' is probably a prebuilt object - it makes no */
+        /* sense to register it then, and it crashes Boehm in */
+        /* quite obscure ways */
+    }
+    else {
+        int j = GC_general_register_disappearing_link(
+                                    (void **)&lnk->gcenc, gcobj);
+        assert(j == GC_SUCCESS);
+    }
 }
 
 static pyobj_t *hash_get_entry(gcobj_t *gcobj)
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -281,7 +281,13 @@
             assert rawrefcount.to_obj(W_Root, ob) == p
             return ob
 
+        prebuilt_p = W_Root(-42)
+        prebuilt_ob = lltype.malloc(PyObjectS, flavor='raw', zero=True,
+                                    immortal=True)
+
         def entry_point(argv):
+            rawrefcount.create_link_pypy(prebuilt_p, prebuilt_ob)
+            prebuilt_ob.c_ob_refcnt += REFCNT_FROM_PYPY
             oblist = [make_ob() for i in range(50)]
             rgc.collect()
             deadlist = []

From pypy.commits at gmail.com  Wed Sep  7 08:44:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 05:44:51 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: hg merge boehm-rawrefcount
Message-ID: <57d00bc3.8f081c0a.57fa7.53f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86928:ce52ed81e2ba
Date: 2016-09-07 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/ce52ed81e2ba/

Log:	hg merge boehm-rawrefcount

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -1,7 +1,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.executioncontext import AsyncAction
+from pypy.interpreter import executioncontext
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
@@ -14,8 +14,9 @@
         self.reset()
         self.programname = lltype.nullptr(rffi.CCHARP.TO)
         self.version = lltype.nullptr(rffi.CCHARP.TO)
-        pyobj_dealloc_action = PyObjDeallocAction(space)
-        self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+        if space.config.translation.gc != "boehm":
+            pyobj_dealloc_action = PyObjDeallocAction(space)
+            self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
@@ -67,6 +68,11 @@
             state.api_lib = str(api.build_bridge(self.space))
         else:
             api.setup_library(self.space)
+            #
+            if self.space.config.translation.gc == "boehm":
+                action = BoehmPyObjDeallocAction(self.space)
+                self.space.actionflag.register_periodic_action(action,
+                    use_bytecode_counter=True)
 
     def install_dll(self, eci):
         """NOT_RPYTHON
@@ -84,8 +90,10 @@
         from pypy.module.cpyext.api import init_static_data_translated
 
         if we_are_translated():
-            rawrefcount.init(llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
-                                      self.dealloc_trigger))
+            if space.config.translation.gc != "boehm":
+                rawrefcount.init(
+                    llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
+                    self.dealloc_trigger))
             init_static_data_translated(space)
 
         setup_new_method_def(space)
@@ -143,15 +151,23 @@
         self.extensions[path] = w_copy
 
 
-class PyObjDeallocAction(AsyncAction):
+def _rawrefcount_perform(space):
+    from pypy.module.cpyext.pyobject import PyObject, decref
+    while True:
+        py_obj = rawrefcount.next_dead(PyObject)
+        if not py_obj:
+            break
+        decref(space, py_obj)
+
+class PyObjDeallocAction(executioncontext.AsyncAction):
     """An action that invokes _Py_Dealloc() on the dying PyObjects.
     """
+    def perform(self, executioncontext, frame):
+        _rawrefcount_perform(self.space)
 
+class BoehmPyObjDeallocAction(executioncontext.PeriodicAsyncAction):
+    # This variant is used with Boehm, which doesn't have the explicit
+    # callback.  Instead we must periodically check ourselves.
     def perform(self, executioncontext, frame):
-        from pypy.module.cpyext.pyobject import PyObject, decref
-
-        while True:
-            py_obj = rawrefcount.next_dead(PyObject)
-            if not py_obj:
-                break
-            decref(self.space, py_obj)
+        if we_are_translated():
+            _rawrefcount_perform(self.space)
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -4,10 +4,11 @@
 #  This is meant for pypy's cpyext module, but is a generally
 #  useful interface over our GC.  XXX "pypy" should be removed here
 #
-import sys, weakref
-from rpython.rtyper.lltypesystem import lltype, llmemory
+import sys, weakref, py
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib import rgc
 
 
@@ -229,6 +230,11 @@
         v_p, v_ob = hop.inputargs(*hop.args_r)
         hop.exception_cannot_occur()
         hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
+        #
+        if hop.rtyper.annotator.translator.config.translation.gc == "boehm":
+            c_func = hop.inputconst(lltype.typeOf(func_boehm_eci),
+                                    func_boehm_eci)
+            hop.genop('direct_call', [c_func])
 
 
 class Entry(ExtRegistryEntry):
@@ -281,3 +287,10 @@
         v_ob = hop.genop('gc_rawrefcount_next_dead', [],
                          resulttype = llmemory.Address)
         return _spec_ob(hop, v_ob)
+
+src_dir = py.path.local(__file__).dirpath() / 'src'
+boehm_eci = ExternalCompilationInfo(
+    post_include_bits     = [(src_dir / 'boehm-rawrefcount.h').read()],
+    separate_module_files = [(src_dir / 'boehm-rawrefcount.c')],
+)
+func_boehm_eci = rffi.llexternal_use_eci(boehm_eci)
diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -0,0 +1,281 @@
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <assert.h>
+#include <limits.h>
+#include <gc/gc.h>
+#include <gc/gc_mark.h>
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+#  define RPY_EXTERN  /* nothing */
+#else
+#  include "common_header.h"
+#endif
+
+
+#define REFCNT_FROM_PYPY  (LONG_MAX / 4 + 1)
+
+typedef struct pypy_header0 gcobj_t;    /* opaque here */
+
+#ifndef _WIN32
+typedef intptr_t Py_ssize_t;
+#else
+typedef long Py_ssize_t;
+#endif
+
+/* this is the first two words of the PyObject structure used in
+   pypy/module/cpyext */
+typedef struct {
+    Py_ssize_t ob_refcnt;
+    Py_ssize_t ob_pypy_link;
+} pyobj_t;
+
+struct link_s {
+    pyobj_t *pyobj;    /* NULL if entry unused */
+    uintptr_t gcenc;
+    struct link_s *next_in_bucket;
+};
+
+#define MARKER_LIST_START  ((pyobj_t *)-1)
+
+static struct link_s **hash_buckets, *hash_list, *hash_free_list;
+static uintptr_t hash_mask_bucket;
+static intptr_t hash_list_walk_next = -1;
+
+static uintptr_t hash_get_hash(gcobj_t *gcobj)
+{
+    assert(gcobj != NULL);
+    uintptr_t h = (uintptr_t)gcobj;
+    assert((h & 1) == 0);
+    h -= (h >> 6);
+    return h & hash_mask_bucket;
+}
+
+static gcobj_t *decode_gcenc(uintptr_t gcenc)
+{
+    if (gcenc & 1)
+        gcenc = ~gcenc;
+    return (gcobj_t *)gcenc;
+}
+
+static void hash_link(struct link_s *lnk)
+{
+    uintptr_t h = hash_get_hash(decode_gcenc(lnk->gcenc));
+    lnk->next_in_bucket = hash_buckets[h];
+    hash_buckets[h] = lnk;
+}
+
+static void boehm_is_about_to_collect(void);
+
+static void hash_grow_table(void)
+{
+    static int rec = 0;
+    assert(!rec);   /* recursive hash_grow_table() */
+    rec = 1;
+
+    if (hash_buckets == NULL)
+        GC_set_start_callback(boehm_is_about_to_collect);
+
+    uintptr_t i, num_buckets = (hash_mask_bucket + 1) * 2;
+    if (num_buckets < 16) num_buckets = 16;
+    assert((num_buckets & (num_buckets - 1)) == 0);  /* power of two */
+
+    /* The new hash_buckets: an array of pointers to struct link_s, of
+       length a power of two, used as a dictionary hash table.  It is
+       not allocated with Boehm because there is no point in Boehm looking
+       in it.
+     */
+    struct link_s **new_buckets = calloc(num_buckets, sizeof(struct link_s *));
+    assert(new_buckets);
+
+    /* The new hash_list: the array of all struct link_s.  Their order
+       is irrelevant.  There is a GC_register_finalizer() on the 'gcenc'
+       field, so we don't move the array; instead we allocate a new array
+       to use in addition to the old one.  There are a total of 2 to 4
+       times as many 'struct link_s' as the length of 'buckets'.
+     */
+    uintptr_t num_list = num_buckets * 2;
+    struct link_s *new_list = GC_MALLOC(num_list * sizeof(struct link_s));
+    for (i = num_list; i-- > 1; ) {
+        new_list[i].next_in_bucket = hash_free_list;
+        hash_free_list = &new_list[i];
+    }
+    /* list[0] is abused to store a pointer to the previous list and
+       the length of the current list */
+    struct link_s *old_list = hash_list;
+    new_list[0].next_in_bucket = old_list;
+    new_list[0].gcenc = num_list;
+    new_list[0].pyobj = MARKER_LIST_START;
+
+    hash_list = new_list;
+    free(hash_buckets);
+    hash_buckets = new_buckets;
+    hash_mask_bucket = num_buckets - 1;
+    hash_list_walk_next = hash_mask_bucket;
+
+    /* re-add all old 'struct link_s' to the hash_buckets */
+    struct link_s *plist = old_list;
+    while (plist != NULL) {
+        uintptr_t count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc != 0)
+                hash_link(&plist[i]);
+        }
+        plist = plist[0].next_in_bucket;
+    }
+    GC_reachable_here(old_list);
+
+    rec = 0;
+}
+
+static void hash_add_entry(gcobj_t *gcobj, pyobj_t *pyobj)
+{
+    if (hash_free_list == NULL) {
+        hash_grow_table();
+    }
+    assert(pyobj->ob_pypy_link == 0);
+
+    struct link_s *lnk = hash_free_list;
+    hash_free_list = lnk->next_in_bucket;
+    lnk->pyobj = pyobj;
+    lnk->gcenc = (uintptr_t)gcobj;
+    pyobj->ob_pypy_link = (Py_ssize_t)lnk;
+
+    hash_link(lnk);
+
+    if (GC_base(gcobj) == NULL) {
+        /* 'gcobj' is probably a prebuilt object - it makes no */
+        /* sense to register it then, and it crashes Boehm in */
+        /* quite obscure ways */
+    }
+    else {
+        int j = GC_general_register_disappearing_link(
+                                    (void **)&lnk->gcenc, gcobj);
+        assert(j == GC_SUCCESS);
+    }
+}
+
+static pyobj_t *hash_get_entry(gcobj_t *gcobj)
+{
+    if (hash_buckets == NULL)
+        return NULL;
+    uintptr_t h = hash_get_hash(gcobj);
+    struct link_s *lnk = hash_buckets[h];
+    while (lnk != NULL) {
+        assert(lnk->pyobj != NULL);
+        if (decode_gcenc(lnk->gcenc) == gcobj)
+            return lnk->pyobj;
+        lnk = lnk->next_in_bucket;
+    }
+    return NULL;
+}
+
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_next_dead(void)
+{
+    while (hash_list_walk_next >= 0) {
+        struct link_s *p, **pp = &hash_buckets[hash_list_walk_next];
+        while (1) {
+            p = *pp;
+            if (p == NULL)
+                break;
+            assert(p->pyobj != NULL);
+            if (p->gcenc == 0) {
+                /* quadratic time on the number of links from the same
+                   bucket chain, but it should be small with very high
+                   probability */
+                pyobj_t *result = p->pyobj;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+                printf("next_dead: %p\n", result);
+#endif
+                assert(result->ob_refcnt == REFCNT_FROM_PYPY);
+                p->pyobj = NULL;
+                *pp = p->next_in_bucket;
+                p->next_in_bucket = hash_free_list;
+                hash_free_list = p;
+                return result;
+            }
+            else {
+                assert(p->gcenc != ~(uintptr_t)0);
+                pp = &p->next_in_bucket;
+            }
+        }
+        hash_list_walk_next--;
+    }
+    return NULL;
+}
+
+RPY_EXTERN
+void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                     /*pyobj_t*/void *pyobj)
+{
+    gcobj_t *gcobj1 = (gcobj_t *)gcobj;
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    assert(pyobj1->ob_pypy_link == 0);
+    /*assert(pyobj1->ob_refcnt >= REFCNT_FROM_PYPY);*/
+    /*^^^ could also be fixed just after the call to create_link_pypy()*/
+
+    hash_add_entry(gcobj1, pyobj1);
+}
+
+RPY_EXTERN
+/*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj)
+{
+    return hash_get_entry((gcobj_t *)gcobj);
+}
+
+RPY_EXTERN
+/*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj)
+{
+    pyobj_t *pyobj1 = (pyobj_t *)pyobj;
+
+    if (pyobj1->ob_pypy_link == 0)
+        return NULL;
+
+    struct link_s *lnk = (struct link_s *)pyobj1->ob_pypy_link;
+    assert(lnk->pyobj == pyobj1);
+    
+    gcobj_t *g = decode_gcenc(lnk->gcenc);
+    assert(g != NULL);
+    return g;
+}
+
+static void boehm_is_about_to_collect(void)
+{
+    struct link_s *plist = hash_list;
+    uintptr_t gcenc_union = 0;
+    while (plist != NULL) {
+        uintptr_t i, count = plist[0].gcenc;
+        for (i = 1; i < count; i++) {
+            if (plist[i].gcenc == 0)
+                continue;
+
+            pyobj_t *p = plist[i].pyobj;
+            assert(p != NULL);
+            assert(p->ob_refcnt >= REFCNT_FROM_PYPY);
+
+#ifdef TEST_BOEHM_RAWREFCOUNT
+            printf("plist[%d].gcenc: %p ", (int)i, plist[i].gcenc);
+#endif
+
+            if ((plist[i].gcenc & 1) ^ (p->ob_refcnt == REFCNT_FROM_PYPY)) {
+                /* ob_refcnt > FROM_PYPY: non-zero regular refcnt, 
+                   the gc obj must stay alive.  decode gcenc.
+                   ---OR---
+                   ob_refcnt == FROM_PYPY: no refs from C code, the
+                   gc obj must not (necessarily) stay alive.  encode gcenc.
+                */
+                plist[i].gcenc = ~plist[i].gcenc;
+            }
+            gcenc_union |= plist[i].gcenc;
+#ifdef TEST_BOEHM_RAWREFCOUNT
+            printf("-> %p\n", plist[i].gcenc);
+#endif
+    }
+        plist = plist[0].next_in_bucket;
+    }
+    if (gcenc_union & 1)   /* if there is at least one item potentially dead */
+        hash_list_walk_next = hash_mask_bucket;
+}
diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -0,0 +1,24 @@
+
+/* Missing:
+   OP_GC_RAWREFCOUNT_INIT(callback, r): the callback is not supported here
+   OP_GC_RAWREFCOUNT_CREATE_LINK_PYOBJ(): not implemented, maybe not needed
+*/
+
+#define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r)   \
+    gc_rawrefcount_create_link_pypy(gcobj, pyobj)
+
+#define OP_GC_RAWREFCOUNT_FROM_OBJ(gcobj, r)   \
+    r = gc_rawrefcount_from_obj(gcobj)
+
+#define OP_GC_RAWREFCOUNT_TO_OBJ(pyobj, r)   \
+    r = gc_rawrefcount_to_obj(pyobj)
+
+#define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
+    r = gc_rawrefcount_next_dead()
+
+
+RPY_EXTERN void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
+                                                /*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_from_obj(/*gcobj_t*/void *gcobj);
+RPY_EXTERN /*gcobj_t*/void *gc_rawrefcount_to_obj(/*pyobj_t*/void *pyobj);
+RPY_EXTERN /*pyobj_t*/void *gc_rawrefcount_next_dead(void);
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -266,3 +266,52 @@
         t, cbuilder = self.compile(entry_point)
         data = cbuilder.cmdexec('hi there')
         assert data.startswith('OK!\n')
+
+
+class TestBoehmTranslated(StandaloneTests):
+
+    def test_full_translation(self):
+
+        def make_ob():
+            p = W_Root(42)
+            ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+            rawrefcount.create_link_pypy(p, ob)
+            ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            assert rawrefcount.from_obj(PyObject, p) == ob
+            assert rawrefcount.to_obj(W_Root, ob) == p
+            return ob
+
+        prebuilt_p = W_Root(-42)
+        prebuilt_ob = lltype.malloc(PyObjectS, flavor='raw', zero=True,
+                                    immortal=True)
+
+        def entry_point(argv):
+            rawrefcount.create_link_pypy(prebuilt_p, prebuilt_ob)
+            prebuilt_ob.c_ob_refcnt += REFCNT_FROM_PYPY
+            oblist = [make_ob() for i in range(50)]
+            rgc.collect()
+            deadlist = []
+            while True:
+                ob = rawrefcount.next_dead(PyObject)
+                if not ob: break
+                deadlist.append(ob)
+            if len(deadlist) == 0:
+                print "no dead object"
+                return 1
+            if len(deadlist) < 30:
+                print "not enough dead objects"
+                return 1
+            for ob in deadlist:
+                if ob not in oblist:
+                    print "unexpected value for dead pointer"
+                    return 1
+                oblist.remove(ob)
+            print "OK!"
+            lltype.free(ob, flavor='raw')
+            return 0
+
+        self.config = get_combined_translation_config(translating=True)
+        self.config.translation.gc = "boehm"
+        t, cbuilder = self.compile(entry_point)
+        data = cbuilder.cmdexec('hi there')
+        assert data.startswith('OK!\n')
diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_rawrefcount_boehm.py
@@ -0,0 +1,230 @@
+import itertools, os, subprocess
+from hypothesis import given, strategies
+from rpython.tool.udir import udir
+
+
+TEST_CODE = r"""
+#define TEST_BOEHM_RAWREFCOUNT
+#include "boehm-rawrefcount.c"
+
+static gcobj_t *alloc_gcobj(void)   /* for tests */
+{
+    gcobj_t *g = GC_MALLOC(1000);
+    printf("gc obj: %p\n", g);
+    return g;
+}
+
+static pyobj_t *alloc_pyobj(void)   /* for tests */
+{
+    pyobj_t *p = malloc(1000);
+    p->ob_refcnt = 1;
+    p->ob_pypy_link = 0;
+    printf("py obj: %p\n", p);
+    return p;
+}
+
+static void decref(pyobj_t *p)      /* for tests */
+{
+    p->ob_refcnt--;
+    if (p->ob_refcnt == 0) {
+        printf("decref to zero: %p\n", p);
+        free(p);
+    }
+    assert(p->ob_refcnt >= REFCNT_FROM_PYPY ||
+           p->ob_refcnt < REFCNT_FROM_PYPY * 0.99);
+}
+
+void run_test(void);     /* forward declaration, produced by the test */
+
+int main(void)
+{
+    run_test();
+    while (gc_rawrefcount_next_dead() != NULL)
+        ;
+    return 0;
+}
+"""
+
+
+operations = strategies.sampled_from([
+    'new_pyobj',
+    'new_gcobj',
+    'create_link',
+    'from_obj',
+    'to_obj',
+    'forget_pyobj',
+    'forget_gcobj',
+    'collect',
+    'dead',
+    ])
+
+
+ at strategies.composite
+def make_code(draw):
+    code = []
+    pyobjs = []
+    gcobjs = []
+    num_gcobj = itertools.count()
+    num_pyobj = itertools.count()
+    links_g2p = {}
+    links_p2g = {}
+
+    def new_gcobj():
+        varname = 'g%d' % next(num_gcobj)
+        code.append('gcobj_t *volatile %s = alloc_gcobj();' % varname)
+        gcobjs.append(varname)
+        return varname
+
+    def new_pyobj():
+        varname = 'p%d' % next(num_pyobj)
+        code.append('pyobj_t *%s = alloc_pyobj();' % varname)
+        pyobjs.append(varname)
+        return varname
+
+    for op in draw(strategies.lists(operations, average_size=250)):
+        if op == 'new_gcobj':
+            new_gcobj()
+        elif op == 'new_pyobj':
+            new_pyobj()
+        elif op == 'create_link':
+            gvars = [varname for varname in gcobjs if varname not in links_g2p]
+            if gvars == []:
+                gvars.append(new_gcobj())
+            pvars = [varname for varname in pyobjs if varname not in links_p2g]
+            if pvars == []:
+                pvars.append(new_pyobj())
+            gvar = draw(strategies.sampled_from(gvars))
+            pvar = draw(strategies.sampled_from(pvars))
+            code.append(r'printf("create_link %%p-%%p\n", %s, %s); '
+                            % (gvar, pvar) +
+                        "%s->ob_refcnt += REFCNT_FROM_PYPY; " % pvar +
+                        "gc_rawrefcount_create_link_pypy(%s, %s);"
+                            % (gvar, pvar))
+            links_g2p[gvar] = pvar
+            links_p2g[pvar] = gvar
+        elif op == 'from_obj':
+            if gcobjs:
+                prnt = False
+                gvar = draw(strategies.sampled_from(gcobjs))
+                if gvar not in links_g2p:
+                    check = "== NULL"
+                elif links_g2p[gvar] in pyobjs:
+                    check = "== %s" % (links_g2p[gvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_from_obj(%s) %s);"
+                            % (gvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", %s, '
+                        'gc_rawrefcount_from_obj(%s));' % (gvar, gvar))
+        elif op == 'to_obj':
+            if pyobjs:
+                prnt = False
+                pvar = draw(strategies.sampled_from(pyobjs))
+                if pvar not in links_p2g:
+                    check = "== NULL"
+                elif links_p2g[pvar] in gcobjs:
+                    check = "== %s" % (links_p2g[pvar],)
+                else:
+                    check = "!= NULL"
+                    prnt = True
+                code.append("assert(gc_rawrefcount_to_obj(%s) %s);"
+                            % (pvar, check))
+                if prnt:
+                    code.append(r'printf("link %%p-%%p\n", '
+                        'gc_rawrefcount_to_obj(%s), %s);' % (pvar, pvar))
+        elif op == 'forget_pyobj':
+            if pyobjs:
+                index = draw(strategies.sampled_from(range(len(pyobjs))))
+                pvar = pyobjs.pop(index)
+                code.append(r'printf("-p%%p\n", %s); ' % pvar +
+                            "decref(%s); %s = NULL;" % (pvar, pvar))
+        elif op == 'forget_gcobj':
+            if gcobjs:
+                index = draw(strategies.sampled_from(range(len(gcobjs))))
+                gvar = gcobjs.pop(index)
+                code.append(r'printf("-g%%p\n", %s); ' % gvar +
+                            "%s = NULL;" % (gvar,))
+        elif op == 'collect':
+            code.append("GC_gcollect();")
+        elif op == 'dead':
+            code.append('gc_rawrefcount_next_dead();')
+        else:
+            assert False, op
+
+    return '\n'.join(code)
+
+
+ at given(make_code())
+def test_random(code):
+    filename = str(udir.join("test-rawrefcount-boehm.c"))
+    with open(filename, "w") as f:
+        print >> f, TEST_CODE
+        print >> f, 'void run_test(void) {'
+        print >> f, code
+        print >> f, '}'
+
+    srcdir = os.path.dirname(os.path.dirname(
+        os.path.abspath(os.path.join(__file__))))
+    srcdir = os.path.join(srcdir, 'src')
+
+    err = os.system("cd '%s' && gcc -Werror -lgc -I%s -o test-rawrefcount-boehm"
+                    " test-rawrefcount-boehm.c" % (udir, srcdir))
+    assert err == 0
+    p = subprocess.Popen("./test-rawrefcount-boehm", stdout=subprocess.PIPE,
+                         cwd=str(udir))
+    stdout, _ = p.communicate()
+    assert p.wait() == 0
+
+    gcobjs = {}
+    pyobjs = {}
+    links_p2g = {}
+    links_g2p = {}
+    for line in stdout.splitlines():
+        if line.startswith('py obj: '):
+            p = line[8:]
+            assert not pyobjs.get(p)
+            pyobjs[p] = True
+            assert p not in links_p2g
+        elif line.startswith('gc obj: '):
+            g = line[8:]
+            assert not gcobjs.get(g)
+            gcobjs[g] = True
+            if g in links_g2p: del links_g2p[g]
+        elif line.startswith('-p'):
+            p = line[2:]
+            assert pyobjs[p] == True
+            pyobjs[p] = False
+        elif line.startswith('-g'):
+            g = line[2:]
+            assert gcobjs[g] == True
+            gcobjs[g] = False
+        elif line.startswith('decref to zero: '):
+            p = line[16:]
+            assert pyobjs[p] == False
+            assert p not in links_p2g
+            del pyobjs[p]
+        elif line.startswith('create_link '):
+            g, p = line[12:].split('-')
+            assert g in gcobjs
+            assert p in pyobjs
+            assert g not in links_g2p
+            assert p not in links_p2g
+            links_g2p[g] = p
+            links_p2g[p] = g
+        elif line.startswith('link '):
+            g, p = line[5:].split('-')
+            assert g in gcobjs
+            assert p in pyobjs
+            assert links_g2p[g] == p
+            assert links_p2g[p] == g
+        elif line.startswith('plist['):
+            pass
+        elif line.startswith('next_dead: '):
+            p = line[11:]
+            assert pyobjs[p] == False
+            del pyobjs[p]
+            del links_p2g[p]
+        else:
+            assert False, repr(line)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -497,6 +497,7 @@
     'gc_rawrefcount_create_link_pyobj': LLOp(),
     'gc_rawrefcount_from_obj':          LLOp(sideeffects=False),
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
+    'gc_rawrefcount_next_dead':         LLOp(),
 
     # ------- JIT & GC interaction, only for some GCs ----------
 

From pypy.commits at gmail.com  Wed Sep  7 08:44:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 05:44:49 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: hg merge default
Message-ID: <57d00bc1.e97ac20a.1eabd.3260@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86927:b13abdc2e1b3
Date: 2016-09-07 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/b13abdc2e1b3/

Log:	hg merge default

diff too long, truncating to 2000 out of 212749 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,9 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,12 +360,15 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
 
   Heinrich-Heine University, Germany 
   Open End AB (formerly AB Strakt), Sweden
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.2"
+__version_info__ = (1, 8, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,43 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -3,7 +3,8 @@
 ============
 
 We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
-This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
 numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
 which now supports the "limited API" mode for c-extensions on 
 CPython >=3.2.
@@ -12,9 +13,7 @@
 support to OpenBSD and Dragon Fly BSD
 
 As always, this release fixed many issues and bugs raised by the
-growing community of PyPy users. 
-
-XXXXX MORE ???
+growing community of PyPy users. We strongly recommend updating.
 
 You can download the PyPy2.7 v5.4 release here:
 
@@ -110,8 +109,8 @@
 
   * (RPython) add `rposix_scandir` portably, needed for Python 3.5
 
-  * Support for memoryview attributes (format, itemsize, ...) which also
-    adds support for `PyMemoryView_FromObject`
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
 
 * Bug Fixes
 
@@ -153,10 +152,6 @@
   * Make `hash(-1)` return -2, as CPython does, and fix all the
     ancilary places this matters
 
-  * Issues reported with our previous release were resolved_ after
-    reports from users on our issue tracker at
-    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
-
   * Fix `PyNumber_Check()` to behave more like CPython
 
   * (VMProf) Try hard to not miss any Python-level frame in the
@@ -169,6 +164,10 @@
   * Fix the mapdict cache for subclasses of builtin types that
     provide a dict
 
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -0,0 +1,64 @@
+==========
+PyPy 5.4.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.4.0, released last week,
+due to the following issues:
+
+  * Update list of contributors in documentation and LICENSE file,
+    this was unfortunately left out of 5.4.0. My apologies to the new
+    contributors
+
+  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+    dynamically loadable file
+
+  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+
+  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+
+  * Fix for an issue where os.access() accepted a float for mode
+
+  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+    the last byte of a unicode string sometimes
+
+  * Update built-in cffi_ to the soon-to-be-released 1.8.2 version
+
+  * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
+    translation with a message asking for help porting it
+
+  * Fix a regression where a PyBytesObject was forced (converted to a RPython
+    object) when not required, reported as issue #2395
+
+Thanks to those who reported the issues.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _cffi: https://cffi.readthedocs.io
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,11 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
 
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1454,6 +1454,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -23,6 +23,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -84,7 +92,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -38,6 +38,8 @@
 
 
 class W_ClassObject(W_Root):
+    _immutable_fields_ = ['bases_w?[*]', 'w_dict?']
+
     def __init__(self, space, w_name, bases, w_dict):
         self.name = space.str_w(w_name)
         make_sure_not_resized(bases)
@@ -75,6 +77,7 @@
                             "__bases__ items must be classes")
         self.bases_w = bases_w
 
+    @jit.unroll_safe
     def is_subclass_of(self, other):
         assert isinstance(other, W_ClassObject)
         if self is other:
@@ -313,7 +316,7 @@
         # This method ignores the instance dict and the __getattr__.
         # Returns None if not found.
         assert isinstance(name, str)
-        w_value = self.w_class.lookup(space, name)
+        w_value = jit.promote(self.w_class).lookup(space, name)
         if w_value is None:
             return None
         w_descr_get = space.lookup(w_value, '__get__')
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -250,6 +250,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.2"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
                 return True
         return False
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        pass
-
     def convert_from_object(self, cdata, w_ob):
         if not self._copy_from_same(cdata, w_ob):
             self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
     )
     def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self.force_lazy_struct()
-        self._check_only_one_argument_for_union(w_ob)
 
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
-            if len(lst_w) > len(self._fields_list):
-                raise oefmt(space.w_ValueError,
-                            "too many initializers for '%s' (got %d)",
-                            self.name, len(lst_w))
-            for i in range(len(lst_w)):
-                optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+            j = 0
+            for w_obj in lst_w:
+                try:
+                    while (self._fields_list[j].flags &
+                               W_CField.BF_IGNORE_IN_CTOR):
+                        j += 1
+                except IndexError:
+                    raise oefmt(space.w_ValueError,
+                                "too many initializers for '%s' (got %d)",
+                                self.name, len(lst_w))
+                optvarsize = self._fields_list[j].write_v(cdata, w_obj,
                                                           optvarsize)
+                j += 1
             return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
 class W_CTypeUnion(W_CTypeStructOrUnion):
     kind = "union"
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        space = self.space
-        n = space.int_w(space.len(w_ob))
-        if n > 1:
-            raise oefmt(space.w_ValueError,
-                        "initializer for '%s': %d items given, but only one "
-                        "supported (use a dict if needed)", self.name, n)
-
 
 class W_CField(W_Root):
     _immutable_ = True
@@ -200,18 +194,21 @@
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
 
-    def __init__(self, ctype, offset, bitshift, bitsize):
+    BF_IGNORE_IN_CTOR = 0x01
+
+    def __init__(self, ctype, offset, bitshift, bitsize, flags):
         self.ctype = ctype
         self.offset = offset
         self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
         self.bitsize = bitsize
+        self.flags = flags       # BF_xxx
 
     def is_bitfield(self):
         return self.bitshift >= 0
 
-    def make_shifted(self, offset):
+    def make_shifted(self, offset, fflags):
         return W_CField(self.ctype, offset + self.offset,
-                        self.bitshift, self.bitsize)
+                        self.bitshift, self.bitsize, self.flags | fflags)
 
     def read(self, cdata):
         cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
     offset = interp_attrproperty('offset', W_CField),
     bitshift = interp_attrproperty('bitshift', W_CField),
     bitsize = interp_attrproperty('bitsize', W_CField),
+    flags = interp_attrproperty('flags', W_CField),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
         if alignment < falign and do_align:
             alignment = falign
         #
+        if is_union and i > 0:
+            fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+        else:
+            fflags = 0
+        #
         if fbitsize < 0:
             # not a bitfield: common case
 
@@ -372,7 +377,7 @@
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8)
+                    fld = srcfld.make_shifted(boffset // 8, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                                           fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
@@ -489,7 +495,7 @@
                     bitshift = 8 * ftype.size - fbitsize- bitshift
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
-                                           bitshift, fbitsize)
+                                           bitshift, fbitsize, fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -597,6 +597,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -122,7 +122,7 @@
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,6 +645,9 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.UCHAR),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
@@ -977,8 +980,10 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
     add_fork_hook('child', reinit_tls)
 
 def init_function(func):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,8 +1,9 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
-from pypy.module.cpyext.pyobject import PyObject
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -33,13 +34,82 @@
     raise an error if the object can't support a simpler view of its memory.
 
     0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    view.c_len = buf.getlength()
+    view.c_obj = make_ref(space, w_obj)
+    ndim = buf.getndim()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_readonly', int(buf.readonly))
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.str2charp(buf.getformat())
+    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+        
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
+def PyBuffer_IsContiguous(space, view, fort):
     """Return 1 if the memory defined by the view is C-style (fortran is
     'C') or Fortran-style (fortran is 'F') contiguous or either one
     (fortran is 'A').  Return 0 otherwise."""
-    # PyPy only supports contiguous Py_buffers for now.
-    return 1
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
+    
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -29,19 +29,17 @@
 ## Solution
 ## --------
 ##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
+## PyBytesObject contains two additional members: the ob_size and an array
+## char ob_sval which holds a \x0 terminated string.
 ##
 ## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   NULL buffer.  The first time PyString_AsString() is called, memory is
-##   allocated (with flavor='raw') and content is copied.
+##   buffer holding \x0.  The first time PyString_AsString() is called, the 
+##   PyStringObject is reallocated, and the string copied into the buffer. The
+##   ob_size reflects the length of the string.
 ##
 ## - A string allocated with PyString_FromStringAndSize(NULL, size) will
 ##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, but the reference won't be stored in the global map; there is no
-##   corresponding object in pypy.  When from_ref() or Py_INCREF() is called,
-##   the pypy string is created, and added to the global map of tracked
-##   objects.  The buffer is then supposed to be immutable.
+##   size+1, as part of the object. The buffer is then supposed to be immutable.
 ##
 ##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
 ##   there is no corresponding pypy object for the string
@@ -156,9 +154,6 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not pyobj_has_w_obj(ref):
-        # XXX Force the ref?
-        bytes_realize(space, ref)
     return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -123,5 +123,4 @@
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
     space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname))
-    importing.exec_code_module(space, w_mod, code)
-    return w_mod
+    return importing.exec_code_module(space, w_mod, code, w_name)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -142,7 +142,8 @@
 typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
 typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -156,12 +157,14 @@
     char *format;
     Py_ssize_t *shape;
     Py_ssize_t *strides;
-    Py_ssize_t *suboffsets;
-
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format;
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
        mono-dimensional buffers. */
     /* Py_ssize_t smalltable[2]; */
-    void *internal;
+    void *internal; /* always NULL for app-level objects */
 } Py_buffer;
 
 
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.0"
-#define PYPY_VERSION_NUM  0x05040000
+#define PYPY_VERSION "5.5.0-alpha0"
+#define PYPY_VERSION_NUM  0x05050000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,7 +1,8 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
@@ -12,6 +13,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -20,21 +22,35 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-    # TODO - fill in fields
-    '''
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if not isinstance(w_obj, W_MemoryView):
+        return view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_obj)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        isstr = False
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        rffi.setintfield(view, 'c_readonly', 1)
+        isstr = True
+    view.c_len = w_obj.getlength()
+    view.c_itemsize = w_obj.buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    shape = w_obj.buf.getshape()
+    strides = w_obj.buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    ''' 
     return view
 
-
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -508,10 +508,9 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Releases a Py_buffer obtained from getbuffer ParseTuple's s*.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
+    Release the buffer view. This should be called when the buffer is 
+    no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
     view.c_obj = lltype.nullptr(PyObject.TO)
+    # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
     def getshape(self):
         return self.shape
 
+    def getstrides(self):
+        return self.strides
+
     def getitemsize(self):
         return self.itemsize
 
+    def getndim(self):
+        return self.ndim
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
   PyMyArray* self = (PyMyArray*)obj;
-  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
-    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
-    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
@@ -188,7 +185,131 @@
     (initproc)PyMyArray_init,     /* tp_init */
 };
 
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+    Py_buffer* view = NULL;
+    PyObject* obj = PyTuple_GetItem(args, 0);
+    PyObject* memoryview = PyMemoryView_FromObject(obj);
+    if (memoryview == NULL)
+        return PyInt_FromLong(-1);
+    view = PyMemoryView_GET_BUFFER(memoryview);
+    Py_DECREF(memoryview);
+    return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG)                                        \
+    buf_flag = PyUnicode_FromString(#FLAG);                         \
+    flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ);  \
+    Py_DECREF(buf_flag);                                            \
+    if (flag_matches == 1) {                                        \
+        Py_DECREF(tmp);                                             \
+        flags |= PyBUF_##FLAG;                                      \
+        continue;                                                   \
+    }                                                               \
+    else if (flag_matches == -1) {                                  \
+        Py_DECREF(tmp);                                             \
+        return NULL;                                                \
+    }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+    PyObject *buffer_obj, *pyflags;
+    PyObject *tmp, *buf_flag;
+    Py_buffer buffer;
+    PyObject *shape, *strides;
+    Py_ssize_t i, n;
+    int flag_matches;
+    int flags = 0;
+
+    if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+        return NULL;
+    }
+
+    n = PySequence_Length(pyflags);
+    if (n < 0) {
+        return NULL;
+    }
+
+    for (i=0; i < n; i++) {
+        tmp = PySequence_GetItem(pyflags, i);
+        if (tmp == NULL) {
+            return NULL;
+        }
+
+        GET_PYBUF_FLAG(SIMPLE);
+        GET_PYBUF_FLAG(WRITABLE);
+        GET_PYBUF_FLAG(STRIDES);
+        GET_PYBUF_FLAG(ND);
+        GET_PYBUF_FLAG(C_CONTIGUOUS);
+        GET_PYBUF_FLAG(F_CONTIGUOUS);
+        GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+        GET_PYBUF_FLAG(INDIRECT);
+        GET_PYBUF_FLAG(FORMAT);
+        GET_PYBUF_FLAG(STRIDED);
+        GET_PYBUF_FLAG(STRIDED_RO);
+        GET_PYBUF_FLAG(RECORDS);
+        GET_PYBUF_FLAG(RECORDS_RO);
+        GET_PYBUF_FLAG(FULL);
+        GET_PYBUF_FLAG(FULL_RO);
+        GET_PYBUF_FLAG(CONTIG);
+        GET_PYBUF_FLAG(CONTIG_RO);
+
+        Py_DECREF(tmp);
+
+        /* One of the flags must match */
+        PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+        return NULL;
+    }
+
+    if (buffer.shape == NULL) {
+        Py_INCREF(Py_None);
+        shape = Py_None;
+    }
+    else {
+        shape = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+        }
+    }
+
+    if (buffer.strides == NULL) {
+        Py_INCREF(Py_None);
+        strides = Py_None;
+    }
+    else {
+        strides = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+        }
+    }
+
+    PyBuffer_Release(&buffer);
+    return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
 static PyMethodDef buffer_functions[] = {
+    {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
+    {"get_buffer_info",   (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
@@ -198,7 +319,7 @@
     "buffer_test",
     "Module Doc",
     -1,
-    buffer_functions;
+    buffer_functions,
     NULL,
     NULL,
     NULL,
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -183,8 +183,27 @@
                  Py_INCREF(Py_None);
                  return Py_None;
              """),
+            ("c_only", "METH_NOARGS",
+            """
+                int ret;
+                char * buf2;
+                PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+                if (!obj)
+                    return NULL;
+                buf2 = PyBytes_AsString(obj);
+                if (!buf2)
+                    return NULL;
+                /* buf should not have been forced, issue #2395 */
+                ret = _PyBytes_Resize(&obj, 512);
+                if (ret < 0)
+                    return NULL;
+                 Py_DECREF(obj);
+                 Py_INCREF(Py_None);
+                 return Py_None;
+            """),
             ])
         module.getbytes()
+        module.c_only()
 
     def test_py_string_as_string_Unicode(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -92,10 +92,20 @@
             link_extra=link_extra,
             libraries=libraries)
     from pypy.module.imp.importing import get_so_extension
-    pydname = soname.new(purebasename=modname, ext=get_so_extension(space))
+    ext = get_so_extension(space)
+    pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
 
+def get_so_suffix():
+    from imp import get_suffixes, C_EXTENSION
+    for suffix, mode, typ in get_suffixes():
+        if typ == C_EXTENSION:
+            return suffix
+    else:
+        raise RuntimeError("This interpreter does not define a filename "
+            "suffix for C extensions!")
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -126,13 +136,9 @@
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra)
-    from imp import get_suffixes, C_EXTENSION
-    pydname = soname
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            pydname = soname.new(purebasename=modname, ext=suffix)
-            soname.rename(pydname)
-            break
+    ext = get_so_suffix()
+    pydname = soname.new(purebasename=modname, ext=ext)
+    soname.rename(pydname)
     return str(pydname)
 
 def freeze_refcnts(self):
@@ -145,6 +151,24 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
+class FakeSpace(object):
+    """Like TinyObjSpace, but different"""
+    def __init__(self, config):
+        from distutils.sysconfig import get_python_inc
+        self.config = config
+        self.include_dir = get_python_inc()
+
+    def passthrough(self, arg):
+        return arg
+    listview = passthrough
+    str_w = passthrough
+
+    def unwrap(self, args):
+        try:
+            return args.str_w(None)
+        except:
+            return args
+
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -433,21 +457,8 @@
         self.imported_module_names = []
 
         if self.runappdirect:
+            fake = FakeSpace(self.space.config)
             def interp2app(func):
-                from distutils.sysconfig import get_python_inc
-                class FakeSpace(object):
-                    def passthrough(self, arg):
-                        return arg
-                    listview = passthrough
-                    str_w = passthrough
-                    def unwrap(self, args):
-                        try:
-                            return args.str_w(None)
-                        except:
-                            return args
-                fake = FakeSpace()
-                fake.include_dir = get_python_inc()
-                fake.config = self.space.config
                 def run(*args, **kwargs):
                     for k in kwargs.keys():
                         if k not in func.unwrap_spec and not k.startswith('w_'):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,14 +1,9 @@
-import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         w_hello = space.newbytes("hello")
         assert api.PyObject_CheckBuffer(w_hello)
         w_view = api.PyMemoryView_FromObject(w_hello)
@@ -17,6 +12,12 @@
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
+    def test_frombuffer(self, space, api):
+        w_buf = space.newbuffer(StringBuffer("hello"))
+        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        ndim = w_view.c_ndim
+        assert ndim == 1
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):
@@ -26,7 +27,25 @@
         y = memoryview(arr)
         assert y.format == 'i'
         assert y.shape == (10,)
+        assert len(y) == 10
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
+        viewlen = module.test_buffer(arr)
+        assert viewlen == y.itemsize * len(y)
 
+    def test_buffer_info(self):
+        from _numpypy import multiarray as np
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        # test_export_flags from numpy test_multiarray
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        # test_relaxed_strides from numpy test_multiarray
+        arr = np.zeros((1, 10))
+        if arr.flags.f_contiguous:
+            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+            assert strides[0] == 8
+            arr = np.ones((10, 1), order='F')
+            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+            assert strides[-1] == 8
+
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -32,9 +32,11 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
                     STRUCT_TYPE = PySequenceMethods
+                elif slot_names[0] == 'c_tp_as_buffer':
+                    STRUCT_TYPE = PyBufferProcs
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -597,6 +597,11 @@
 
 @jit.dont_look_inside
 def load_module(space, w_modulename, find_info, reuse=False):
+    """Like load_module() in CPython's import.c, this will normally
+    make a module object, store it in sys.modules, execute code in it,
+    and then fetch it again from sys.modules.  But this logic is not
+    used if we're calling a PEP302 loader.
+    """
     if find_info is None:
         return
 
@@ -625,17 +630,15 @@
 
         try:
             if find_info.modtype == PY_SOURCE:
-                load_source_module(
+                return load_source_module(
                     space, w_modulename, w_mod,
                     find_info.filename, find_info.stream.readall(),
                     find_info.stream.try_to_find_file_descriptor())
-                return w_mod
             elif find_info.modtype == PY_COMPILED:
                 magic = _r_long(find_info.stream)
                 timestamp = _r_long(find_info.stream)
-                load_compiled_module(space, w_modulename, w_mod, find_info.filename,
+                return load_compiled_module(space, w_modulename, w_mod, find_info.filename,
                                      magic, timestamp, find_info.stream.readall())
-                return w_mod
             elif find_info.modtype == PKG_DIRECTORY:
                 w_path = space.newlist([space.wrap(find_info.filename)])
                 space.setattr(w_mod, space.wrap('__path__'), w_path)
@@ -644,14 +647,13 @@
                 if find_info is None:
                     return w_mod
                 try:
-                    load_module(space, w_modulename, find_info, reuse=True)
+                    w_mod = load_module(space, w_modulename, find_info,
+                                        reuse=True)
                 finally:
                     try:
                         find_info.stream.close()
                     except StreamErrors:
                         pass
-                # fetch the module again, in case of "substitution"
-                w_mod = check_sys_modules(space, w_modulename)
                 return w_mod
             elif find_info.modtype == C_EXTENSION and has_so_extension(space):
                 load_c_extension(space, find_info.filename, space.str_w(w_modulename))
@@ -677,13 +679,6 @@
         try:
             if find_info:
                 w_mod = load_module(space, w_modulename, find_info)
-                try:
-                    w_mod = space.getitem(space.sys.get("modules"),
-                                          w_modulename)
-                except OperationError as oe:
-                    if not oe.match(space, space.w_KeyError):
-                        raise
-                    raise OperationError(space.w_ImportError, w_modulename)
                 if w_parent is not None:
                     space.setattr(w_parent, space.wrap(partname), w_mod)
                 return w_mod
@@ -875,20 +870,32 @@
     pycode = ec.compiler.compile(source, pathname, 'exec', 0)
     return pycode
 
-def exec_code_module(space, w_mod, code_w):
+def exec_code_module(space, w_mod, code_w, w_modulename, check_afterwards=True):
+    """
+    Execute a code object in the module's dict.  Returns
+    'sys.modules[modulename]', which must exist.
+    """
     w_dict = space.getattr(w_mod, space.wrap('__dict__'))
     space.call_method(w_dict, 'setdefault',
                       space.wrap('__builtins__'),
                       space.wrap(space.builtin))
     code_w.exec_code(space, w_dict, w_dict)
 
+    if check_afterwards:
+        w_mod = check_sys_modules(space, w_modulename)
+        if w_mod is None:
+            raise oefmt(space.w_ImportError,
+                        "Loaded module %R not found in sys.modules",
+                        w_modulename)
+    return w_mod
+
 
 @jit.dont_look_inside
 def load_source_module(space, w_modulename, w_mod, pathname, source, fd,
-                       write_pyc=True):
+                       write_pyc=True, check_afterwards=True):
     """
-    Load a source module from a given file and return its module
-    object.
+    Load a source module from a given file.  Returns the result
+    of sys.modules[modulename], which must exist.
     """
     w = space.wrap
 
@@ -927,9 +934,8 @@
         code_w.remove_docstrings(space)
 
     update_code_filenames(space, code_w, pathname)
-    exec_code_module(space, w_mod, code_w)
-
-    return w_mod
+    return exec_code_module(space, w_mod, code_w, w_modulename,
+                            check_afterwards=check_afterwards)
 
 def update_code_filenames(space, code_w, pathname, oldname=None):
     assert isinstance(code_w, PyCode)
@@ -1012,10 +1018,10 @@
 
 @jit.dont_look_inside
 def load_compiled_module(space, w_modulename, w_mod, cpathname, magic,
-                         timestamp, source):
+                         timestamp, source, check_afterwards=True):
     """
-    Load a module from a compiled file, execute it, and return its
-    module object.
+    Load a module from a compiled file and execute it.  Returns
+    'sys.modules[modulename]', which must exist.
     """
     log_pyverbose(space, 1, "import %s # compiled from %s\n" %
                   (space.str_w(w_modulename), cpathname))
@@ -1032,9 +1038,8 @@
     if optimize >= 2:
         code_w.remove_docstrings(space)
 
-    exec_code_module(space, w_mod, code_w)
-
-    return w_mod
+    return exec_code_module(space, w_mod, code_w, w_modulename,
+                            check_afterwards=check_afterwards)
 
 def open_exclusive(space, cpathname, mode):
     try:
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -98,33 +98,35 @@
     w_mod = space.wrap(Module(space, w_modulename))
     importing._prepare_module(space, w_mod, filename, None)
 
-    importing.load_source_module(
+    w_mod = importing.load_source_module(
         space, w_modulename, w_mod,
         filename, stream.readall(), stream.try_to_find_file_descriptor())
     if space.is_none(w_file):
         stream.close()
     return w_mod
 
- at unwrap_spec(filename='str0')
-def _run_compiled_module(space, w_modulename, filename, w_file, w_module):
+ at unwrap_spec(filename='str0', check_afterwards=int)
+def _run_compiled_module(space, w_modulename, filename, w_file, w_module,
+                         check_afterwards=False):
     # the function 'imp._run_compiled_module' is a pypy-only extension
     stream = get_file(space, w_file, filename, 'rb')
 
     magic = importing._r_long(stream)
     timestamp = importing._r_long(stream)
 
-    importing.load_compiled_module(
+    w_mod = importing.load_compiled_module(
         space, w_modulename, w_module, filename, magic, timestamp,
-        stream.readall())
+        stream.readall(), check_afterwards=check_afterwards)
     if space.is_none(w_file):
         stream.close()
+    return w_mod
 
 @unwrap_spec(filename='str0')
 def load_compiled(space, w_modulename, filename, w_file=None):
     w_mod = space.wrap(Module(space, w_modulename))
     importing._prepare_module(space, w_mod, filename, None)
-    _run_compiled_module(space, w_modulename, filename, w_file, w_mod)
-    return w_mod
+    return _run_compiled_module(space, w_modulename, filename, w_file, w_mod,
+                                check_afterwards=True)
 
 @unwrap_spec(filename=str)
 def load_dynamic(space, w_modulename, filename, w_file=None):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -118,7 +118,7 @@
         filename = str(p.join("x.py"))
         stream = streamio.open_file_as_stream(filename, "r")
         try:
-            importing.load_source_module(
+            _load_source_module(
                 space, w_modname, w(importing.Module(space, w_modname)),
                 filename, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -139,6 +139,15 @@
 
     return str(root)
 
+def _load_source_module(space, w_modname, w_mod, *args, **kwds):
+    kwds.setdefault('check_afterwards', False)
+    return importing.load_source_module(space, w_modname, w_mod, *args, **kwds)
+
+def _load_compiled_module(space, w_modname, w_mod, *args, **kwds):
+    kwds.setdefault('check_afterwards', False)
+    return importing.load_compiled_module(space, w_modname, w_mod,
+                                          *args, **kwds)
+
 
 def _setup(space):
     dn = setup_directory_structure(space)
@@ -887,8 +896,7 @@
             w_mod = space.wrap(Module(space, w_modulename))
             magic = importing._r_long(stream)
             timestamp = importing._r_long(stream)
-            w_ret = importing.load_compiled_module(space,
-                                                   w_modulename,
+            w_ret = _load_compiled_module(space,   w_modulename,
                                                    w_mod,
                                                    cpathname,
                                                    magic,
@@ -946,7 +954,7 @@
         pathname = _testfilesource()
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -968,7 +976,7 @@
         pathname = _testfilesource()
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor(),
@@ -987,7 +995,7 @@
         try:
             space.setattr(space.sys, space.wrap('dont_write_bytecode'),
                           space.w_True)
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1006,7 +1014,7 @@
         pathname = _testfilesource(source="<Syntax Error>")
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1026,7 +1034,7 @@
         pathname = _testfilesource(source="a = unknown_name")
         stream = streamio.open_file_as_stream(pathname, "r")
         try:
-            w_ret = importing.load_source_module(
+            w_ret = _load_source_module(
                 space, w_modulename, w_mod,
                 pathname, stream.readall(),
                 stream.try_to_find_file_descriptor())
@@ -1114,7 +1122,7 @@
                     magic = importing._r_long(stream)
                     timestamp = importing._r_long(stream)
                     space2.raises_w(space2.w_ImportError,
-                                    importing.load_compiled_module,
+                                    _load_compiled_module,
                                     space2,
                                     w_modulename,
                                     w_mod,
@@ -1326,10 +1334,7 @@
         # use an import hook that doesn't update sys.modules, then the
         # import succeeds; but at the same time, you can have the same
         # result without an import hook (see test_del_from_sys_modules)
-        # and then the import fails.  This looks like even more mess
-        # to replicate, so we ignore it until someone really hits this
-        # case...
-        skip("looks like an inconsistency in CPython")
+        # and then the import fails.  Mess mess mess.
 
         class ImportHook(object):
             def find_module(self, fullname, path=None):
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -225,15 +225,6 @@
 
     def dump_w_obj(self, w_obj):
         space = self.space
-        if space.type(w_obj).is_heaptype():
-            try:
-                buf = space.readbuf_w(w_obj)
-            except OperationError as e:
-                if not e.match(space, space.w_TypeError):
-                    raise
-                self.raise_exc("unmarshallable object")
-            else:
-                w_obj = space.newbuffer(buf)
         try:
             self.put_w_obj(w_obj)
         except rstackovf.StackOverflow:
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -186,6 +186,8 @@
             assert str(exc.value) == 'unmarshallable object'
             exc = raises(ValueError, marshal.dumps, subtype())
             assert str(exc.value) == 'unmarshallable object'
+            exc = raises(ValueError, marshal.dumps, (subtype(),))
+            assert str(exc.value) == 'unmarshallable object'
 
     def test_valid_subtypes(self):
         import marshal
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
     def getdictvalue(self, space, key):
         return self.items[key]
 

From pypy.commits at gmail.com  Wed Sep  7 08:51:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 05:51:33 -0700 (PDT)
Subject: [pypy-commit] pypy boehm-rawrefcount: test and fix
Message-ID: <57d00d55.c70a1c0a.5ad4f.5536@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: boehm-rawrefcount
Changeset: r86929:226d5e4b2e41
Date: 2016-09-07 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/226d5e4b2e41/

Log:	test and fix

diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
--- a/rpython/rlib/src/boehm-rawrefcount.c
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -190,6 +190,7 @@
                 printf("next_dead: %p\n", result);
 #endif
                 assert(result->ob_refcnt == REFCNT_FROM_PYPY);
+                result->ob_refcnt = 1;
                 p->pyobj = NULL;
                 *pp = p->next_in_bucket;
                 p->next_in_bucket = hash_free_list;
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -254,6 +254,9 @@
             if rawrefcount.next_dead(PyObject) != ob:
                 print "NEXT_DEAD != OB"
                 return 1
+            if ob.c_ob_refcnt != 1:
+                print "next_dead().ob_refcnt != 1"
+                return 1
             if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
                 print "NEXT_DEAD second time != NULL"
                 return 1
@@ -294,6 +297,9 @@
             while True:
                 ob = rawrefcount.next_dead(PyObject)
                 if not ob: break
+                if ob.c_ob_refcnt != 1:
+                    print "next_dead().ob_refcnt != 1"
+                    return 1
                 deadlist.append(ob)
             if len(deadlist) == 0:
                 print "no dead object"

From pypy.commits at gmail.com  Wed Sep  7 09:00:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 06:00:06 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: hg merge boehm-rawrefcount
Message-ID: <57d00f56.0575c20a.329f0.f1de@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86930:22608a742f02
Date: 2016-09-07 14:59 +0200
http://bitbucket.org/pypy/pypy/changeset/22608a742f02/

Log:	hg merge boehm-rawrefcount

diff --git a/rpython/rlib/src/boehm-rawrefcount.c b/rpython/rlib/src/boehm-rawrefcount.c
--- a/rpython/rlib/src/boehm-rawrefcount.c
+++ b/rpython/rlib/src/boehm-rawrefcount.c
@@ -190,6 +190,7 @@
                 printf("next_dead: %p\n", result);
 #endif
                 assert(result->ob_refcnt == REFCNT_FROM_PYPY);
+                result->ob_refcnt = 1;
                 p->pyobj = NULL;
                 *pp = p->next_in_bucket;
                 p->next_in_bucket = hash_free_list;
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -254,6 +254,9 @@
             if rawrefcount.next_dead(PyObject) != ob:
                 print "NEXT_DEAD != OB"
                 return 1
+            if ob.c_ob_refcnt != 1:
+                print "next_dead().ob_refcnt != 1"
+                return 1
             if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
                 print "NEXT_DEAD second time != NULL"
                 return 1
@@ -294,6 +297,9 @@
             while True:
                 ob = rawrefcount.next_dead(PyObject)
                 if not ob: break
+                if ob.c_ob_refcnt != 1:
+                    print "next_dead().ob_refcnt != 1"
+                    return 1
                 deadlist.append(ob)
             if len(deadlist) == 0:
                 print "no dead object"

From pypy.commits at gmail.com  Wed Sep  7 09:51:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 06:51:04 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Add test file (not working at
 all so far)
Message-ID: <57d01b48.2146c20a.7f80b.3291@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86931:6b424d821c6c
Date: 2016-09-07 15:50 +0200
http://bitbucket.org/pypy/pypy/changeset/6b424d821c6c/

Log:	Add test file (not working at all so far)

diff --git a/rpython/translator/revdb/test/test_rawrefcount.py b/rpython/translator/revdb/test/test_rawrefcount.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/test/test_rawrefcount.py
@@ -0,0 +1,61 @@
+from rpython.rlib import objectmodel, rgc, revdb
+from rpython.rtyper.lltypesystem import lltype
+from rpython.translator.revdb.test.test_basic import InteractiveTests
+from rpython.translator.revdb.test.test_basic import compile, fetch_rdb, run
+from rpython.translator.revdb.message import *
+
+from rpython.rlib import rawrefcount
+
+
+class TestRawRefcount(InteractiveTests):
+    expected_stop_points = 27
+
+    def setup_class(cls):
+        class W_Root(object):
+            def __init__(self, n):
+                self.n = n
+        PyObjectS = lltype.Struct('PyObjectS',
+                                  ('c_ob_refcnt', lltype.Signed),
+                                  ('c_ob_pypy_link', lltype.Signed))
+        PyObject = lltype.Ptr(PyObjectS)
+        w1 = W_Root(-42)
+        ob1 = lltype.malloc(PyObjectS, flavor='raw', zero=True,
+                            immortal=True)
+        ob1.c_ob_refcnt = rawrefcount.REFCNT_FROM_PYPY
+
+        def main(argv):
+            rawrefcount.create_link_pypy(w1, ob1)
+            w = None
+            ob = lltype.nullptr(PyObjectS)
+            oblist = []
+            for op in argv[1:]:
+                revdb.stop_point()
+                w = W_Root(42)
+                ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+                ob.c_ob_refcnt = rawrefcount.REFCNT_FROM_PYPY
+                rawrefcount.create_link_pypy(w, ob)
+                oblist.append(ob)
+            del oblist[-1]
+            #
+            rgc.collect()
+            assert rawrefcount.from_obj(PyObject, w) == ob
+            assert rawrefcount.to_obj(W_Root, ob) == w
+            while True:
+                ob = rawrefcount.next_dead(PyObject)
+                if not ob:
+                    break
+                assert ob in oblist
+                oblist.remove(ob)
+            objectmodel.keepalive_until_here(w)
+            revdb.stop_point()
+            return 9
+        compile(cls, main, backendopt=False)
+        ARGS26 = 'a b c d e f g h i j k l m n o p q r s t u v w x y z'
+        run(cls, ARGS26)
+        rdb = fetch_rdb(cls, [cls.exename] + ARGS26.split())
+        assert rdb.number_of_stop_points() == cls.expected_stop_points
+
+    def test_go(self):
+        child = self.replay()
+        child.send(Message(CMD_FORWARD, 50))
+        child.expect(ANSWER_AT_END)

From pypy.commits at gmail.com  Wed Sep  7 10:07:18 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 07 Sep 2016 07:07:18 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57d01f16.e856c20a.a904.b99d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86933:0c3342bbecb5
Date: 2016-09-07 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/0c3342bbecb5/

Log:	merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -30,3 +30,6 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.1
+Version: 1.8.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.1"
-__version_info__ = (1, 8, 1)
+__version__ = "1.8.2"
+__version_info__ = (1, 8, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.1"
+                               "\ncompiled with cffi version: 1.8.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -0,0 +1,64 @@
+==========
+PyPy 5.4.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.4.0, released last week,
+due to the following issues:
+
+  * Update list of contributors in documentation and LICENSE file,
+    this was unfortunately left out of 5.4.0. My apologies to the new
+    contributors
+
+  * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a
+    dynamically loadable file
+
+  * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython
+
+  * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
+
+  * Fix for an issue for translating the sandbox
+
+  * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
+    the last byte of a unicode string sometimes
+
+  * Update built-in cffi_ to version 1.8.1
+
+  * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
+    translation with a message asking for help porting it
+
+  * Fix a regression where a PyBytesObject was forced (converted to a RPython
+    object) when not required, reported as issue #2395
+
+Thanks to those who reported the issues.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _cffi: https://cffi.readthedocs.io
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,9 @@
 
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1428,6 +1428,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.1"
+VERSION = "1.8.2"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
                 return True
         return False
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        pass
-
     def convert_from_object(self, cdata, w_ob):
         if not self._copy_from_same(cdata, w_ob):
             self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
     )
     def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self.force_lazy_struct()
-        self._check_only_one_argument_for_union(w_ob)
 
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
-            if len(lst_w) > len(self._fields_list):
-                raise oefmt(space.w_ValueError,
-                            "too many initializers for '%s' (got %d)",
-                            self.name, len(lst_w))
-            for i in range(len(lst_w)):
-                optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+            j = 0
+            for w_obj in lst_w:
+                try:
+                    while (self._fields_list[j].flags &
+                               W_CField.BF_IGNORE_IN_CTOR):
+                        j += 1
+                except IndexError:
+                    raise oefmt(space.w_ValueError,
+                                "too many initializers for '%s' (got %d)",
+                                self.name, len(lst_w))
+                optvarsize = self._fields_list[j].write_v(cdata, w_obj,
                                                           optvarsize)
+                j += 1
             return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
 class W_CTypeUnion(W_CTypeStructOrUnion):
     kind = "union"
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        space = self.space
-        n = space.int_w(space.len(w_ob))
-        if n > 1:
-            raise oefmt(space.w_ValueError,
-                        "initializer for '%s': %d items given, but only one "
-                        "supported (use a dict if needed)", self.name, n)
-
 
 class W_CField(W_Root):
     _immutable_ = True
@@ -200,18 +194,21 @@
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
 
-    def __init__(self, ctype, offset, bitshift, bitsize):
+    BF_IGNORE_IN_CTOR = 0x01
+
+    def __init__(self, ctype, offset, bitshift, bitsize, flags):
         self.ctype = ctype
         self.offset = offset
         self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
         self.bitsize = bitsize
+        self.flags = flags       # BF_xxx
 
     def is_bitfield(self):
         return self.bitshift >= 0
 
-    def make_shifted(self, offset):
+    def make_shifted(self, offset, fflags):
         return W_CField(self.ctype, offset + self.offset,
-                        self.bitshift, self.bitsize)
+                        self.bitshift, self.bitsize, self.flags | fflags)
 
     def read(self, cdata):
         cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
     offset = interp_attrproperty('offset', W_CField),
     bitshift = interp_attrproperty('bitshift', W_CField),
     bitsize = interp_attrproperty('bitsize', W_CField),
+    flags = interp_attrproperty('flags', W_CField),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
         if alignment < falign and do_align:
             alignment = falign
         #
+        if is_union and i > 0:
+            fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+        else:
+            fflags = 0
+        #
         if fbitsize < 0:
             # not a bitfield: common case
 
@@ -372,7 +377,7 @@
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8)
+                    fld = srcfld.make_shifted(boffset // 8, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                                           fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
@@ -489,7 +495,7 @@
                     bitshift = 8 * ftype.size - fbitsize- bitshift
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
-                                           bitshift, fbitsize)
+                                           bitshift, fbitsize, fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -450,7 +450,12 @@
                 # For compatibility
                 assert exc.value.errno == _ssl.SSL_ERROR_WANT_READ
             finally:
-                c.shutdown()
+                try:
+                    c.shutdown()
+                except _ssl.SSLError:
+                    # If the expected exception was raised, the SSLContext
+                    # can't be shut down yet
+                    pass
         finally:
             s.close()
 
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -597,6 +597,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -122,7 +122,7 @@
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,6 +645,9 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.UCHAR),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,8 +1,9 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
-from pypy.module.cpyext.pyobject import PyObject
+    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -33,13 +34,82 @@
     raise an error if the object can't support a simpler view of its memory.
 
     0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    view.c_len = buf.getlength()
+    view.c_obj = make_ref(space, w_obj)
+    ndim = buf.getndim()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_readonly', int(buf.readonly))
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.str2charp(buf.getformat())
+    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+        
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
+def PyBuffer_IsContiguous(space, view, fort):
     """Return 1 if the memory defined by the view is C-style (fortran is
     'C') or Fortran-style (fortran is 'F') contiguous or either one
     (fortran is 'A').  Return 0 otherwise."""
-    # PyPy only supports contiguous Py_buffers for now.
-    return 1
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
+    
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -14,45 +14,33 @@
 ## Implementation of PyBytesObject
 ## ================================
 ##
-## The problem
-## -----------
+## PyBytesObject has its own ob_sval buffer, so we have two copies of a string;
+## one in the PyBytesObject returned from various C-API functions and another
+## in the corresponding RPython object.
 ##
-## PyString_AsString() must return a (non-movable) pointer to the underlying
-## ob_sval, whereas pypy strings are movable.  C code may temporarily store
-## this address and use it, as long as it owns a reference to the PyObject.
-## There is no "release" function to specify that the pointer is not needed
-## any more.
+## The following calls can create a PyBytesObject without a correspoinding
+## RPython object:
 ##
-## Also, the pointer may be used to fill the initial value of string. This is
-## valid only when the string was just allocated, and is not used elsewhere.
+## PyBytes_FromStringAndSize(NULL, n) / PyString_FromStringAndSize(NULL, n)
 ##
-## Solution
-## --------
+## In the PyBytesObject returned, the ob_sval buffer may be modified as
+## long as the freshly allocated PyBytesObject is not "forced" via a call
+## to any of the more sophisticated C-API functions. 
 ##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
-##
-## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   NULL buffer.  The first time PyString_AsString() is called, memory is
-##   allocated (with flavor='raw') and content is copied.
-##
-## - A string allocated with PyString_FromStringAndSize(NULL, size) will
-##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, but the reference won't be stored in the global map; there is no
-##   corresponding object in pypy.  When from_ref() or Py_INCREF() is called,
-##   the pypy string is created, and added to the global map of tracked
-##   objects.  The buffer is then supposed to be immutable.
-##
-##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
-##   there is no corresponding pypy object for the string
-##
-## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
-##   similar object.
-##
-## - PyString_Size() doesn't need to force the object.
+## Care has been taken in implementing the functions below, so that
+## if they are called with a non-forced PyBytesObject, they will not 
+## unintentionally force the creation of a RPython object. As long as only these
+## are used, the ob_sval buffer is still modifiable:
+## 
+## PyBytes_AsString / PyString_AsString 
+## PyBytes_AS_STRING / PyString_AS_STRING
+## PyBytes_AsStringAndSize / PyString_AsStringAndSize
+## PyBytes_Size / PyString_Size
+## PyBytes_Resize / PyString_Resize
+## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string.
+##   corresponds to the pypy gc-managed string, 
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -156,9 +144,6 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not pyobj_has_w_obj(ref):
-        # XXX Force the ref?
-        bytes_realize(space, ref)
     return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
@@ -182,9 +167,6 @@
             raise oefmt(space.w_TypeError,
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
-    if not pyobj_has_w_obj(ref):
-        # force the ref
-        bytes_realize(space, ref)
     ref_str = rffi.cast(PyBytesObject, ref)
     data[0] = ref_str.c_ob_sval
     if length:
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -142,7 +142,8 @@
 typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
 typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -156,12 +157,14 @@
     char *format;
     Py_ssize_t *shape;
     Py_ssize_t *strides;
-    Py_ssize_t *suboffsets;
-
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format;
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
        mono-dimensional buffers. */
     /* Py_ssize_t smalltable[2]; */
-    void *internal;
+    void *internal; /* always NULL for app-level objects */
 } Py_buffer;
 
 
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,7 +1,8 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
@@ -12,6 +13,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -20,21 +22,35 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-    # TODO - fill in fields
-    '''
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if not isinstance(w_obj, W_MemoryView):
+        return view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_obj)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        isstr = False
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        rffi.setintfield(view, 'c_readonly', 1)
+        isstr = True
+    view.c_len = w_obj.getlength()
+    view.c_itemsize = w_obj.buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    shape = w_obj.buf.getshape()
+    strides = w_obj.buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    ''' 
     return view
 
-
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -508,10 +508,9 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Releases a Py_buffer obtained from getbuffer ParseTuple's s*.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
+    Release the buffer view. This should be called when the buffer is 
+    no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
     view.c_obj = lltype.nullptr(PyObject.TO)
+    # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
     def getshape(self):
         return self.shape
 
+    def getstrides(self):
+        return self.strides
+
     def getitemsize(self):
         return self.itemsize
 
+    def getndim(self):
+        return self.ndim
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
   PyMyArray* self = (PyMyArray*)obj;
-  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
-    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
-    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
@@ -188,7 +185,131 @@
     (initproc)PyMyArray_init,     /* tp_init */
 };
 
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+    Py_buffer* view = NULL;
+    PyObject* obj = PyTuple_GetItem(args, 0);
+    PyObject* memoryview = PyMemoryView_FromObject(obj);
+    if (memoryview == NULL)
+        return PyInt_FromLong(-1);
+    view = PyMemoryView_GET_BUFFER(memoryview);
+    Py_DECREF(memoryview);
+    return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG)                                        \
+    buf_flag = PyUnicode_FromString(#FLAG);                         \
+    flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ);  \
+    Py_DECREF(buf_flag);                                            \
+    if (flag_matches == 1) {                                        \
+        Py_DECREF(tmp);                                             \
+        flags |= PyBUF_##FLAG;                                      \
+        continue;                                                   \
+    }                                                               \
+    else if (flag_matches == -1) {                                  \
+        Py_DECREF(tmp);                                             \
+        return NULL;                                                \
+    }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+    PyObject *buffer_obj, *pyflags;
+    PyObject *tmp, *buf_flag;
+    Py_buffer buffer;
+    PyObject *shape, *strides;
+    Py_ssize_t i, n;
+    int flag_matches;
+    int flags = 0;
+
+    if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+        return NULL;
+    }
+
+    n = PySequence_Length(pyflags);
+    if (n < 0) {
+        return NULL;
+    }
+
+    for (i=0; i < n; i++) {
+        tmp = PySequence_GetItem(pyflags, i);
+        if (tmp == NULL) {
+            return NULL;
+        }
+
+        GET_PYBUF_FLAG(SIMPLE);
+        GET_PYBUF_FLAG(WRITABLE);
+        GET_PYBUF_FLAG(STRIDES);
+        GET_PYBUF_FLAG(ND);
+        GET_PYBUF_FLAG(C_CONTIGUOUS);
+        GET_PYBUF_FLAG(F_CONTIGUOUS);
+        GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+        GET_PYBUF_FLAG(INDIRECT);
+        GET_PYBUF_FLAG(FORMAT);
+        GET_PYBUF_FLAG(STRIDED);
+        GET_PYBUF_FLAG(STRIDED_RO);
+        GET_PYBUF_FLAG(RECORDS);
+        GET_PYBUF_FLAG(RECORDS_RO);
+        GET_PYBUF_FLAG(FULL);
+        GET_PYBUF_FLAG(FULL_RO);
+        GET_PYBUF_FLAG(CONTIG);
+        GET_PYBUF_FLAG(CONTIG_RO);
+
+        Py_DECREF(tmp);
+
+        /* One of the flags must match */
+        PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+        return NULL;
+    }
+
+    if (buffer.shape == NULL) {
+        Py_INCREF(Py_None);
+        shape = Py_None;
+    }
+    else {
+        shape = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+        }
+    }
+
+    if (buffer.strides == NULL) {
+        Py_INCREF(Py_None);
+        strides = Py_None;
+    }
+    else {
+        strides = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+        }
+    }
+
+    PyBuffer_Release(&buffer);
+    return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
 static PyMethodDef buffer_functions[] = {
+    {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
+    {"get_buffer_info",   (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -183,8 +183,27 @@
                  Py_INCREF(Py_None);
                  return Py_None;
              """),
+            ("c_only", "METH_NOARGS",
+            """
+                int ret;
+                char * buf2;
+                PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+                if (!obj)
+                    return NULL;
+                buf2 = PyBytes_AsString(obj);
+                if (!buf2)
+                    return NULL;
+                /* buf should not have been forced, issue #2395 */
+                ret = _PyBytes_Resize(&obj, 512);
+                if (ret < 0)
+                    return NULL;
+                 Py_DECREF(obj);
+                 Py_INCREF(Py_None);
+                 return Py_None;
+            """),
             ])
         module.getbytes()
+        module.c_only()
 
     def test_py_string_as_string_Unicode(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
@@ -12,6 +12,12 @@
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
+    def test_frombuffer(self, space, api):
+        w_buf = space.newbuffer(StringBuffer("hello"))
+        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        ndim = w_view.c_ndim
+        assert ndim == 1
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):
@@ -21,6 +27,25 @@
         y = memoryview(arr)
         assert y.format == 'i'
         assert y.shape == (10,)
+        assert len(y) == 10
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
+        viewlen = module.test_buffer(arr)
+        assert viewlen == y.itemsize * len(y)
+
+    def test_buffer_info(self):
+        from _numpypy import multiarray as np
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        # test_export_flags from numpy test_multiarray
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        # test_relaxed_strides from numpy test_multiarray
+        arr = np.zeros((1, 10))
+        if arr.flags.f_contiguous:
+            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+            assert strides[0] == 8
+            arr = np.ones((10, 1), order='F')
+            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+            assert strides[-1] == 8
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
                     STRUCT_TYPE = PySequenceMethods
+                elif slot_names[0] == 'c_tp_as_buffer':
+                    STRUCT_TYPE = PyBufferProcs
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -259,10 +259,8 @@
         raise oefmt(space.w_ValueError, "Empty module name")
     w = space.wrap
 
-    if w_fromlist is not None and space.is_true(w_fromlist):
-        fromlist_w = space.fixedview(w_fromlist)
-    else:
-        fromlist_w = None
+    if w_fromlist is not None and not space.is_true(w_fromlist):
+        w_fromlist = None
 
     rel_modulename = None
     if (level != 0 and w_globals is not None and
@@ -284,19 +282,19 @@
                     w_mod = None
                 else:
                     w_mod = absolute_import(space, rel_modulename, rel_level,
-                                            fromlist_w, tentative=True)
+                                            w_fromlist, tentative=True)
             else:
                 w_mod = absolute_import(space, rel_modulename, rel_level,
-                                        fromlist_w, tentative=False)
+                                        w_fromlist, tentative=False)
             if w_mod is not None:
                 return w_mod
 
-    w_mod = absolute_import(space, modulename, 0, fromlist_w, tentative=0)
+    w_mod = absolute_import(space, modulename, 0, w_fromlist, tentative=0)
     if rel_modulename is not None:
         space.setitem(space.sys.get('modules'), w(rel_modulename), space.w_None)
     return w_mod
 
-def absolute_import(space, modulename, baselevel, fromlist_w, tentative):
+def absolute_import(space, modulename, baselevel, w_fromlist, tentative):
     # Short path: check in sys.modules, but only if there is no conflict
     # on the import lock.  In the situation of 'import' statements
     # inside tight loops, this should be true, and absolute_import_try()
@@ -304,25 +302,25 @@
     # if the import lock is currently held by another thread, then we
     # have to wait, and so shouldn't use the fast path.
     if not getimportlock(space).lock_held_by_someone_else():
-        w_mod = absolute_import_try(space, modulename, baselevel, fromlist_w)
+        w_mod = absolute_import_try(space, modulename, baselevel, w_fromlist)
         if w_mod is not None and not space.is_w(w_mod, space.w_None):
             return w_mod
     return absolute_import_with_lock(space, modulename, baselevel,
-                                     fromlist_w, tentative)
+                                     w_fromlist, tentative)
 
 @jit.dont_look_inside
 def absolute_import_with_lock(space, modulename, baselevel,
-                              fromlist_w, tentative):
+                              w_fromlist, tentative):
     lock = getimportlock(space)
     lock.acquire_lock()
     try:
         return _absolute_import(space, modulename, baselevel,
-                                fromlist_w, tentative)
+                                w_fromlist, tentative)
     finally:
         lock.release_lock(silent_after_fork=True)
 
 @jit.unroll_safe
-def absolute_import_try(space, modulename, baselevel, fromlist_w):
+def absolute_import_try(space, modulename, baselevel, w_fromlist):
     """ Only look up sys.modules, not actually try to load anything
     """
     w_path = None
@@ -330,7 +328,7 @@
     if '.' not in modulename:
         w_mod = check_sys_modules_w(space, modulename)
         first = w_mod
-        if fromlist_w is not None and w_mod is not None:
+        if w_fromlist is not None and w_mod is not None:
             w_path = try_getattr(space, w_mod, space.wrap('__path__'))
     else:
         level = 0
@@ -345,28 +343,36 @@
                 return None
             if level == baselevel:
                 first = w_mod
-            if fromlist_w is not None:
+            if w_fromlist is not None:
                 w_path = try_getattr(space, w_mod, space.wrap('__path__'))
             level += 1
-    if fromlist_w is not None:
+    if w_fromlist is not None:
+        # bit artificial code but important to not just unwrap w_fromlist
+        # to get a better trace. if it is unwrapped, the immutability of the
+        # tuple is lost
         if w_path is not None:
-            if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],
-                                                   space.wrap('*')):
+            length = space.len_w(w_fromlist)
+            if length == 1 and space.eq_w(
+                    space.getitem(w_fromlist, space.wrap(0)),
+                    space.wrap('*')):
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
-                    fromlist_w = space.fixedview(w_all)
+                    w_fromlist = w_all
                 else:
-                    fromlist_w = []
+                    w_fromlist = None
                     # "from x import *" with x already imported and no x.__all__
                     # always succeeds without doing more imports.  It will
                     # just copy everything from x.__dict__ as it is now.
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    return None
+
+            if w_fromlist is not None:
+                for i in range(length):
+                    w_name = space.getitem(w_fromlist, space.wrap(i))
+                    if try_getattr(space, w_mod, w_name) is None:
+                        return None
         return w_mod
     return first
 
-def _absolute_import(space, modulename, baselevel, fromlist_w, tentative):
+def _absolute_import(space, modulename, baselevel, w_fromlist, tentative):
     w = space.wrap
 
     if '/' in modulename or '\\' in modulename:
@@ -394,18 +400,23 @@
         w_path = try_getattr(space, w_mod, w('__path__'))
         level += 1
 
-    if fromlist_w is not None:
+    if w_fromlist is not None:
         if w_path is not None:
-            if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],w('*')):
+            length = space.len_w(w_fromlist)
+            if length == 1 and space.eq_w(
+                    space.getitem(w_fromlist, space.wrap(0)),
+                    space.wrap('*')):
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
-                    fromlist_w = space.fixedview(w_all)
+                    w_fromlist = w_all
                 else:
-                    fromlist_w = []
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    load_part(space, w_path, prefix, space.str0_w(w_name),
-                              w_mod, tentative=1)
+                    w_fromlist = None
+            if w_fromlist is not None:
+                for i in range(length):
+                    w_name = space.getitem(w_fromlist, space.wrap(i))
+                    if try_getattr(space, w_mod, w_name) is None:
+                        load_part(space, w_path, prefix, space.str0_w(w_name),
+                                  w_mod, tentative=1)
         return w_mod
     else:
         return first
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
     def getdictvalue(self, space, key):
         return self.items[key]
 
+    def descr_memoryview(self, space, buf):
+        raise oefmt(space.w_TypeError, "error")
+
 class IterDictObject(W_Root):
     def __init__(self, space, w_dict):
         self.space = space
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -377,7 +377,25 @@
     def __exit__(self, typ, value, traceback):
         keepalive_until_here(self)
 
-    def get_buffer(self, space, readonly):
+    def get_buffer(self, space, flags):
+        errtype = space.w_ValueError # should be BufferError, numpy does this instead
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_F_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not Fortran contiguous")
+        if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                     self.flags & NPY.ARRAY_C_CONTIGUOUS)):
+           raise oefmt(errtype, "ndarray is not contiguous")
+        if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and
+            not self.flags & NPY.ARRAY_WRITEABLE):
+           raise oefmt(errtype, "buffer source array is read-only")
+        readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
         return ArrayBuffer(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
@@ -695,6 +713,7 @@
                  index + self.impl.start)
 
     def setitem(self, index, v):
+        # XXX what if self.readonly?
         raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstring import strip_spaces
@@ -42,7 +43,7 @@
         raise oefmt(space.w_ValueError,
                     "object __array__ method not producing an array")
 
-def try_interface_method(space, w_object):
+def try_interface_method(space, w_object, copy):
     try:
         w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
         if w_interface is None:
@@ -81,17 +82,20 @@
             raise oefmt(space.w_ValueError,
                     "__array_interface__ could not decode dtype %R", w_dtype
                     )
-        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
+        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
+                                   space.isinstance_w(w_data, space.w_list)):
             data_w = space.listview(w_data)
-            data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
-            read_only = True # XXX why not space.is_true(data_w[1])
+            w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+            read_only = space.is_true(data_w[1]) or copy
             offset = 0
-            return W_NDimArray.from_shape_and_storage(space, shape, data, 
-                                    dtype, strides=strides, start=offset), read_only
+            w_base = w_object
+            if read_only:
+                w_base = None
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+                                dtype, w_base=w_base, strides=strides,
+                                start=offset), read_only
         if w_data is None:
-            data = w_object
-        else:
-            data = w_data
+            w_data = w_object
         w_offset = space.finditem(w_interface, space.wrap('offset'))
         if w_offset is None:
             offset = 0
@@ -101,7 +105,7 @@
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
                    "__array_interface__ strides not fully supported yet") 
-        arr = frombuffer(space, data, dtype, support.product(shape), offset)
+        arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
         
@@ -110,6 +114,78 @@
             return None, False
         raise
 
+def _descriptor_from_pep3118_format(space, c_format):
+    descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+    if descr:
+        return descr
+    msg = "invalid PEP 3118 format string: '%s'" % c_format
+    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+    return None 
+
+def _array_from_buffer_3118(space, w_object, dtype):
+    try:
+        w_buf = space.call_method(space.builtin, "memoryview", w_object)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            # object does not have buffer interface
+            return w_object
+        raise
+    format = space.getattr(w_buf,space.newbytes('format'))
+    if format:
+        descr = _descriptor_from_pep3118_format(space, space.str_w(format))
+        if not descr:
+            return w_object
+        if dtype and descr:
+            raise oefmt(space.w_NotImplementedError,
+                "creating an array from a memoryview while specifying dtype "
+                "not supported")
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
+            msg = ("Item size computed from the PEP 3118 buffer format "
+                  "string does not match the actual item size.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+        dtype = descr 
+    elif not dtype:
+        dtype = descriptor.get_dtype_cache(space).w_stringdtype
+        dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
+    nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim')))
+    shape = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('shape')))]
+    strides = []
+    buflen = space.len_w(w_buf) * dtype.elsize
+    if shape:
+        strides = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('strides')))]
+        if not strides:
+            d = buflen
+            strides = [0] * nd
+            for k in range(nd):
+                if shape[k] > 0:
+                    d /= shape[k]
+                    strides[k] = d
+    else:
+        if nd == 1:
+            shape = [buflen / dtype.elsize, ]
+            strides = [dtype.elsize, ]
+        elif nd > 1:
+            msg = ("ndim computed from the PEP 3118 buffer format "
+                   "is greater than 1, but shape is NULL.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+    try:
+        w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
+    except OperationError as e:
+        if e.match(space, space.w_ValueError):
+            return w_object
+        else:
+            raise e
+    writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
+    w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
+               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
+               writable=writable, strides=strides)
+    if w_ret:
+        return w_ret
+    return w_object
 
 @unwrap_spec(ndmin=int, copy=bool, subok=bool)
 def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -127,6 +203,7 @@
 
 def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
 
+    from pypy.module.micronumpy.boxes import W_GenericBox
     # numpy testing calls array(type(array([]))) and expects a ValueError
     if space.isinstance_w(w_object, space.w_type):
         raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
@@ -134,13 +211,19 @@
     dtype = descriptor.decode_w_dtype(space, w_dtype)
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
-        if w_array is not None:
+        if w_array is None:
+            if (    not space.isinstance_w(w_object, space.w_str) and 
+                    not space.isinstance_w(w_object, space.w_unicode) and
+                    not isinstance(w_object, W_GenericBox)):
+                # use buffer interface
+                w_object = _array_from_buffer_3118(space, w_object, dtype)
+        else:
             # continue with w_array, but do further operations in place
             w_object = w_array
             copy = False
             dtype = w_object.get_dtype()
     if not isinstance(w_object, W_NDimArray):
-        w_array, _copy = try_interface_method(space, w_object)
+        w_array, _copy = try_interface_method(space, w_object, copy)
         if w_array is not None:
             w_object = w_array
             copy = _copy
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -805,19 +805,19 @@
         return w_result
 
     def buffer_w(self, space, flags):
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, flags)
 
     def readbuf_w(self, space):
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO)
 
     def writebuf_w(self, space):
-        return self.implementation.get_buffer(space, False)
+        return self.implementation.get_buffer(space, space.BUF_FULL)
 
     def charbuf_w(self, space):
-        return self.implementation.get_buffer(space, True).as_str()
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
 
     def descr_get_data(self, space):
-        return space.newbuffer(self.implementation.get_buffer(space, False))
+        return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3215,7 +3215,9 @@
         raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)}))
 
         a = array([1, 2, 3])
-        b = array(Dummy(a.__array_interface__))
+        d = Dummy(a.__array_interface__)
+        b = array(d)
+        assert b.base is None
         b[1] = 200
         assert a[1] == 2 # upstream compatibility, is this a bug?
         interface_a = a.__array_interface__
@@ -3226,6 +3228,8 @@
         interface_b.pop('data')
         interface_a.pop('data')
         assert interface_a == interface_b
+        b = array(d, copy=False)
+        assert b.base is d
 
         b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
                          'data': 'a'*100}))
@@ -3594,6 +3598,7 @@
         cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
         cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
         cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+        cls.w_one = cls.space.wrap(struct.pack('i', 1))
 
     def test_frombuffer(self):
         import numpy as np
@@ -3645,8 +3650,6 @@
         else:
             EMPTY = None
         x = np.array([1, 2, 3, 4, 5], dtype='i')
-        y = memoryview('abc')
-        assert y.format == 'B'
         y = memoryview(x)
         assert y.format == 'i'
         assert y.shape == (5,)
@@ -3654,6 +3657,16 @@
         assert y.strides == (4,)
         assert y.suboffsets == EMPTY
         assert y.itemsize == 4
+        assert isinstance(y, memoryview)
+        assert y[0] == self.one
+        assert (np.array(y) == x).all()
+
+        x = np.array([0, 0, 0, 0], dtype='O')
+        y = memoryview(x)
+        # handles conversion of address to pinned object?
+        z = np.array(y)
+        assert z.dtype == 'O'
+        assert (z == x).all()
 
     def test_fromstring(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -702,3 +702,32 @@
         ret = obj.sum()
         print type(ret)
         assert ret.info == 'spam'
+
+    def test_ndarray_subclass_assigns_base(self):
+        import numpy as np
+        init_called = []
+        class _DummyArray(object):
+            """ Dummy object that just exists to hang __array_interface__ dictionaries
+            and possibly keep alive a reference to a base array.
+            """
+            def __init__(self, interface, base=None):
+                self.__array_interface__ = interface
+                init_called.append(1)
+                self.base = base
+
+        x = np.zeros(10)
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 1
+        assert y.base is d
+
+        x = np.zeros((0,), dtype='float32')
+        intf = x.__array_interface__.copy()
+        intf["strides"] = x.strides
+        x.__array_interface__["strides"] = x.strides
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 2
+        assert y.base is d
+
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1851,7 +1851,7 @@
                     arr.gcstruct)
 
     def read(self, arr, i, offset, dtype):
-        if arr.gcstruct is V_OBJECTSTORE:
+        if arr.gcstruct is V_OBJECTSTORE and not arr.base():
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot read object from array with no gc hook")
         return self.box(self._read(arr.storage, i, offset))
diff --git a/pypy/module/pypyjit/test_pypy_c/test_import.py b/pypy/module/pypyjit/test_pypy_c/test_import.py
--- a/pypy/module/pypyjit/test_pypy_c/test_import.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_import.py
@@ -38,3 +38,27 @@
         # call_may_force(absolute_import_with_lock).
         for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")):
             assert 'call' not in opname    # no call-like opcode
+
+    def test_import_fast_path(self, tmpdir):
+        print tmpdir
+        pkg = tmpdir.join('mypkg').ensure(dir=True)
+        subdir = pkg.join("sub").ensure(dir=True)
+        pkg.join('__init__.py').write("")
+        subdir.join('__init__.py').write("")
+        subdir.join('mod.py').write(str(py.code.Source("""
+            def do_the_import():
+                import sys
+        """)))
+        def main(path, n):
+            def do_the_import():
+                from mypkg.sub import mod
+            import sys
+            sys.path.append(path)
+            for i in range(n):
+                do_the_import()
+        #
+        log = self.run(main, [str(tmpdir), 300])
+        loop, = log.loops_by_filename(self.filepath)
+        # check that no string compares and other calls are there
+        for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")):
+            assert 'call' not in opname    # no call-like opcode
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1415,6 +1415,7 @@
         assert p.b == 12
         assert p.c == 14
         assert p.d == 14
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [0, 0, 0, 0])
 
     def test_nested_field_offset_align(self):
         ffi = FFI(backend=self.Backend())
@@ -1454,14 +1455,42 @@
         assert p.b == 0
         assert p.c == 14
         assert p.d == 14
-        p = ffi.new("union foo_u *", {'b': 12})
-        assert p.a == 0
+        p = ffi.new("union foo_u *", {'a': -63, 'b': 12})
+        assert p.a == -63
         assert p.b == 12
-        assert p.c == 0
-        assert p.d == 0
-        # we cannot specify several items in the dict, even though
-        # in theory in this particular case it would make sense
-        # to give both 'a' and 'b'
+        assert p.c == -63
+        assert p.d == -63
+        p = ffi.new("union foo_u *", [123, 456])
+        assert p.a == 123
+        assert p.b == 456
+        assert p.c == 123
+        assert p.d == 123
+        py.test.raises(ValueError, ffi.new, "union foo_u *", [0, 0, 0])
+
+    def test_nested_anonymous_struct_2(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            struct foo_s {
+                int a;
+                union { int b; union { int c, d; }; };
+                int e;
+            };
+        """)
+        assert ffi.sizeof("struct foo_s") == 3 * SIZE_OF_INT
+        p = ffi.new("struct foo_s *", [11, 22, 33])
+        assert p.a == 11
+        assert p.b == p.c == p.d == 22
+        assert p.e == 33
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [11, 22, 33, 44])
+        FOO = ffi.typeof("struct foo_s")
+        fields = [(name, fld.offset, fld.flags) for (name, fld) in FOO.fields]
+        assert fields == [
+            ('a', 0 * SIZE_OF_INT, 0),
+            ('b', 1 * SIZE_OF_INT, 0),
+            ('c', 1 * SIZE_OF_INT, 1),
+            ('d', 1 * SIZE_OF_INT, 1),
+            ('e', 2 * SIZE_OF_INT, 0),
+        ]
 
     def test_cast_to_array_type(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
@@ -35,6 +35,9 @@
     def test_nested_anonymous_union(self):
         py.test.skip("ctypes backend: not supported: nested anonymous union")
 
+    def test_nested_anonymous_struct_2(self):
+        py.test.skip("ctypes backend: not supported: nested anonymous union")
+
     def test_CData_CType_2(self):
         if sys.version_info >= (3,):
             py.test.skip("ctypes backend: not supported in Python 3: CType")
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -64,6 +64,8 @@
 
     def setitem_str(self, w_dict, key, w_value):
         cell = self.getdictvalue_no_unwrapping(w_dict, key)
+        #if (key == '__package__' or key == "__path__") and cell is not None and w_value is not cell:
+        #    print "WARNING", key, w_value, cell, self
         return self._setitem_str_cell_known(cell, w_dict, key, w_value)
 
     def _setitem_str_cell_known(self, cell, w_dict, key, w_value):
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -14,6 +14,7 @@
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
+    _attrs_ = ['buf']
 
     def __init__(self, buf):
         assert isinstance(buf, Buffer)
@@ -115,7 +116,7 @@
             self.buf.setslice(start, value.as_str())
 
     def descr_len(self, space):
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.buf.getlength() / self.buf.getitemsize())
 
     def w_get_format(self, space):
         return space.wrap(self.buf.getformat())
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
 min=4
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Wed Sep  7 10:07:20 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 07 Sep 2016 07:07:20 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: new test to ensure that
 operations that only the necessary index calculations are emitted
Message-ID: <57d01f18.010c1c0a.172b6.c7b6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86934:a867b0a573a4
Date: 2016-09-07 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/a867b0a573a4/

Log:	new test to ensure that operations that only the necessary index
	calculations are emitted

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -1384,5 +1384,23 @@
                 for arg in op.getfailargs():
                     assert not arg.is_constant()
 
+    def test_delay_pure_ops(self):
+        """ Pure operations can be delayed. Often (e.g. for index calc.) this means they can be omitted.
+        """
+        trace = self.parse_loop("""
+        [p0,i0]
+        f0 = raw_load_f(p0, i0, descr=floatarraydescr)
+        i1 = int_add(i0,8)
+        f1 = raw_load_f(p0, i1, descr=floatarraydescr)
+        i2 = int_add(i1,8)
+        jump(p0,i2)
+        """)
+        self.schedule(trace)
+        self.ensure_operations([
+            'v0[2xf64] = vec_load_f(p0, i0, 8, 0, descr=floatarraydescr)',
+            'i2 = int_add(i0, 16)',
+            'jump(p0,i2)',
+        ], trace)
+
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass

From pypy.commits at gmail.com  Wed Sep  7 10:07:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 07 Sep 2016 07:07:16 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: syntax error
Message-ID: <57d01f14.04a81c0a.afd36.741b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86932:ffce6cd646d6
Date: 2016-09-05 16:45 +0200
http://bitbucket.org/pypy/pypy/changeset/ffce6cd646d6/

Log:	syntax error

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -308,7 +308,7 @@
             self.mc.CMPPD_xxi(lhsloc.value, rhsloc.value, 0)
         self.flush_vec_cc(rx86.Conditions["E"], lhsloc, resloc, sizeloc.value)
 
-    def flush_vec_cc(self, rev_cond, lhsloc, resloc, size)
+    def flush_vec_cc(self, rev_cond, lhsloc, resloc, size):
         # After emitting an instruction that leaves a boolean result in
         # a condition code (cc), call this.  In the common case, result_loc
         # will be set to SPP by the regalloc, which in this case means

From pypy.commits at gmail.com  Wed Sep  7 10:12:25 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 07:12:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix translation
Message-ID: <57d02049.45c8c20a.4d79b.5d6e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86935:812903497ffc
Date: 2016-09-07 15:11 +0100
http://bitbucket.org/pypy/pypy/changeset/812903497ffc/

Log:	fix translation

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -26,7 +26,6 @@
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
-    _attrs_ = ['buf']
 
     def __init__(self, buf, format=None, itemsize=1, ndim=-1,
                  shape=None, strides=None, suboffsets=None):

From pypy.commits at gmail.com  Wed Sep  7 10:14:37 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 07 Sep 2016 07:14:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix __all__ code
Message-ID: <57d020cd.53b81c0a.262cd.7d35@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86936:02dc0dd6309c
Date: 2016-09-07 14:57 +0200
http://bitbucket.org/pypy/pypy/changeset/02dc0dd6309c/

Log:	fix __all__ code

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -358,6 +358,7 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     w_fromlist = w_all
+                    length = space.len_w(w_fromlist)
                 else:
                     w_fromlist = None
                     # "from x import *" with x already imported and no x.__all__
@@ -409,6 +410,7 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     w_fromlist = w_all
+                    length = space.len_w(w_fromlist)
                 else:
                     w_fromlist = None
             if w_fromlist is not None:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -65,8 +65,9 @@
              )
     setuppkg("pkg.pkg2", a='', b='')
     setuppkg("pkg.withall",
-             __init__  = "__all__ = ['foobar']",
-             foobar    = "found = 123")
+             __init__  = "__all__ = ['foobar', 'barbaz']",
+             foobar    = "found = 123",
+             barbaz    = "other = 543")
     setuppkg("pkg.withoutall",
              __init__  = "",
              foobar    = "found = 123")
@@ -707,6 +708,7 @@
             d = {}
             exec "from pkg.withall import *" in d
             assert d["foobar"].found == 123
+            assert d["barbaz"].other == 543
 
     def test_import_star_does_not_find_submodules_without___all__(self):
         for case in ["not-imported-yet", "already-imported"]:

From pypy.commits at gmail.com  Wed Sep  7 10:14:39 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 07 Sep 2016 07:14:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge
Message-ID: <57d020cf.c186c20a.99002.507f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86937:cd6eb8be0310
Date: 2016-09-07 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/cd6eb8be0310/

Log:	merge

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -358,6 +358,7 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     w_fromlist = w_all
+                    length = space.len_w(w_fromlist)
                 else:
                     w_fromlist = None
                     # "from x import *" with x already imported and no x.__all__
@@ -409,6 +410,7 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     w_fromlist = w_all
+                    length = space.len_w(w_fromlist)
                 else:
                     w_fromlist = None
             if w_fromlist is not None:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -65,8 +65,9 @@
              )
     setuppkg("pkg.pkg2", a='', b='')
     setuppkg("pkg.withall",
-             __init__  = "__all__ = ['foobar']",
-             foobar    = "found = 123")
+             __init__  = "__all__ = ['foobar', 'barbaz']",
+             foobar    = "found = 123",
+             barbaz    = "other = 543")
     setuppkg("pkg.withoutall",
              __init__  = "",
              foobar    = "found = 123")
@@ -707,6 +708,7 @@
             d = {}
             exec "from pkg.withall import *" in d
             assert d["foobar"].found == 123
+            assert d["barbaz"].other == 543
 
     def test_import_star_does_not_find_submodules_without___all__(self):
         for case in ["not-imported-yet", "already-imported"]:

From pypy.commits at gmail.com  Wed Sep  7 10:25:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 07:25:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57d02369.88711c0a.4e105.8389@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r86938:5db8cf7e71cd
Date: 2016-09-07 15:24 +0100
http://bitbucket.org/pypy/pypy/changeset/5db8cf7e71cd/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 2403 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -30,3 +30,6 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.2"
+__version_info__ = (1, 8, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,43 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -0,0 +1,64 @@
+==========
+PyPy 5.4.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.4.0, released last week,
+due to the following issues:
+
+  * Update list of contributors in documentation and LICENSE file,
+    this was unfortunately left out of 5.4.0. My apologies to the new
+    contributors
+
+  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+    dynamically loadable file
+
+  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+
+  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+
+  * Fix for an issue where os.access() accepted a float for mode
+
+  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+    the last byte of a unicode string sometimes
+
+  * Update built-in cffi_ to version 1.8.1
+
+  * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
+    translation with a message asking for help porting it
+
+  * Fix a regression where a PyBytesObject was forced (converted to a RPython
+    object) when not required, reported as issue #2395
+
+Thanks to those who reported the issues.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _cffi: https://cffi.readthedocs.io
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,9 @@
 
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1437,6 +1437,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.2"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
                 return True
         return False
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        pass
-
     def convert_from_object(self, cdata, w_ob):
         if not self._copy_from_same(cdata, w_ob):
             self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
     )
     def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self.force_lazy_struct()
-        self._check_only_one_argument_for_union(w_ob)
 
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
-            if len(lst_w) > len(self._fields_list):
-                raise oefmt(space.w_ValueError,
-                            "too many initializers for '%s' (got %d)",
-                            self.name, len(lst_w))
-            for i in range(len(lst_w)):
-                optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+            j = 0
+            for w_obj in lst_w:
+                try:
+                    while (self._fields_list[j].flags &
+                               W_CField.BF_IGNORE_IN_CTOR):
+                        j += 1
+                except IndexError:
+                    raise oefmt(space.w_ValueError,
+                                "too many initializers for '%s' (got %d)",
+                                self.name, len(lst_w))
+                optvarsize = self._fields_list[j].write_v(cdata, w_obj,
                                                           optvarsize)
+                j += 1
             return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
 class W_CTypeUnion(W_CTypeStructOrUnion):
     kind = "union"
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        space = self.space
-        n = space.int_w(space.len(w_ob))
-        if n > 1:
-            raise oefmt(space.w_ValueError,
-                        "initializer for '%s': %d items given, but only one "
-                        "supported (use a dict if needed)", self.name, n)
-
 
 class W_CField(W_Root):
     _immutable_ = True
@@ -200,18 +194,21 @@
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
 
-    def __init__(self, ctype, offset, bitshift, bitsize):
+    BF_IGNORE_IN_CTOR = 0x01
+
+    def __init__(self, ctype, offset, bitshift, bitsize, flags):
         self.ctype = ctype
         self.offset = offset
         self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
         self.bitsize = bitsize
+        self.flags = flags       # BF_xxx
 
     def is_bitfield(self):
         return self.bitshift >= 0
 
-    def make_shifted(self, offset):
+    def make_shifted(self, offset, fflags):
         return W_CField(self.ctype, offset + self.offset,
-                        self.bitshift, self.bitsize)
+                        self.bitshift, self.bitsize, self.flags | fflags)
 
     def read(self, cdata):
         cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
     offset = interp_attrproperty('offset', W_CField),
     bitshift = interp_attrproperty('bitshift', W_CField),
     bitsize = interp_attrproperty('bitsize', W_CField),
+    flags = interp_attrproperty('flags', W_CField),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
         if alignment < falign and do_align:
             alignment = falign
         #
+        if is_union and i > 0:
+            fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+        else:
+            fflags = 0
+        #
         if fbitsize < 0:
             # not a bitfield: common case
 
@@ -372,7 +377,7 @@
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8)
+                    fld = srcfld.make_shifted(boffset // 8, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                                           fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
@@ -489,7 +495,7 @@
                     bitshift = 8 * ftype.size - fbitsize- bitshift
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
-                                           bitshift, fbitsize)
+                                           bitshift, fbitsize, fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -121,7 +121,7 @@
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
 """.split()
 for name in constant_names:
@@ -647,6 +647,9 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.UCHAR),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,20 +1,66 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import buffer
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_buffer)
 from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
-# PyObject_GetBuffer has been removed, it is defined in abstract.c
-# PyObject_CheckBuffer is also already defined
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+        
 
 @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
+def PyBuffer_IsContiguous(space, view, fort):
     """Return 1 if the memory defined by the view is C-style (fortran is
     'C') or Fortran-style (fortran is 'F') contiguous or either one
     (fortran is 'A').  Return 0 otherwise."""
-    # PyPy only supports contiguous Py_buffers for now.
-    return 1
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
+    
 
 class CBuffer(buffer.Buffer):
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -14,45 +14,33 @@
 ## Implementation of PyBytesObject
 ## ================================
 ##
-## The problem
-## -----------
+## PyBytesObject has its own ob_sval buffer, so we have two copies of a string;
+## one in the PyBytesObject returned from various C-API functions and another
+## in the corresponding RPython object.
 ##
-## PyBytes_AsString() must return a (non-movable) pointer to the underlying
-## ob_sval, whereas pypy strings are movable.  C code may temporarily store
-## this address and use it, as long as it owns a reference to the PyObject.
-## There is no "release" function to specify that the pointer is not needed
-## any more.
+## The following calls can create a PyBytesObject without a correspoinding
+## RPython object:
 ##
-## Also, the pointer may be used to fill the initial value of string. This is
-## valid only when the string was just allocated, and is not used elsewhere.
+## PyBytes_FromStringAndSize(NULL, n) / PyString_FromStringAndSize(NULL, n)
 ##
-## Solution
-## --------
+## In the PyBytesObject returned, the ob_sval buffer may be modified as
+## long as the freshly allocated PyBytesObject is not "forced" via a call
+## to any of the more sophisticated C-API functions. 
 ##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
-##
-## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   NULL buffer.  The first time PyBytes_AsString() is called, memory is
-##   allocated (with flavor='raw') and content is copied.
-##
-## - A string allocated with PyBytes_FromStringAndSize(NULL, size) will
-##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, but the reference won't be stored in the global map; there is no
-##   corresponding object in pypy.  When from_ref() or Py_INCREF() is called,
-##   the pypy string is created, and added to the global map of tracked
-##   objects.  The buffer is then supposed to be immutable.
-##
-##-  A buffer obtained from PyBytes_AS_STRING() could be mutable iff
-##   there is no corresponding pypy object for the string
-##
-## - _PyBytes_Resize() works only on not-yet-pypy'd strings, and returns a
-##   similar object.
-##
-## - PyBytes_Size() doesn't need to force the object.
+## Care has been taken in implementing the functions below, so that
+## if they are called with a non-forced PyBytesObject, they will not 
+## unintentionally force the creation of a RPython object. As long as only these
+## are used, the ob_sval buffer is still modifiable:
+## 
+## PyBytes_AsString / PyString_AsString 
+## PyBytes_AS_STRING / PyString_AS_STRING
+## PyBytes_AsStringAndSize / PyString_AsStringAndSize
+## PyBytes_Size / PyString_Size
+## PyBytes_Resize / PyString_Resize
+## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string.
+##   corresponds to the pypy gc-managed string, 
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -150,9 +138,6 @@
         raise oefmt(space.w_TypeError,
             "expected bytes, %T found", from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not pyobj_has_w_obj(ref):
-        # XXX Force the ref?
-        bytes_realize(space, ref)
     return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
@@ -170,9 +155,6 @@
     if not PyBytes_Check(space, ref):
         raise oefmt(space.w_TypeError,
             "expected bytes, %T found", from_ref(space, ref))
-    if not pyobj_has_w_obj(ref):
-        # force the ref
-        bytes_realize(space, ref)
     ref_str = rffi.cast(PyBytesObject, ref)
     data[0] = ref_str.c_ob_sval
     if length:
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -131,7 +131,8 @@
 typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
 
 
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -145,12 +146,14 @@
     char *format;
     Py_ssize_t *shape;
     Py_ssize_t *strides;
-    Py_ssize_t *suboffsets;
-
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format;
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
        mono-dimensional buffers. */
     /* Py_ssize_t smalltable[2]; */
-    void *internal;
+    void *internal; /* always NULL for app-level objects */
 } Py_buffer;
 
 
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.1-alpha0"
-#define PYPY_VERSION_NUM  0x05040100
+#define PYPY_VERSION "5.5.0-alpha0"
+#define PYPY_VERSION_NUM  0x05050000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,7 +1,8 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.pyobject import PyObject, from_ref
@@ -16,6 +17,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -24,24 +26,38 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-    # TODO - fill in fields
-    '''
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+    if not isinstance(w_obj, W_MemoryView):
+        return view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_obj)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        isstr = False
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        rffi.setintfield(view, 'c_readonly', 1)
+        isstr = True
+    view.c_len = w_obj.getlength()
+    view.c_itemsize = w_obj.buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    shape = w_obj.buf.getshape()
+    strides = w_obj.buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
     view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
     view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    ''' 
     return view
 
-
 @cpython_api([lltype.Ptr(Py_buffer)], PyObject)
 def PyMemoryView_FromBuffer(space, view):
     """Create a memoryview object wrapping the given buffer structure view.
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
     def getshape(self):
         return self.shape
 
+    def getstrides(self):
+        return self.strides
+
     def getitemsize(self):
         return self.itemsize
 
+    def getndim(self):
+        return self.ndim
+
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
   PyMyArray* self = (PyMyArray*)obj;
-  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
-    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
-    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
@@ -188,7 +185,131 @@
     (initproc)PyMyArray_init,     /* tp_init */
 };
 
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+    Py_buffer* view = NULL;
+    PyObject* obj = PyTuple_GetItem(args, 0);
+    PyObject* memoryview = PyMemoryView_FromObject(obj);
+    if (memoryview == NULL)
+        return PyInt_FromLong(-1);
+    view = PyMemoryView_GET_BUFFER(memoryview);
+    Py_DECREF(memoryview);
+    return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG)                                        \
+    buf_flag = PyUnicode_FromString(#FLAG);                         \
+    flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ);  \
+    Py_DECREF(buf_flag);                                            \
+    if (flag_matches == 1) {                                        \
+        Py_DECREF(tmp);                                             \
+        flags |= PyBUF_##FLAG;                                      \
+        continue;                                                   \
+    }                                                               \
+    else if (flag_matches == -1) {                                  \
+        Py_DECREF(tmp);                                             \
+        return NULL;                                                \
+    }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+    PyObject *buffer_obj, *pyflags;
+    PyObject *tmp, *buf_flag;
+    Py_buffer buffer;
+    PyObject *shape, *strides;
+    Py_ssize_t i, n;
+    int flag_matches;
+    int flags = 0;
+
+    if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+        return NULL;
+    }
+
+    n = PySequence_Length(pyflags);
+    if (n < 0) {
+        return NULL;
+    }
+
+    for (i=0; i < n; i++) {
+        tmp = PySequence_GetItem(pyflags, i);
+        if (tmp == NULL) {
+            return NULL;
+        }
+
+        GET_PYBUF_FLAG(SIMPLE);
+        GET_PYBUF_FLAG(WRITABLE);
+        GET_PYBUF_FLAG(STRIDES);
+        GET_PYBUF_FLAG(ND);
+        GET_PYBUF_FLAG(C_CONTIGUOUS);
+        GET_PYBUF_FLAG(F_CONTIGUOUS);
+        GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+        GET_PYBUF_FLAG(INDIRECT);
+        GET_PYBUF_FLAG(FORMAT);
+        GET_PYBUF_FLAG(STRIDED);
+        GET_PYBUF_FLAG(STRIDED_RO);
+        GET_PYBUF_FLAG(RECORDS);
+        GET_PYBUF_FLAG(RECORDS_RO);
+        GET_PYBUF_FLAG(FULL);
+        GET_PYBUF_FLAG(FULL_RO);
+        GET_PYBUF_FLAG(CONTIG);
+        GET_PYBUF_FLAG(CONTIG_RO);
+
+        Py_DECREF(tmp);
+
+        /* One of the flags must match */
+        PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+        return NULL;
+    }
+
+    if (buffer.shape == NULL) {
+        Py_INCREF(Py_None);
+        shape = Py_None;
+    }
+    else {
+        shape = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+        }
+    }
+
+    if (buffer.strides == NULL) {
+        Py_INCREF(Py_None);
+        strides = Py_None;
+    }
+    else {
+        strides = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+        }
+    }
+
+    PyBuffer_Release(&buffer);
+    return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
 static PyMethodDef buffer_functions[] = {
+    {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
+    {"get_buffer_info",   (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -179,8 +179,27 @@
                  Py_INCREF(Py_None);
                  return Py_None;
              """),
+            ("c_only", "METH_NOARGS",
+            """
+                int ret;
+                char * buf2;
+                PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+                if (!obj)
+                    return NULL;
+                buf2 = PyBytes_AsString(obj);
+                if (!buf2)
+                    return NULL;
+                /* buf should not have been forced, issue #2395 */
+                ret = _PyBytes_Resize(&obj, 512);
+                if (ret < 0)
+                    return NULL;
+                 Py_DECREF(obj);
+                 Py_INCREF(Py_None);
+                 return Py_None;
+            """),
             ])
         module.getbytes()
+        module.c_only()
 
 
 class TestBytes(BaseApiTest):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
@@ -11,6 +11,13 @@
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
+    def test_frombuffer(self, space, api):
+        w_buf = space.newbuffer(StringBuffer("hello"))
+        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        ndim = w_view.c_ndim
+        assert ndim == 1
+
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     def test_fillWithObject(self):
         module = self.import_extension('foo', [
@@ -62,6 +69,25 @@
         y = memoryview(arr)
         assert y.format == 'i'
         assert y.shape == (10,)
+        assert len(y) == 10
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
+        viewlen = module.test_buffer(arr)
+        assert viewlen == y.itemsize * len(y)
+
+    def test_buffer_info(self):
+        from _numpypy import multiarray as np
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        # test_export_flags from numpy test_multiarray
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        # test_relaxed_strides from numpy test_multiarray
+        arr = np.zeros((1, 10))
+        if arr.flags.f_contiguous:
+            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+            assert strides[0] == 8
+            arr = np.ones((10, 1), order='F')
+            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+            assert strides[-1] == 8
+
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -41,9 +41,11 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
                     STRUCT_TYPE = PySequenceMethods
+                elif slot_names[0] == 'c_tp_as_buffer':
+                    STRUCT_TYPE = PyBufferProcs
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
     def getdictvalue(self, space, key):
         return self.items[key]
 
+    def descr_memoryview(self, space, buf):
+        raise oefmt(space.w_TypeError, "error")
+
 class IterDictObject(W_Root):
     def __init__(self, space, w_dict):
         self.space = space
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -377,7 +377,25 @@
     def __exit__(self, typ, value, traceback):
         keepalive_until_here(self)
 
-    def get_buffer(self, space, readonly):
+    def get_buffer(self, space, flags):
+        errtype = space.w_ValueError # should be BufferError, numpy does this instead
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_F_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not Fortran contiguous")
+        if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                     self.flags & NPY.ARRAY_C_CONTIGUOUS)):
+           raise oefmt(errtype, "ndarray is not contiguous")
+        if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and
+            not self.flags & NPY.ARRAY_WRITEABLE):
+           raise oefmt(errtype, "buffer source array is read-only")
+        readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
         return ArrayBuffer(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
@@ -695,6 +713,7 @@
                  index + self.impl.start)
 
     def setitem(self, index, v):
+        # XXX what if self.readonly?
         raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstring import strip_spaces
@@ -42,7 +43,7 @@
         raise oefmt(space.w_ValueError,
                     "object __array__ method not producing an array")
 
-def try_interface_method(space, w_object):
+def try_interface_method(space, w_object, copy):
     try:
         w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
         if w_interface is None:
@@ -81,17 +82,20 @@
             raise oefmt(space.w_ValueError,
                     "__array_interface__ could not decode dtype %R", w_dtype
                     )
-        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
+        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
+                                   space.isinstance_w(w_data, space.w_list)):
             data_w = space.listview(w_data)
-            data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
-            read_only = True # XXX why not space.is_true(data_w[1])
+            w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+            read_only = space.is_true(data_w[1]) or copy
             offset = 0
-            return W_NDimArray.from_shape_and_storage(space, shape, data, 
-                                    dtype, strides=strides, start=offset), read_only
+            w_base = w_object
+            if read_only:
+                w_base = None
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+                                dtype, w_base=w_base, strides=strides,
+                                start=offset), read_only
         if w_data is None:
-            data = w_object
-        else:
-            data = w_data
+            w_data = w_object
         w_offset = space.finditem(w_interface, space.wrap('offset'))
         if w_offset is None:
             offset = 0
@@ -101,7 +105,7 @@
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
                    "__array_interface__ strides not fully supported yet") 
-        arr = frombuffer(space, data, dtype, support.product(shape), offset)
+        arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
         
@@ -110,6 +114,78 @@
             return None, False
         raise
 
+def _descriptor_from_pep3118_format(space, c_format):
+    descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+    if descr:
+        return descr
+    msg = "invalid PEP 3118 format string: '%s'" % c_format
+    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+    return None 
+
+def _array_from_buffer_3118(space, w_object, dtype):
+    try:
+        w_buf = space.call_method(space.builtin, "memoryview", w_object)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            # object does not have buffer interface
+            return w_object
+        raise
+    format = space.getattr(w_buf,space.newbytes('format'))
+    if format:
+        descr = _descriptor_from_pep3118_format(space, space.str_w(format))
+        if not descr:
+            return w_object
+        if dtype and descr:
+            raise oefmt(space.w_NotImplementedError,
+                "creating an array from a memoryview while specifying dtype "
+                "not supported")
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
+            msg = ("Item size computed from the PEP 3118 buffer format "
+                  "string does not match the actual item size.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+        dtype = descr 
+    elif not dtype:
+        dtype = descriptor.get_dtype_cache(space).w_stringdtype
+        dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
+    nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim')))
+    shape = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('shape')))]
+    strides = []
+    buflen = space.len_w(w_buf) * dtype.elsize
+    if shape:
+        strides = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('strides')))]
+        if not strides:
+            d = buflen
+            strides = [0] * nd
+            for k in range(nd):
+                if shape[k] > 0:
+                    d /= shape[k]
+                    strides[k] = d
+    else:
+        if nd == 1:
+            shape = [buflen / dtype.elsize, ]
+            strides = [dtype.elsize, ]
+        elif nd > 1:
+            msg = ("ndim computed from the PEP 3118 buffer format "
+                   "is greater than 1, but shape is NULL.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+    try:
+        w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
+    except OperationError as e:
+        if e.match(space, space.w_ValueError):
+            return w_object
+        else:
+            raise e
+    writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
+    w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
+               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
+               writable=writable, strides=strides)
+    if w_ret:
+        return w_ret
+    return w_object
 
 @unwrap_spec(ndmin=int, copy=bool, subok=bool)
 def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -127,6 +203,7 @@
 
 def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
 
+    from pypy.module.micronumpy.boxes import W_GenericBox
     # numpy testing calls array(type(array([]))) and expects a ValueError
     if space.isinstance_w(w_object, space.w_type):
         raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
@@ -134,13 +211,19 @@
     dtype = descriptor.decode_w_dtype(space, w_dtype)
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
-        if w_array is not None:
+        if w_array is None:
+            if (    not space.isinstance_w(w_object, space.w_str) and 
+                    not space.isinstance_w(w_object, space.w_unicode) and
+                    not isinstance(w_object, W_GenericBox)):
+                # use buffer interface
+                w_object = _array_from_buffer_3118(space, w_object, dtype)
+        else:
             # continue with w_array, but do further operations in place
             w_object = w_array
             copy = False
             dtype = w_object.get_dtype()
     if not isinstance(w_object, W_NDimArray):
-        w_array, _copy = try_interface_method(space, w_object)
+        w_array, _copy = try_interface_method(space, w_object, copy)
         if w_array is not None:
             w_object = w_array
             copy = _copy
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -806,10 +806,10 @@
 
     def buffer_w(self, space, flags):
         # XXX format isn't always 'B' probably
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, flags)
 
     def descr_get_data(self, space):
-        return space.newbuffer(self.implementation.get_buffer(space, False))
+        return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3206,7 +3206,9 @@
         raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)}))
 
         a = array([1, 2, 3])
-        b = array(Dummy(a.__array_interface__))
+        d = Dummy(a.__array_interface__)
+        b = array(d)
+        assert b.base is None
         b[1] = 200
         assert a[1] == 2 # upstream compatibility, is this a bug?
         interface_a = a.__array_interface__
@@ -3217,6 +3219,8 @@
         interface_b.pop('data')
         interface_a.pop('data')
         assert interface_a == interface_b
+        b = array(d, copy=False)
+        assert b.base is d
 
         b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
                          'data': 'a'*100}))
@@ -3585,6 +3589,7 @@
         cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
         cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
         cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+        cls.w_one = cls.space.wrap(struct.pack('i', 1))
 
     def test_frombuffer(self):
         import numpy as np
@@ -3636,8 +3641,6 @@
         else:
             EMPTY = None
         x = np.array([1, 2, 3, 4, 5], dtype='i')
-        y = memoryview('abc')
-        assert y.format == 'B'
         y = memoryview(x)
         assert y.format == 'i'
         assert y.shape == (5,)
@@ -3645,6 +3648,16 @@
         assert y.strides == (4,)
         assert y.suboffsets == EMPTY
         assert y.itemsize == 4
+        assert isinstance(y, memoryview)
+        assert y[0] == self.one
+        assert (np.array(y) == x).all()
+
+        x = np.array([0, 0, 0, 0], dtype='O')
+        y = memoryview(x)
+        # handles conversion of address to pinned object?
+        z = np.array(y)
+        assert z.dtype == 'O'
+        assert (z == x).all()
 
     def test_fromstring(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -702,3 +702,32 @@
         ret = obj.sum()
         print type(ret)
         assert ret.info == 'spam'
+
+    def test_ndarray_subclass_assigns_base(self):
+        import numpy as np
+        init_called = []
+        class _DummyArray(object):
+            """ Dummy object that just exists to hang __array_interface__ dictionaries
+            and possibly keep alive a reference to a base array.
+            """
+            def __init__(self, interface, base=None):
+                self.__array_interface__ = interface
+                init_called.append(1)
+                self.base = base
+
+        x = np.zeros(10)
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 1
+        assert y.base is d
+
+        x = np.zeros((0,), dtype='float32')
+        intf = x.__array_interface__.copy()
+        intf["strides"] = x.strides
+        x.__array_interface__["strides"] = x.strides
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 2
+        assert y.base is d
+
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1851,7 +1851,7 @@
                     arr.gcstruct)
 
     def read(self, arr, i, offset, dtype):
-        if arr.gcstruct is V_OBJECTSTORE:
+        if arr.gcstruct is V_OBJECTSTORE and not arr.base():
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot read object from array with no gc hook")
         return self.box(self._read(arr.storage, i, offset))
diff --git a/pypy/module/pypyjit/test_pypy_c/test_import.py b/pypy/module/pypyjit/test_pypy_c/test_import.py
--- a/pypy/module/pypyjit/test_pypy_c/test_import.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_import.py
@@ -38,3 +38,27 @@
         # call_may_force(absolute_import_with_lock).
         for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")):
             assert 'call' not in opname    # no call-like opcode
+
+    def test_import_fast_path(self, tmpdir):
+        print tmpdir
+        pkg = tmpdir.join('mypkg').ensure(dir=True)
+        subdir = pkg.join("sub").ensure(dir=True)
+        pkg.join('__init__.py').write("")
+        subdir.join('__init__.py').write("")
+        subdir.join('mod.py').write(str(py.code.Source("""
+            def do_the_import():
+                import sys
+        """)))
+        def main(path, n):
+            def do_the_import():
+                from mypkg.sub import mod
+            import sys
+            sys.path.append(path)
+            for i in range(n):
+                do_the_import()
+        #
+        log = self.run(main, [str(tmpdir), 300])
+        loop, = log.loops_by_filename(self.filepath)
+        # check that no string compares and other calls are there
+        for opname in log.opnames(loop.allops(opcode="IMPORT_NAME")):
+            assert 'call' not in opname    # no call-like opcode
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -187,6 +187,43 @@
 
         """)
 
+    def test_oldstyle_methcall(self):
+        def main():
+            def g(): pass
+            class A:
+                def f(self):
+                    return self.x + 1
+            class I(A):
+                pass
+            class J(I):
+                pass
+
+
+            class B(J):
+                def __init__(self, x):
+                    self.x = x
+
+            i = 0
+            b = B(1)
+            while i < 1000:
+                g()
+                v = b.f() # ID: meth
+                i += v
+            return i
+
+        log = self.run(main, [], threshold=80)
+        loop, = log.loops_by_filename(self.filepath, is_entry_bridge=True)
+        assert loop.match_by_id('meth',
+        '''
+    guard_nonnull_class(p18, ..., descr=...)
+    p52 = getfield_gc_r(p18, descr=...) # read map
+    guard_value(p52, ConstPtr(ptr53), descr=...)
+    p54 = getfield_gc_r(p18, descr=...) # read class
+    guard_value(p54, ConstPtr(ptr55), descr=...)
+    p56 = force_token() # done
+        ''')
+
+
     def test_oldstyle_newstyle_mix(self):
         def main():
             class A:
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 4, 1, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 5, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1415,6 +1415,7 @@
         assert p.b == 12
         assert p.c == 14
         assert p.d == 14
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [0, 0, 0, 0])
 
     def test_nested_field_offset_align(self):
         ffi = FFI(backend=self.Backend())
@@ -1454,14 +1455,42 @@
         assert p.b == 0
         assert p.c == 14
         assert p.d == 14
-        p = ffi.new("union foo_u *", {'b': 12})
-        assert p.a == 0
+        p = ffi.new("union foo_u *", {'a': -63, 'b': 12})
+        assert p.a == -63
         assert p.b == 12
-        assert p.c == 0
-        assert p.d == 0
-        # we cannot specify several items in the dict, even though
-        # in theory in this particular case it would make sense
-        # to give both 'a' and 'b'
+        assert p.c == -63
+        assert p.d == -63
+        p = ffi.new("union foo_u *", [123, 456])
+        assert p.a == 123
+        assert p.b == 456
+        assert p.c == 123
+        assert p.d == 123
+        py.test.raises(ValueError, ffi.new, "union foo_u *", [0, 0, 0])
+
+    def test_nested_anonymous_struct_2(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            struct foo_s {
+                int a;
+                union { int b; union { int c, d; }; };
+                int e;
+            };
+        """)
+        assert ffi.sizeof("struct foo_s") == 3 * SIZE_OF_INT
+        p = ffi.new("struct foo_s *", [11, 22, 33])
+        assert p.a == 11
+        assert p.b == p.c == p.d == 22
+        assert p.e == 33
+        py.test.raises(ValueError, ffi.new, "struct foo_s *", [11, 22, 33, 44])
+        FOO = ffi.typeof("struct foo_s")
+        fields = [(name, fld.offset, fld.flags) for (name, fld) in FOO.fields]
+        assert fields == [
+            ('a', 0 * SIZE_OF_INT, 0),
+            ('b', 1 * SIZE_OF_INT, 0),
+            ('c', 1 * SIZE_OF_INT, 1),
+            ('d', 1 * SIZE_OF_INT, 1),
+            ('e', 2 * SIZE_OF_INT, 0),
+        ]
 
     def test_cast_to_array_type(self):
         ffi = FFI(backend=self.Backend())
@@ -1479,6 +1508,7 @@
             assert p1[0] == 123
             seen.append(1)
         q = ffi.gc(p, destructor)
+        assert ffi.typeof(q) is ffi.typeof(p)
         import gc; gc.collect()
         assert seen == []
         del q
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ctypes.py
@@ -35,6 +35,9 @@
     def test_nested_anonymous_union(self):
         py.test.skip("ctypes backend: not supported: nested anonymous union")
 
+    def test_nested_anonymous_struct_2(self):
+        py.test.skip("ctypes backend: not supported: nested anonymous union")
+
     def test_CData_CType_2(self):
         if sys.version_info >= (3,):
             py.test.skip("ctypes backend: not supported in Python 3: CType")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -149,3 +149,28 @@
         p = snip_setuptools_verify2.C.getpwuid(0)
         assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
         ''')
+
+    def test_set_py_limited_api(self):
+        from cffi.setuptools_ext import _set_py_limited_api
+        try:
+            import setuptools
+        except ImportError as e:
+            py.test.skip(str(e))
+        orig_version = setuptools.__version__
+        try:
+            setuptools.__version__ = '26.0.0'
+            from setuptools import Extension
+
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+            setuptools.__version__ = '25.0'
+            kwds = _set_py_limited_api(Extension, {})
+            assert not kwds
+
+            setuptools.__version__ = 'development'
+            kwds = _set_py_limited_api(Extension, {})
+            assert kwds['py_limited_api'] == True
+
+        finally:
+            setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1975,9 +1975,9 @@
 
 def test_function_returns_partial_struct():
     ffi = FFI()
-    ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+    ffi.cdef("struct aaa { int a; ...; }; struct aaa f1(int);")
     lib = verify(ffi, "test_function_returns_partial_struct", """
-        struct a { int b, a, c; };
-        static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+        struct aaa { int b, a, c; };
+        static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
@@ -194,6 +194,29 @@
             _fields_ = [('t', enum)]
         assert isinstance(S().t, enum)
 
+    def test_no_missing_shape_to_ffi_type(self):
+        # whitebox test
+        import sys
+        if '__pypy__' not in sys.builtin_module_names:
+            skip("only for pypy's ctypes")
+        skip("re-enable after adding 'g' to _shape_to_ffi_type.typemap, "
+             "which I think needs fighting all the way up from "
+             "rpython.rlib.libffi")
+        from _ctypes.basics import _shape_to_ffi_type
+        from _rawffi import Array
+        for i in range(1, 256):
+            try:
+                Array(chr(i))
+            except ValueError:
+                pass
+            else:
+                assert chr(i) in _shape_to_ffi_type.typemap
+
+    @py.test.mark.xfail
+    def test_pointer_to_long_double(self):
+        import ctypes
+        ctypes.POINTER(ctypes.c_longdouble)
+
 ##    def test_perf(self):
 ##        check_perf()
 
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -64,6 +64,8 @@
 
     def setitem_str(self, w_dict, key, w_value):
         cell = self.getdictvalue_no_unwrapping(w_dict, key)
+        #if (key == '__package__' or key == "__path__") and cell is not None and w_value is not cell:
+        #    print "WARNING", key, w_value, cell, self
         return self._setitem_str_cell_known(cell, w_dict, key, w_value)
 
     def _setitem_str_cell_known(self, cell, w_dict, key, w_value):
diff --git a/pypy/objspace/std/test/test_random_attr.py b/pypy/objspace/std/test/test_random_attr.py
--- a/pypy/objspace/std/test/test_random_attr.py
+++ b/pypy/objspace/std/test/test_random_attr.py
@@ -1,4 +1,5 @@
 import pytest
+pytest.skip("This cannot possibly work on pypy3")
 import sys
 try:
     import __pypy__
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
 min=4
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -164,8 +164,15 @@
             # annotations that are passed in, and don't annotate the old
             # graph -- it's already low-level operations!
             for a, s_newarg in zip(block.inputargs, cells):
-                s_oldarg = self.binding(a)
-                assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
+                s_oldarg = a.annotation
+                # XXX: Should use s_oldarg.contains(s_newarg) but that breaks
+                # PyPy translation
+                if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg:
+                    raise annmodel.AnnotatorError(
+                        "Late-stage annotation is not allowed to modify the "
+                        "existing annotation for variable %s: %s" %
+                            (a, s_oldarg))
+
         else:
             assert not self.frozen
             if block not in self.annotated:
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -17,7 +17,7 @@
 from rpython.flowspace.model import Variable, Constant, const
 from rpython.flowspace.operation import op
 from rpython.rlib import rarithmetic
-from rpython.annotator.model import AnnotatorError
+from rpython.annotator.model import AnnotatorError, TLS
 
 BINARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values()
                         if oper.dispatch == 2])
@@ -436,6 +436,11 @@
 class __extend__(pairtype(SomeFloat, SomeFloat)):
 
     def union((flt1, flt2)):
+        if not TLS.allow_int_to_float:
+            # in this mode, if one of the two is actually the
+            # subclass SomeInteger, complain
+            if isinstance(flt1, SomeInteger) or isinstance(flt2, SomeInteger):
+                raise UnionError(flt1, flt2)
         return SomeFloat()
 
     add = sub = mul = union
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -44,6 +44,7 @@
     # A global attribute :-(  Patch it with 'True' to enable checking of
     # the no_nul attribute...
     check_str_without_nul = False
+    allow_int_to_float = True
 TLS = State()
 
 class SomeObject(object):
@@ -749,6 +750,7 @@
                 s1 = pair(s1, s2).union()
     else:
         # this is just a performance shortcut
+        # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
         if s1 != s2:
             s1 = pair(s1, s2).union()
     return s1
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -47,6 +47,8 @@
         cls.ulonglong = clibffi.cast_type_to_ffitype(rffi.ULONGLONG)
         cls.signed = clibffi.cast_type_to_ffitype(rffi.SIGNED)
         cls.wchar_t = clibffi.cast_type_to_ffitype(lltype.UniChar)
+        # XXX long double support: clibffi.ffi_type_longdouble, but then
+        # XXX fix the whole rest of this file to add a case for long double
         del cls._import
 
     @staticmethod
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -98,8 +98,15 @@
         try:
             ctypes.CDLL(name)
         except OSError as e:
+            # common case: ctypes fails too, with the real dlerror()
+            # message in str(e).  Return that error message.
             return str(e)
         else:
+            # uncommon case: may happen if 'name' is a linker script
+            # (which the C-level dlopen() can't handle) and we are
+            # directly running on pypy (whose implementation of ctypes
+            # or cffi will resolve linker scripts).  In that case, 
+            # unsure what we can do.
             return ("opening %r with ctypes.CDLL() works, "
                     "but not with c_dlopen()??" % (name,))
 
@@ -160,7 +167,18 @@
             mode = _dlopen_default_mode()
         elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
             mode |= RTLD_NOW
+        #
+        # haaaack for 'pypy py.test -A' if libm.so is a linker script
+        # (see reason in _dlerror_on_dlopen_untranslated())
+        must_free = False
+        if not we_are_translated() and platform.name == "linux":
+            if name and rffi.charp2str(name) == 'libm.so':
+                name = rffi.str2charp('libm.so.6')
+                must_free = True
+        #
         res = c_dlopen(name, rffi.cast(rffi.INT, mode))
+        if must_free:
+            rffi.free_charp(name)
         if not res:
             if not we_are_translated():
                 err = _dlerror_on_dlopen_untranslated(name)
diff --git a/rpython/rlib/rmarshal.py b/rpython/rlib/rmarshal.py
--- a/rpython/rlib/rmarshal.py
+++ b/rpython/rlib/rmarshal.py
@@ -346,11 +346,15 @@
     # on s_bigger.  It relies on the fact that s_bigger was created with
     # an expression like 'annotation([s_item])' which returns a ListDef with
     # no bookkeeper, on which side-effects are not allowed.
+    saved = annmodel.TLS.allow_int_to_float
     try:
+        annmodel.TLS.allow_int_to_float = False
         s_union = annmodel.unionof(s_bigger, s_smaller)
         return s_bigger.contains(s_union)
     except (annmodel.UnionError, TooLateForChange):
         return False
+    finally:
+        annmodel.TLS.allow_int_to_float = saved
 
 
 class __extend__(pairtype(MTag, annmodel.SomeObject)):
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,6 +97,21 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
+if OPENSSL_VERSION_NUMBER >= 0x10100000:
+    eci.pre_include_bits = ()
+    eci.post_include_bits = ()
+    raise Exception("""OpenSSL version >= 1.1 not supported yet.
+
+    This program requires OpenSSL version 1.0.x, and may also
+    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
+    some work to update to; contributions are welcome.  Sorry,
+    you need to install an older version of OpenSSL for now.
+    Make sure this older version is the one picked up by this
+    program when it runs the compiler.
+    
+    This is the configuration used: %r""" % (eci,))
+
+
 class CConfig:
     _compilation_info_ = eci
 
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1,5 +1,5 @@
 import sys
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, enforceargs
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.rarithmetic import r_uint, intmask, widen
 from rpython.rlib.unicodedata import unicodedb
@@ -137,7 +137,29 @@
                                  result=result)
     return result.build(), pos
 
- at specialize.argtype(6)
+def _invalid_cont_byte(ordch):
+    return ordch>>6 != 0x2    # 0b10
+
+_invalid_byte_2_of_2 = _invalid_cont_byte
+_invalid_byte_3_of_3 = _invalid_cont_byte
+_invalid_byte_3_of_4 = _invalid_cont_byte
+_invalid_byte_4_of_4 = _invalid_cont_byte
+
+ at enforceargs(allow_surrogates=bool)
+def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xe0 and ordch2 < 0xa0)
+            # surrogates shouldn't be valid UTF-8!
+            or (ordch1 == 0xed and ordch2 > 0x9f and not allow_surrogates))
+
+def _invalid_byte_2_of_4(ordch1, ordch2):
+    return (ordch2>>6 != 0x2 or    # 0b10
+            (ordch1 == 0xf0 and ordch2 < 0x90) or
+            (ordch1 == 0xf4 and ordch2 > 0x8f))
+
+# note: this specialize() is here for rtyper/rstr.py, which calls this
+# function too but with its own fixed errorhandler
+ at specialize.arg_or_var(4)
 def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
                           allow_surrogates, result):
     if size == 0:
@@ -157,22 +179,23 @@
         if pos + n > size:
             if not final:
                 break
+            # argh, this obscure block of code is mostly a copy of
+            # what follows :-(
             charsleft = size - pos - 1 # either 0, 1, 2
-            # note: when we get the 'unexpected end of data' we don't care
-            # about the pos anymore and we just ignore the value
+            # note: when we get the 'unexpected end of data' we need
+            # to care about the pos returned; it can be lower than size,
+            # in case we need to continue running this loop
             if not charsleft:
                 # there's only the start byte and nothing else
                 r, pos = errorhandler(errors, 'utf8',
                                       'unexpected end of data',
                                       s, pos, pos+1)
                 result.append(r)
-                break

From pypy.commits at gmail.com  Wed Sep  7 10:30:47 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 07:30:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fix bytebuffer test
Message-ID: <57d02497.4a301c0a.60059.7ce2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86939:c68432b5be9f
Date: 2016-09-07 15:29 +0100
http://bitbucket.org/pypy/pypy/changeset/c68432b5be9f/

Log:	fix bytebuffer test

diff --git a/pypy/module/__pypy__/test/test_bytebuffer.py b/pypy/module/__pypy__/test/test_bytebuffer.py
--- a/pypy/module/__pypy__/test/test_bytebuffer.py
+++ b/pypy/module/__pypy__/test/test_bytebuffer.py
@@ -14,12 +14,9 @@
         assert b[-3] == ord(b'+')
         exc = raises(ValueError, "b[3:5] = b'abc'")
         assert str(exc.value) == "cannot modify size of memoryview object"
-        raises(NotImplementedError, "b[3:7:2] = b'abc'")
 
         b = bytebuffer(10)
         b[1:3] = b'xy'
         assert bytes(b) == b"\x00xy" + b"\x00" * 7
-        # XXX: supported in 3.3
-        raises(NotImplementedError, "b[4:8:2] = b'zw'")
-        #b[4:8:2] = b'zw'
-        #assert bytes(b) == b"\x00xy\x00z\x00w" + b"\x00" * 3
+        b[4:8:2] = b'zw'
+        assert bytes(b) == b"\x00xy\x00z\x00w" + b"\x00" * 3

From pypy.commits at gmail.com  Wed Sep  7 10:51:09 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 07:51:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: 2to3fy cpyext's test_memoryobject.py
Message-ID: <57d0295d.081dc20a.e249d.bf82@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r86940:aa12ac889991
Date: 2016-09-07 15:50 +0100
http://bitbucket.org/pypy/pypy/changeset/aa12ac889991/

Log:	2to3fy cpyext's test_memoryobject.py

diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -192,10 +192,10 @@
     PyObject* obj = PyTuple_GetItem(args, 0);
     PyObject* memoryview = PyMemoryView_FromObject(obj);
     if (memoryview == NULL)
-        return PyInt_FromLong(-1);
+        return PyLong_FromLong(-1);
     view = PyMemoryView_GET_BUFFER(memoryview);
     Py_DECREF(memoryview);
-    return PyInt_FromLong(view->len);
+    return PyLong_FromLong(view->len);
 }
 
 /* Copied from numpy tests */
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,8 @@
+import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
@@ -76,6 +78,7 @@
         viewlen = module.test_buffer(arr)
         assert viewlen == y.itemsize * len(y)
 
+    @pytest.mark.skipif(True, reason="no _numpypy on py3k")
     def test_buffer_info(self):
         from _numpypy import multiarray as np
         module = self.import_module(name='buffer_test')

From pypy.commits at gmail.com  Wed Sep  7 11:31:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 08:31:08 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: in-progress
Message-ID: <57d032bc.c5cb1c0a.0496.d81d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86941:fad74c992868
Date: 2016-09-07 17:30 +0200
http://bitbucket.org/pypy/pypy/changeset/fad74c992868/

Log:	in-progress

diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
--- a/rpython/rlib/src/boehm-rawrefcount.h
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -4,6 +4,9 @@
    OP_GC_RAWREFCOUNT_CREATE_LINK_PYOBJ(): not implemented, maybe not needed
 */
 
+#ifdef RPY_REVERSE_DEBUGGER
+/* these macros are defined in src-revdb/revdb_include.h */
+#else
 #define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r)   \
     gc_rawrefcount_create_link_pypy(gcobj, pyobj)
 
@@ -15,6 +18,7 @@
 
 #define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
     r = gc_rawrefcount_next_dead()
+#endif
 
 
 RPY_EXTERN void gc_rawrefcount_create_link_pypy(/*gcobj_t*/void *gcobj, 
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1762,7 +1762,8 @@
     return result;
 }
 
-RPY_EXTERN RPyString *rpy_reverse_db_dtoa(double d)
+RPY_EXTERN
+RPyString *rpy_reverse_db_dtoa(double d)
 {
     char buffer[128], *p;
     RPyString *result;
@@ -1782,6 +1783,153 @@
 }
 
 
+static void *rawrefcount_tree;    /* {pyobj: gcobj} */
+
+struct rawrefcount_link2_s {
+    void *pyobj;
+    void *gcobj;
+};
+
+static int _rrtree_compare(const void *obj1, const void *obj2)
+{
+    const struct rawrefcount_link2_s *r1 = obj1;
+    const struct rawrefcount_link2_s *r2 = obj2;
+    void *p1 = r1->pyobj;
+    void *p2 = r2->pyobj;
+    if (p1 < p2)
+        return -1;
+    if (p1 == p2)
+        return 0;
+    else
+        return 1;
+}
+
+static void _rrtree_add(void *pyobj, void *gcobj)
+{
+    /* Note: we always allocate an indirection through a 
+       struct rawrefcount_link2_s, so that Boehm knows that
+       'gcobj' must be kept alive. */
+    struct rawrefcount_link2_s *node, **item;
+    node = GC_MALLOC_UNCOLLECTABLE(sizeof(struct rawrefcount_link2_s));
+    node->pyobj = pyobj;
+    node->gcobj = gcobj;
+    item = tsearch(node, &rawrefcount_tree, _rrtree_compare);
+    if (item == NULL) {
+        fprintf(stderr, "_rrtree_add: out of memory\n");
+        exit(1);
+    }
+    if (*item != node) {
+        fprintf(stderr, "_rrtree_add: duplicate object\n");
+        exit(1);
+    }
+}
+
+RPY_EXTERN
+void rpy_reverse_db_rawrefcount_create_link_pypy(void *gcobj, void *pyobj)
+{
+    if (!RPY_RDB_REPLAY) {
+        gc_rawrefcount_create_link_pypy(gcobj, pyobj);
+    }
+    else {
+        _rrtree_add(pyobj, gcobj);
+    }
+}
+
+RPY_EXTERN
+void *rpy_reverse_db_rawrefcount_from_obj(void *gcobj)
+{
+    void *r;
+    RPY_REVDB_EMIT(r = gc_rawrefcount_from_obj(gcobj);, void *_e, r);
+    return r;
+}
+
+RPY_EXTERN
+void *rpy_reverse_db_rawrefcount_to_obj(void *pyobj)
+{
+    unsigned char flag;
+
+    if (!RPY_RDB_REPLAY) {
+        void *r = gc_rawrefcount_to_obj(pyobj);
+        RPY_REVDB_EMIT(flag = 0xEE + !r;, unsigned char _e, flag);
+        return r;
+    }
+    else {
+        RPY_REVDB_EMIT(abort();, unsigned char _e, flag);
+        switch (flag) {
+
+        case 0xEF:
+            /* when recording, this call to to_obj() returned NULL */
+            return NULL;
+
+        case 0xEE:
+            /* when recording, this call to to_obj() didn't return NULL */
+            break;
+
+        default:
+            fprintf(stderr, "bad byte in rawrefcount_to_obj\n");
+            exit(1);
+        }
+
+        struct rawrefcount_link2_s **item, dummy;
+        dummy.pyobj = pyobj;
+        item = tfind(&dummy, &rawrefcount_tree, _rrtree_compare);
+        if (item == NULL) {
+            fprintf(stderr, "rawrefcount_to_obj: not found in tree\n");
+            exit(1);
+        }
+        return (*item)->gcobj;
+    }
+}
+
+RPY_EXTERN
+void *rpy_reverse_db_rawrefcount_next_dead(void)
+{
+    unsigned char flag;
+
+    if (!RPY_RDB_REPLAY) {
+        void *r = gc_rawrefcount_next_dead();
+        RPY_REVDB_EMIT(flag = 0xEC + !r;, unsigned char _e, flag);
+        if (r) {
+            RPY_REVDB_EMIT(;, void *_e, r);
+        }
+        return r;
+    }
+    else {
+        RPY_REVDB_EMIT(abort();, unsigned char _e, flag);
+        switch (flag) {
+
+        case 0xED:
+            /* when recording, this call to next_dead() returned NULL */
+            return NULL;
+
+        case 0xEE:
+            /* when recording, this call to next_dead() didn't return NULL */
+            break;
+
+        default:
+            fprintf(stderr, "bad byte in rawrefcount_next_dead\n");
+            exit(1);
+        }
+
+        void *pyobj;
+        RPY_REVDB_EMIT(abort();, void *_e, pyobj);
+
+        struct rawrefcount_link2_s **item, *entry, dummy;
+        dummy.pyobj = pyobj;
+        item = tfind(&dummy, &rawrefcount_tree, _rrtree_compare);
+        if (item == NULL) {
+            fprintf(stderr, "rawrefcount_next_dead: not found in tree\n");
+            exit(1);
+        }
+        entry = *item;
+        tdelete(entry, &rawrefcount_tree, _rrtree_compare);
+        GC_FREE(entry);
+
+        return pyobj;
+    }
+}
+
+
 /* ------------------------------------------------------------ */
 
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -38,7 +38,7 @@
     if (rpy_rev_fileno >= 0) {                                          \
         fprintf(stderr,                                                 \
                 "%s %s:%d: %0*llx\n",                                   \
-                mode, __FILE__, __LINE__, 2 * sizeof(_e),               \
+                mode, __FILE__, (int)__LINE__, (int)(2 * sizeof(_e)),   \
                 ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1)); \
     }
 #endif
@@ -51,7 +51,7 @@
         seeing_uid(uid);                                                \
         fprintf(stderr,                                                 \
                 "[nobj] %s:%d: obj %llu\n",                             \
-                __FILE__, __LINE__, (unsigned long long) uid);          \
+                __FILE__, (int)__LINE__, (unsigned long long) uid);     \
     }
 #endif
 
@@ -251,6 +251,19 @@
     } while (0)
 
 
+#define OP_GC_RAWREFCOUNT_CREATE_LINK_PYPY(gcobj, pyobj, r)   \
+    rpy_reverse_db_rawrefcount_create_link_pypy(gcobj, pyobj)
+
+#define OP_GC_RAWREFCOUNT_FROM_OBJ(gcobj, r)   \
+    r = rpy_reverse_db_rawrefcount_from_obj(gcobj)
+
+#define OP_GC_RAWREFCOUNT_TO_OBJ(pyobj, r)   \
+    r = rpy_reverse_db_rawrefcount_to_obj(pyobj)
+
+#define OP_GC_RAWREFCOUNT_NEXT_DEAD(r)   \
+    r = rpy_reverse_db_rawrefcount_next_dead()
+
+
 RPY_EXTERN void rpy_reverse_db_flush(void);  /* must be called with the lock */
 RPY_EXTERN void rpy_reverse_db_fetch(const char *file, int line);
 RPY_EXTERN void rpy_reverse_db_stop_point(long place);
@@ -275,5 +288,10 @@
 RPY_EXTERN void rpy_reverse_db_set_thread_breakpoint(int64_t tnum);
 RPY_EXTERN double rpy_reverse_db_strtod(RPyString *s);
 RPY_EXTERN RPyString *rpy_reverse_db_dtoa(double d);
+RPY_EXTERN void rpy_reverse_db_rawrefcount_create_link_pypy(void *gcobj, 
+                                                            void *pyobj);
+RPY_EXTERN void *rpy_reverse_db_rawrefcount_from_obj(void *gcobj);
+RPY_EXTERN void *rpy_reverse_db_rawrefcount_to_obj(void *pyobj);
+RPY_EXTERN void *rpy_reverse_db_rawrefcount_next_dead(void);
 
 /* ------------------------------------------------------------ */

From pypy.commits at gmail.com  Wed Sep  7 11:31:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 08:31:57 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix, basic test passes
Message-ID: <57d032ed.c62f1c0a.f3205.9b42@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86942:5a7baf69f279
Date: 2016-09-07 17:31 +0200
http://bitbucket.org/pypy/pypy/changeset/5a7baf69f279/

Log:	fix, basic test passes

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1902,7 +1902,7 @@
             /* when recording, this call to next_dead() returned NULL */
             return NULL;
 
-        case 0xEE:
+        case 0xEC:
             /* when recording, this call to next_dead() didn't return NULL */
             break;
 

From pypy.commits at gmail.com  Wed Sep  7 11:35:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 08:35:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix: _utf8 cannot be a quasi-immutable.
 Will document why and do a
Message-ID: <57d033c9.e440c20a.7ebf.75f1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86943:628ba5d9a6f8
Date: 2016-09-07 16:44 +0100
http://bitbucket.org/pypy/pypy/changeset/628ba5d9a6f8/

Log:	Fix: _utf8 cannot be a quasi-immutable. Will document why and do a
	proper fix

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -25,7 +25,7 @@
 
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
-    _immutable_fields_ = ['_value', '_utf8?']
+    _immutable_fields_ = ['_value']
 
     def __init__(self, unistr):
         assert isinstance(unistr, unicode)

From pypy.commits at gmail.com  Wed Sep  7 11:36:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 08:36:04 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: more things to do
Message-ID: <57d033e4.04e21c0a.5fe1.9c4e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5700:c70f20cfb870
Date: 2016-09-07 17:35 +0200
http://bitbucket.org/pypy/extradoc/changeset/c70f20cfb870/

Log:	more things to do

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -48,6 +48,12 @@
   _get_module_lock.<locals>.cb at 0x00007f118e2c0020> ignored``
   we're getting them now on start-up, investigate
 
+* ``print 5`` should give
+  ``SyntaxError: Missing parentheses in call to 'print'``
+
+* ``_utf8`` in W_UnicodeObject used to be quasi-immutable,
+  document why it doesn't work and do a proper fix
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Wed Sep  7 12:45:37 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 09:45:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: factor common code between
 compile_extension_module() and compile_extension_module_applevel()
Message-ID: <57d04431.04a81c0a.afd36.b6e1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86944:16d4af61e753
Date: 2016-09-07 17:44 +0100
http://bitbucket.org/pypy/pypy/changeset/16d4af61e753/

Log:	factor common code between compile_extension_module() and
	compile_extension_module_applevel()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -53,6 +53,26 @@
         libraries=libraries)
     return soname
 
+def get_extra_args(space):
+    state = space.fromcache(State)
+    api_library = state.api_lib
+    if sys.platform == 'win32':
+        libraries = [api_library]
+        # '%s' undefined; assuming extern returning int
+        compile_extra = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+                              (space.int_w(w_maj), space.int_w(w_min))]
+    else:
+        libraries = []
+        if sys.platform.startswith('linux'):
+            compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
+
+
 def compile_extension_module(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -82,17 +102,21 @@
             link_extra = ["-g"]
         else:
             compile_extra = link_extra = None
+    from pypy.module.imp.importing import get_so_extension
+    ext = get_so_extension(space)
+    include_extra = api.include_dirs
+    extra_libs = libraries
+    return _compile_ext(modname, include_dirs, source_files, source_strings, include_extra, compile_extra, link_extra, extra_libs, ext)
 
+def _compile_ext(modname, include_dirs, source_files, source_strings, include_extra, compile_extra, link_extra, extra_libs, ext):
     modname = modname.split('.')[-1]
     soname = create_so(modname,
-            include_dirs=api.include_dirs + include_dirs,
+            include_dirs=include_extra + include_dirs,
             source_files=source_files,
             source_strings=source_strings,
             compile_extra=compile_extra,
             link_extra=link_extra,
-            libraries=libraries)
-    from pypy.module.imp.importing import get_so_extension
-    ext = get_so_extension(space)
+            libraries=extra_libs)
     pydname = soname.new(purebasename=modname, ext=ext)
     soname.rename(pydname)
     return str(pydname)
@@ -128,18 +152,11 @@
         compile_extra = [
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
         link_extra = None
+    ext = get_so_suffix()
+    include_extra = [space.include_dir]
+    extra_libs = None
+    return _compile_ext(modname, include_dirs, source_files, source_strings, include_extra, compile_extra, link_extra, extra_libs, ext)
 
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-            include_dirs=[space.include_dir] + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra)
-    ext = get_so_suffix()
-    pydname = soname.new(purebasename=modname, ext=ext)
-    soname.rename(pydname)
-    return str(pydname)
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()

From pypy.commits at gmail.com  Wed Sep  7 13:10:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 10:10:08 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Comments. Disable a code path
 that (by mistake) casts between a GC
Message-ID: <57d049f0.02d31c0a.f7eda.ba3a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86945:5d46dd70887c
Date: 2016-09-07 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/5d46dd70887c/

Log:	Comments. Disable a code path that (by mistake) casts between a GC
	pointer and a char-sized integer.

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -41,6 +41,7 @@
     view.c_len = w_obj.getlength()
     view.c_itemsize = w_obj.buf.getitemsize()
     rffi.setintfield(view, 'c_ndim', ndim)
+    raise NotImplementedError   # bogus lines follow, remove this when fixed
     view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
     view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
     view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1895,6 +1895,17 @@
         return r;
     }
     else {
+        /* Note: when replaying, the lifetime of the gcobj is a bit
+           extended when compared with recording.  That shouldn't have
+           a visible effect.  More precisely, when replaying,
+           create_link_pypy() calls _rrtree_add(), which makes a
+           struct rawrefcount_link2_s, which keeps gcobj alive; and
+           that structure is only freed here, when we call next_dead()
+           and return the corresponding pyobj.  When recording, the
+           cause-and-effect relationship is in the opposite direction:
+           when Boehm frees the gcobj, it causes the pyobj to show up
+           (sometimes later) in a next_dead() call.
+        */
         RPY_REVDB_EMIT(abort();, unsigned char _e, flag);
         switch (flag) {
 

From pypy.commits at gmail.com  Wed Sep  7 13:38:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 10:38:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: For now,
 we should get most of the intended effect with @jit.elidable
Message-ID: <57d0509c.48811c0a.97e41.c705@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r86946:1fc97c413001
Date: 2016-09-07 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/1fc97c413001/

Log:	For now, we should get most of the intended effect with
	@jit.elidable

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -7,6 +7,7 @@
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
     unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
+from rpython.rlib import jit
 
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
@@ -76,6 +77,7 @@
     def unicode_w(self, space):
         return self._value
 
+    @jit.elidable
     def identifier_w(self, space):
         identifier = self._utf8
         if identifier is not None:

From pypy.commits at gmail.com  Wed Sep  7 13:51:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 10:51:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: Bundle all the static bits of
 compilation info into a SystemCompilationInfo object
Message-ID: <57d053b1.8f081c0a.57fa7.ce10@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86947:006a4e6fc3a9
Date: 2016-09-07 18:51 +0100
http://bitbucket.org/pypy/pypy/changeset/006a4e6fc3a9/

Log:	Bundle all the static bits of compilation info into a
	SystemCompilationInfo object

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -53,7 +53,21 @@
         libraries=libraries)
     return soname
 
-def get_extra_args(space):
+class SystemCompilationInfo(object):
+    """Bundles all the generic information required to compile extensions.
+
+    Note: here, 'system' means OS + target interpreter + test config + ...
+    """
+    def __init__(self, include_extra=None, compile_extra=None, link_extra=None,
+            extra_libs=None, ext=None):
+        self.include_extra = include_extra or []
+        self.compile_extra = compile_extra
+        self.link_extra = link_extra
+        self.extra_libs = extra_libs
+        self.ext = ext
+
+def get_cpyext_info(space):
+    from pypy.module.imp.importing import get_so_extension
     state = space.fromcache(State)
     api_library = state.api_lib
     if sys.platform == 'win32':
@@ -63,14 +77,21 @@
         # prevent linking with PythonXX.lib
         w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
         link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
-                              (space.int_w(w_maj), space.int_w(w_min))]
+            (space.int_w(w_maj), space.int_w(w_min))]
     else:
         libraries = []
         if sys.platform.startswith('linux'):
-            compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+            compile_extra = [
+                "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
             link_extra = ["-g"]
         else:
             compile_extra = link_extra = None
+    return SystemCompilationInfo(
+        include_extra=api.include_dirs,
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        extra_libs=libraries,
+        ext=get_so_extension(space))
 
 
 def compile_extension_module(space, modname, include_dirs=[],
@@ -85,39 +106,21 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    state = space.fromcache(State)
-    api_library = state.api_lib
-    if sys.platform == 'win32':
-        libraries = [api_library]
-        # '%s' undefined; assuming extern returning int
-        compile_extra = ["/we4013"]
-        # prevent linking with PythonXX.lib
-        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
-                              (space.int_w(w_maj), space.int_w(w_min))]
-    else:
-        libraries = []
-        if sys.platform.startswith('linux'):
-            compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
-            link_extra = ["-g"]
-        else:
-            compile_extra = link_extra = None
-    from pypy.module.imp.importing import get_so_extension
-    ext = get_so_extension(space)
-    include_extra = api.include_dirs
-    extra_libs = libraries
-    return _compile_ext(modname, include_dirs, source_files, source_strings, include_extra, compile_extra, link_extra, extra_libs, ext)
+    sys_info = get_cpyext_info(space)
+    return _compile_ext(
+        modname, include_dirs, source_files, source_strings, sys_info)
 
-def _compile_ext(modname, include_dirs, source_files, source_strings, include_extra, compile_extra, link_extra, extra_libs, ext):
+
+def _compile_ext(modname, include_dirs, source_files, source_strings, sys_info):
     modname = modname.split('.')[-1]
     soname = create_so(modname,
-            include_dirs=include_extra + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra,
-            libraries=extra_libs)
-    pydname = soname.new(purebasename=modname, ext=ext)
+        include_dirs=sys_info.include_extra + include_dirs,
+        source_files=source_files,
+        source_strings=source_strings,
+        compile_extra=sys_info.compile_extra,
+        link_extra=sys_info.link_extra,
+        libraries=sys_info.extra_libs)
+    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
     soname.rename(pydname)
     return str(pydname)
 
@@ -130,6 +133,24 @@
         raise RuntimeError("This interpreter does not define a filename "
             "suffix for C extensions!")
 
+def get_sys_info_app(space):
+    if sys.platform == 'win32':
+        compile_extra = ["/we4013"]
+        link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
+    elif sys.platform == 'darwin':
+        compile_extra = link_extra = None
+        pass
+    elif sys.platform.startswith('linux'):
+        compile_extra = [
+            "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
+        link_extra = None
+    ext = get_so_suffix()
+    return SystemCompilationInfo(
+        include_extra=[space.include_dir],
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        ext=get_so_suffix())
+
 def compile_extension_module_applevel(space, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
@@ -142,20 +163,9 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    if sys.platform == 'win32':
-        compile_extra = ["/we4013"]
-        link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
-    elif sys.platform == 'darwin':
-        compile_extra = link_extra = None
-        pass
-    elif sys.platform.startswith('linux'):
-        compile_extra = [
-            "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
-        link_extra = None
-    ext = get_so_suffix()
-    include_extra = [space.include_dir]
-    extra_libs = None
-    return _compile_ext(modname, include_dirs, source_files, source_strings, include_extra, compile_extra, link_extra, extra_libs, ext)
+    sys_info = get_sys_info_app(space)
+    return _compile_ext(
+        modname, include_dirs, source_files, source_strings, sys_info)
 
 
 def freeze_refcnts(self):

From pypy.commits at gmail.com  Wed Sep  7 14:13:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 11:13:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merge compile_extension_module() and
 compile_extension_module_applevel() into a single version that takes a
 sys_info instead of a space
Message-ID: <57d058de.4676c20a.2e4e.e6fc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86948:df92d833f490
Date: 2016-09-07 19:13 +0100
http://bitbucket.org/pypy/pypy/changeset/df92d833f490/

Log:	Merge compile_extension_module() and
	compile_extension_module_applevel() into a single version that takes
	a sys_info instead of a space

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -94,7 +94,7 @@
         ext=get_so_extension(space))
 
 
-def compile_extension_module(space, modname, include_dirs=[],
+def compile_extension_module(sys_info, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
@@ -106,12 +106,6 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    sys_info = get_cpyext_info(space)
-    return _compile_ext(
-        modname, include_dirs, source_files, source_strings, sys_info)
-
-
-def _compile_ext(modname, include_dirs, source_files, source_strings, sys_info):
     modname = modname.split('.')[-1]
     soname = create_so(modname,
         include_dirs=sys_info.include_extra + include_dirs,
@@ -151,22 +145,6 @@
         link_extra=link_extra,
         ext=get_so_suffix())
 
-def compile_extension_module_applevel(space, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
-    sys_info = get_sys_info_app(space)
-    return _compile_ext(
-        modname, include_dirs, source_files, source_strings, sys_info)
-
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -347,8 +325,8 @@
                 assert separate_module_sources is not None
             else:
                 separate_module_sources = []
-            pydname = self.compile_extension_module(
-                space, name,
+            pydname = compile_extension_module(
+                self.sys_info, name,
                 source_files=separate_module_files,
                 source_strings=separate_module_sources)
             return space.wrap(pydname)
@@ -403,7 +381,7 @@
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
                 kwds = dict(source_files=[filename])
-            mod = self.compile_extension_module(space, name,
+            mod = compile_extension_module(self.sys_info, name,
                     include_dirs=include_dirs, **kwds)
 
             if load_it:
@@ -495,11 +473,11 @@
                 return run
             def wrap(func):
                 return func
-            self.compile_extension_module = compile_extension_module_applevel
+            self.sys_info = get_sys_info_app(space)
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
-            self.compile_extension_module = compile_extension_module
+            self.sys_info = get_cpyext_info(space)
         self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
         self.w_reimport_module = wrap(interp2app(reimport_module))

From pypy.commits at gmail.com  Wed Sep  7 14:55:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 11:55:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: Kill FakeSpace.include_dir
Message-ID: <57d062b8.0cce1c0a.30a6.df4b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86949:de3eabaf671b
Date: 2016-09-07 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/de3eabaf671b/

Log:	Kill FakeSpace.include_dir

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -127,7 +127,8 @@
         raise RuntimeError("This interpreter does not define a filename "
             "suffix for C extensions!")
 
-def get_sys_info_app(space):
+def get_sys_info_app():
+    from distutils.sysconfig import get_python_inc
     if sys.platform == 'win32':
         compile_extra = ["/we4013"]
         link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
@@ -140,7 +141,7 @@
         link_extra = None
     ext = get_so_suffix()
     return SystemCompilationInfo(
-        include_extra=[space.include_dir],
+        include_extra=[get_python_inc()],
         compile_extra=compile_extra,
         link_extra=link_extra,
         ext=get_so_suffix())
@@ -159,9 +160,7 @@
 class FakeSpace(object):
     """Like TinyObjSpace, but different"""
     def __init__(self, config):
-        from distutils.sysconfig import get_python_inc
         self.config = config
-        self.include_dir = get_python_inc()
 
     def passthrough(self, arg):
         return arg
@@ -473,7 +472,7 @@
                 return run
             def wrap(func):
                 return func
-            self.sys_info = get_sys_info_app(space)
+            self.sys_info = get_sys_info_app()
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap

From pypy.commits at gmail.com  Wed Sep  7 18:43:17 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 07 Sep 2016 15:43:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: Oops!
Message-ID: <57d09805.6937c20a.8dc4e.acd4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86950:e3d66063913c
Date: 2016-09-07 23:41 +0100
http://bitbucket.org/pypy/pypy/changeset/e3d66063913c/

Log:	Oops!

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -476,7 +476,7 @@
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
-            self.sys_info = get_cpyext_info(space)
+            self.sys_info = get_cpyext_info(self.space)
         self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
         self.w_reimport_module = wrap(interp2app(reimport_module))

From pypy.commits at gmail.com  Wed Sep  7 18:47:25 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 07 Sep 2016 15:47:25 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Add test case for
 generating the proper guard on a raw pointer during short preamble creation
Message-ID: <57d098fd.8f081c0a.57fa7.2fcc@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86951:9fea14c01cc4
Date: 2016-09-07 18:46 -0400
http://bitbucket.org/pypy/pypy/changeset/9fea14c01cc4/

Log:	Add test case for generating the proper guard on a raw pointer
	during short preamble creation

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4508,3 +4508,38 @@
             return i
         self.meta_interp(f, [])
 
+    def test_round_trip_raw_pointer(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+
+        class Box(object):
+            _immutable_fields_ = ['_ptr']
+            _ptr = lltype.nullptr(rffi.CCHARP.TO)
+
+        def new_int_buffer(value):
+            data = lltype.malloc(rffi.CCHARP.TO, noConst(rffi.sizeof(rffi.INT)), flavor='raw', zero=True)
+            rffi.cast(rffi.INTP, data)[0] = rffi.cast(rffi.INT, value)
+            return data
+
+        def read_int_buffer(buf):
+            return rffi.cast(rffi.INTP, buf)[0]
+
+        def f():
+            i = 0
+            val = Box()
+            val._ptr = new_int_buffer(1)
+
+            set_param(None, 'retrace_limit', -1)
+            while i < 100:
+                driver.jit_merge_point(i=i, val=val)
+                driver.can_enter_jit(i=i, val=val)
+                tgt = val._ptr
+                if i & 0b100:
+                    i += 1
+                i += int(read_int_buffer(val._ptr))
+                lltype.free(val._ptr, flavor='raw')
+                val._ptr = new_int_buffer(1)
+            lltype.free(val._ptr, flavor='raw')
+
+        self.meta_interp(f, [])
+        self.check_resops(guard_nonnull=0)
+

From pypy.commits at gmail.com  Wed Sep  7 22:38:12 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 07 Sep 2016 19:38:12 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Add some explanation for
 the new test
Message-ID: <57d0cf14.e17cc20a.b3f92.57aa@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86952:febf0f869fc5
Date: 2016-09-07 22:37 -0400
http://bitbucket.org/pypy/pypy/changeset/febf0f869fc5/

Log:	Add some explanation for the new test

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4509,10 +4509,25 @@
         self.meta_interp(f, [])
 
     def test_round_trip_raw_pointer(self):
+        # The goal of this test to to get a raw pointer op into the short preamble
+        # so we can check that the proper guards are generated
+        # In this case, the resulting short preamble contains
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # i2 = int_eq(i1, 0)
+        # guard_false(i2)
+        #
+        # as opposed to what the JIT used to produce
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # guard_nonnull(i1)
+        #
+        # Which will probably generate correct assembly, but the optimization
+        # pipline expects guard_nonnull arguments to be pointer ops and may crash
+        # and may crash on other input types.
         driver = JitDriver(greens=[], reds=['i', 'val'])
 
         class Box(object):
-            _immutable_fields_ = ['_ptr']
             _ptr = lltype.nullptr(rffi.CCHARP.TO)
 
         def new_int_buffer(value):
@@ -4533,6 +4548,7 @@
                 driver.jit_merge_point(i=i, val=val)
                 driver.can_enter_jit(i=i, val=val)
                 tgt = val._ptr
+                # Just to produce a side exit
                 if i & 0b100:
                     i += 1
                 i += int(read_int_buffer(val._ptr))

From pypy.commits at gmail.com  Thu Sep  8 00:59:01 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 07 Sep 2016 21:59:01 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Remove useless line
Message-ID: <57d0f015.6937c20a.8dc4e.f5fe@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86953:c5f3d521062a
Date: 2016-09-08 00:58 -0400
http://bitbucket.org/pypy/pypy/changeset/c5f3d521062a/

Log:	Remove useless line

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4547,7 +4547,6 @@
             while i < 100:
                 driver.jit_merge_point(i=i, val=val)
                 driver.can_enter_jit(i=i, val=val)
-                tgt = val._ptr
                 # Just to produce a side exit
                 if i & 0b100:
                     i += 1

From pypy.commits at gmail.com  Thu Sep  8 02:16:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Sep 2016 23:16:18 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57d10232.c186c20a.99002.62d9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r789:3af077f71333
Date: 2016-09-08 08:16 +0200
http://bitbucket.org/pypy/pypy.org/changeset/3af077f71333/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $65097 of $105000 (62.0%)
+   $65099 of $105000 (62.0%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5552 left
+    we have $5553 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Thu Sep  8 07:54:57 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 08 Sep 2016 04:54:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: build the hashmap only once,
 not *every time* we call _has_reflected_op
Message-ID: <57d15191.e97ac20a.1eabd.faa2@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r86954:4753c209847a
Date: 2016-09-08 13:54 +0200
http://bitbucket.org/pypy/pypy/changeset/4753c209847a/

Log:	build the hashmap only once, not *every time* we call
	_has_reflected_op

	(found while reading control flow graphs)

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -392,31 +392,39 @@
         extobj_w = space.newlist([space.wrap(8192), space.wrap(0), space.w_None])
         return extobj_w
 
+
+_reflected_ops = {
+        'add': 'radd',
+        'subtract': 'rsub',
+        'multiply': 'rmul',
+        'divide': 'rdiv',
+        'true_divide': 'rtruediv',
+        'floor_divide': 'rfloordiv',
+        'remainder': 'rmod',
+        'power': 'rpow',
+        'left_shift': 'rlshift',
+        'right_shift': 'rrshift',
+        'bitwise_and': 'rand',
+        'bitwise_xor': 'rxor',
+        'bitwise_or': 'ror',
+        #/* Comparisons */
+        'equal': 'eq',
+        'not_equal': 'ne',
+        'greater': 'lt',
+        'less': 'gt',
+        'greater_equal': 'le',
+        'less_equal': 'ge',
+}
+
+for key, value in _reflected_ops.items():
+    _reflected_ops[key] = "__" + value + "__"
+del key
+del value
+
 def _has_reflected_op(space, w_obj, op):
-    refops ={ 'add': 'radd',
-            'subtract': 'rsub',
-            'multiply': 'rmul',
-            'divide': 'rdiv',
-            'true_divide': 'rtruediv',
-            'floor_divide': 'rfloordiv',
-            'remainder': 'rmod',
-            'power': 'rpow',
-            'left_shift': 'rlshift',
-            'right_shift': 'rrshift',
-            'bitwise_and': 'rand',
-            'bitwise_xor': 'rxor',
-            'bitwise_or': 'ror',
-            #/* Comparisons */
-            'equal': 'eq',
-            'not_equal': 'ne',
-            'greater': 'lt',
-            'less': 'gt',
-            'greater_equal': 'le',
-            'less_equal': 'ge',
-        }
-    if op not in refops:
+    if op not in _reflected_ops:
         return False
-    return space.getattr(w_obj, space.wrap('__' + refops[op] + '__')) is not None
+    return space.getattr(w_obj, space.wrap(_reflected_ops[op])) is not None
 
 def safe_casting_mode(casting):
     assert casting is not None

From pypy.commits at gmail.com  Thu Sep  8 08:37:09 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 08 Sep 2016 05:37:09 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: drafting new implementation
Message-ID: <57d15b75.87adc20a.c9daa.fe38@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86955:ed75d303a710
Date: 2016-09-07 17:14 +0200
http://bitbucket.org/pypy/pypy/changeset/ed75d303a710/

Log:	drafting new implementation

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -136,6 +136,7 @@
         self.schedule_position = -1
         self.priority = 0
         self._stack = False
+        self.delayed = None
 
     def is_imaginary(self):
         return False
@@ -148,6 +149,7 @@
 
     def getopnum(self):
         return self.op.getopnum()
+
     def getopname(self):
         return self.op.getopname()
 
@@ -157,6 +159,9 @@
     def can_be_relaxed(self):
         return self.op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE)
 
+    def is_pure(self):
+        return rop.is_always_pure(self.op.getopnum())
+
     def edge_to(self, to, arg=None, failarg=False, label=None):
         if self is to:
             return
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -65,9 +65,9 @@
             if node.depends_count() == 0:
                 self.worklist.insert(0, node)
 
-    def emit(self, node, scheduler):
+    def try_emit_or_delay(self, node, scheduler):
         # implement me in subclass. e.g. as in VecScheduleState
-        return False
+        raise NotImplementedError
 
     def delay(self, node):
         return False
@@ -180,14 +180,7 @@
         while state.has_more():
             node = self.next(state)
             if node:
-                if not state.emit(node, self):
-                    if not node.emitted:
-                        state.pre_emit(node)
-                        self.mark_emitted(node, state)
-                        if not node.is_imaginary():
-                            op = node.getoperation()
-                            state.seen[op] = None
-                            state.oplist.append(op)
+                state.try_emit_or_delay(node, scheduler)
                 continue
 
             # it happens that packs can emit many nodes that have been
@@ -551,6 +544,10 @@
                     failargs[i] = self.renamer.rename_map.get(seed, seed)
             op.setfailargs(failargs)
 
+        delayed = node.delayed
+        if delayed:
+            import pdb; pdb.set_trace()
+
     def profitable(self):
         return self.costmodel.profitable()
 
@@ -560,18 +557,45 @@
         for arg in self.graph.loop.label.getarglist():
             self.seen[arg] = None
 
-    def emit(self, node, scheduler):
-        """ If you implement a scheduler this operations is called
-            to emit the actual operation into the oplist of the scheduler.
-        """
-        if node.pack:
+    def try_emit_or_delay(self, scheduler, state):
+        # emission might be blocked by other nodes if this node has a pack!
+        if self.pack:
             assert node.pack.numops() > 1
             for node in node.pack.operations:
                 self.pre_emit(node)
                 scheduler.mark_emitted(node, self, unpack=False)
             turn_into_vector(self, node.pack)
-            return True
-        return False
+            return
+        elif not node.emitted:
+            if not node.is_imaginary() and node.is_pure():
+                # this operation might never be emitted. only if it is really needed
+                self.delay_emit(scheduler, node)
+                return
+            # emit a now!
+            state.pre_emit(node)
+            self.mark_emitted(node, state)
+            if not node.is_imaginary():
+                op = node.getoperation()
+                state.seen[op] = None
+                state.oplist.append(op)
+
+    def delay_emit(self, scheduler, node):
+        """ it has been decided that the operation might be scheduled later """
+        delayed = node.delayed or []
+        delayed.append(self)
+        node.delayed = None
+        for to in self.provides():
+            self.delegate_delay(to, delayed)
+        self.mark_emitted(node, state)
+
+    def delegate_delay(self, node, delayed):
+        """ Chain up delays, this can reduce many more of the operations """
+        if node.delayed is None:
+            node.delayed = delayed
+        else:
+            delayedlist = node.delayed
+            for d in delayed:
+                delayedlist.append(d)
 
     def delay(self, node):
         if node.pack:

From pypy.commits at gmail.com  Thu Sep  8 08:37:11 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 08 Sep 2016 05:37:11 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: operations now can be delayed
 (if they are pure),
 operation arguments can pull them just before the op itself is emitted
Message-ID: <57d15b77.4a301c0a.60059.2cf1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r86956:545565fe89bc
Date: 2016-09-08 14:36 +0200
http://bitbucket.org/pypy/pypy/changeset/545565fe89bc/

Log:	operations now can be delayed (if they are pure), operation
	arguments can pull them just before the op itself is emitted

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -1042,29 +1042,35 @@
         assert isinstance(other, IndexVar)
         return self.constant - other.constant
 
+    def get_operations(self):
+        var = self.var
+        tolist = []
+        if self.coefficient_mul != 1:
+            args = [var, ConstInt(self.coefficient_mul)]
+            var = ResOperation(rop.INT_MUL, args)
+            tolist.append(var)
+        if self.coefficient_div != 1:
+            assert 0   # should never be the case with handling
+                       # of INT_PY_DIV commented out in this file...
+        if self.constant > 0:
+            args = [var, ConstInt(self.constant)]
+            var = ResOperation(rop.INT_ADD, args)
+            tolist.append(var)
+        if self.constant < 0:
+            args = [var, ConstInt(self.constant)]
+            var = ResOperation(rop.INT_SUB, args)
+            tolist.append(var)
+        return tolist
+
     def emit_operations(self, opt, result_box=None):
         var = self.var
         if self.is_identity():
             return var
-        if self.coefficient_mul != 1:
-            args = [var, ConstInt(self.coefficient_mul)]
-            var = ResOperation(rop.INT_MUL, args)
-            opt.emit_operation(var)
-        if self.coefficient_div != 1:
-            assert 0   # XXX for now; should never be the case with handling
-                       # of INT_PY_DIV commented out in this file...
-            #args = [var, ConstInt(self.coefficient_div)]
-            #var = ResOperation(rop.INT_FLOORDIV, args)
-            #opt.emit_operation(var)
-        if self.constant > 0:
-            args = [var, ConstInt(self.constant)]
-            var = ResOperation(rop.INT_ADD, args)
-            opt.emit_operation(var)
-        if self.constant < 0:
-            args = [var, ConstInt(self.constant)]
-            var = ResOperation(rop.INT_SUB, args)
-            opt.emit_operation(var)
-        return var 
+        last = None
+        for op in self.get_operations():
+            opt.emit_operation(op)
+            last = op
+        return last
 
     def compare(self, other):
         """ Returns if the two are compareable as a first result
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -36,9 +36,48 @@
         self.invariant_oplist = []
         self.invariant_vector_vars = []
         self.seen = {}
+        self.delayed = []
+
+    def resolve_delayed(self, needs_resolving, delayed, op):
+        # recursive solving of all delayed objects
+        if not delayed:
+            return
+        args = op.getarglist()
+        for arg in args:
+            if arg.is_constant() or arg.is_inputarg():
+                continue
+            if arg not in self.seen:
+                needs_resolving[arg] = None
+        indexvars = self.graph.index_vars
+        i = len(delayed)-1
+        while i >= 0:
+            node = delayed[i]
+            op = node.getoperation()
+            if op in needs_resolving:
+                # either it is a normal operation, or we know that there is a linear combination
+                if op in indexvars:
+                    indexvar = indexvars[op]
+                    for operation in indexvar.get_operations():
+                        self.oplist.append(operation)
+                        last = operation
+                    self.renamer.start_renaming(op, last)
+                    del needs_resolving[op]
+                else: 
+                    del needs_resolving[op]
+                    self.resolve_delayed(needs_resolving, delayed, op)
+                    self.oplist.append(op)
+            i -= 1
+
 
     def post_schedule(self):
         loop = self.graph.loop
+        #
+        if self.delayed:
+            # some operations can be delayed until the jump instruction,
+            # handle them here
+            self.resolve_delayed({}, self.delayed, loop.jump)
+
+        #
         self.renamer.rename(loop.jump)
         self.ensure_args_unpacked(loop.jump)
         loop.operations = self.oplist
@@ -65,7 +104,7 @@
             if node.depends_count() == 0:
                 self.worklist.insert(0, node)
 
-    def try_emit_or_delay(self, node, scheduler):
+    def try_emit_or_delay(self, node):
         # implement me in subclass. e.g. as in VecScheduleState
         raise NotImplementedError
 
@@ -137,42 +176,6 @@
             return True
         return node.depends_count() != 0
 
-    def mark_emitted(self, node, state, unpack=True):
-        """ An operation has been emitted, adds new operations to the worklist
-            whenever their dependency count drops to zero.
-            Keeps worklist sorted (see priority) """
-        worklist = state.worklist
-        provides = node.provides()[:]
-        for dep in provides: # COPY
-            target = dep.to
-            node.remove_edge_to(target)
-            if not target.emitted and target.depends_count() == 0:
-                # sorts them by priority
-                i = len(worklist)-1
-                while i >= 0:
-                    cur = worklist[i]
-                    c = (cur.priority - target.priority)
-                    if c < 0: # meaning itnode.priority < target.priority:
-                        worklist.insert(i+1, target)
-                        break
-                    elif c == 0:
-                        # if they have the same priority, sort them
-                        # using the original position in the trace
-                        if target.getindex() < cur.getindex():
-                            worklist.insert(i+1, target)
-                            break
-                    i -= 1
-                else:
-                    worklist.insert(0, target)
-        node.clear_dependencies()
-        node.emitted = True
-        if not node.is_imaginary():
-            op = node.getoperation()
-            state.renamer.rename(op)
-            if unpack:
-                state.ensure_args_unpacked(op)
-            state.post_emit(node)
-
     def walk_and_emit(self, state):
         """ Emit all the operations into the oplist parameter.
             Initiates the scheduling. """
@@ -180,7 +183,7 @@
         while state.has_more():
             node = self.next(state)
             if node:
-                state.try_emit_or_delay(node, scheduler)
+                state.try_emit_or_delay(node)
                 continue
 
             # it happens that packs can emit many nodes that have been
@@ -522,7 +525,7 @@
     def post_emit(self, node):
         pass
 
-    def pre_emit(self, node):
+    def pre_emit(self, node, pack_first=True):
         op = node.getoperation()
         if op.is_guard():
             # add accumulation info to the descriptor
@@ -546,7 +549,21 @@
 
         delayed = node.delayed
         if delayed:
-            import pdb; pdb.set_trace()
+            # there are some nodes that have been delayed just for this operation
+            if pack_first:
+                self.resolve_delayed({}, delayed, op)
+            for node in delayed:
+                if node is not None:
+                    provides = node.provides()
+                    if len(provides) == 0:
+                        # add this node to the final delay list
+                        # might be emitted before jumping!
+                        self.delayed.append(node)
+                    else:
+                        for to in node.provides():
+                            tnode = to.target_node()
+                            self.delegate_delay(tnode, [node])
+                        node.delayed = None
 
     def profitable(self):
         return self.costmodel.profitable()
@@ -557,36 +574,41 @@
         for arg in self.graph.loop.label.getarglist():
             self.seen[arg] = None
 
-    def try_emit_or_delay(self, scheduler, state):
+    def try_emit_or_delay(self, node):
         # emission might be blocked by other nodes if this node has a pack!
-        if self.pack:
+        if node.pack:
             assert node.pack.numops() > 1
-            for node in node.pack.operations:
-                self.pre_emit(node)
-                scheduler.mark_emitted(node, self, unpack=False)
+            for i,node in enumerate(node.pack.operations):
+                self.pre_emit(node, i==0)
+                self.mark_emitted(node, unpack=False)
             turn_into_vector(self, node.pack)
             return
         elif not node.emitted:
             if not node.is_imaginary() and node.is_pure():
                 # this operation might never be emitted. only if it is really needed
-                self.delay_emit(scheduler, node)
+                self.delay_emit(node)
                 return
             # emit a now!
-            state.pre_emit(node)
-            self.mark_emitted(node, state)
+            self.pre_emit(node)
+            self.mark_emitted(node)
             if not node.is_imaginary():
                 op = node.getoperation()
-                state.seen[op] = None
-                state.oplist.append(op)
+                self.seen[op] = None
+                self.oplist.append(op)
 
-    def delay_emit(self, scheduler, node):
+    def delay_emit(self, node):
         """ it has been decided that the operation might be scheduled later """
         delayed = node.delayed or []
-        delayed.append(self)
+        delayed.append(node)
         node.delayed = None
-        for to in self.provides():
-            self.delegate_delay(to, delayed)
-        self.mark_emitted(node, state)
+        provides = node.provides()
+        if len(provides) == 0:
+            self.delayed.append(node)
+        else:
+            for to in node.provides():
+                tnode = to.target_node()
+                self.delegate_delay(tnode, delayed[:])
+        self.mark_emitted(node)
 
     def delegate_delay(self, node, delayed):
         """ Chain up delays, this can reduce many more of the operations """
@@ -597,6 +619,42 @@
             for d in delayed:
                 delayedlist.append(d)
 
+    def mark_emitted(state, node, unpack=True):
+        """ An operation has been emitted, adds new operations to the worklist
+            whenever their dependency count drops to zero.
+            Keeps worklist sorted (see priority) """
+        worklist = state.worklist
+        provides = node.provides()[:]
+        for dep in provides: # COPY
+            target = dep.to
+            node.remove_edge_to(target)
+            if not target.emitted and target.depends_count() == 0:
+                # sorts them by priority
+                i = len(worklist)-1
+                while i >= 0:
+                    cur = worklist[i]
+                    c = (cur.priority - target.priority)
+                    if c < 0: # meaning itnode.priority < target.priority:
+                        worklist.insert(i+1, target)
+                        break
+                    elif c == 0:
+                        # if they have the same priority, sort them
+                        # using the original position in the trace
+                        if target.getindex() < cur.getindex():
+                            worklist.insert(i+1, target)
+                            break
+                    i -= 1
+                else:
+                    worklist.insert(0, target)
+        node.clear_dependencies()
+        node.emitted = True
+        if not node.is_imaginary():
+            op = node.getoperation()
+            state.renamer.rename(op)
+            if unpack:
+                state.ensure_args_unpacked(op)
+            state.post_emit(node)
+
     def delay(self, node):
         if node.pack:
             pack = node.pack
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -1395,11 +1395,10 @@
         i2 = int_add(i1,8)
         jump(p0,i2)
         """)
-        self.schedule(trace)
+        self.schedule(trace, unroll_factor=0)
         self.ensure_operations([
             'v0[2xf64] = vec_load_f(p0, i0, 8, 0, descr=floatarraydescr)',
             'i2 = int_add(i0, 16)',
-            'jump(p0,i2)',
         ], trace)
 
 class TestLLtype(BaseTestVectorize, LLtypeMixin):

From pypy.commits at gmail.com  Thu Sep  8 09:32:32 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 08 Sep 2016 06:32:32 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Remove more junk
Message-ID: <57d16870.811f1c0a.8ba16.1593@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86957:2960c83b0fbf
Date: 2016-09-08 09:31 -0400
http://bitbucket.org/pypy/pypy/changeset/2960c83b0fbf/

Log:	Remove more junk

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4531,7 +4531,7 @@
             _ptr = lltype.nullptr(rffi.CCHARP.TO)
 
         def new_int_buffer(value):
-            data = lltype.malloc(rffi.CCHARP.TO, noConst(rffi.sizeof(rffi.INT)), flavor='raw', zero=True)
+            data = lltype.malloc(rffi.CCHARP.TO, rffi.sizeof(rffi.INT), flavor='raw')
             rffi.cast(rffi.INTP, data)[0] = rffi.cast(rffi.INT, value)
             return data
 

From pypy.commits at gmail.com  Thu Sep  8 09:40:08 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 08 Sep 2016 06:40:08 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Merge
Message-ID: <57d16a38.06a81c0a.1ca02.4c71@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r86958:3a3067bcf4c9
Date: 2016-09-08 09:39 -0400
http://bitbucket.org/pypy/pypy/changeset/3a3067bcf4c9/

Log:	Merge

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -30,3 +30,4 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -0,0 +1,61 @@
+==========
+PyPy 5.4.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.4.0, released last week,
+due to the following issues:
+
+  * Update list of contributors in documentation and LICENSE file, 
+    this was unfortunately left out of 5.4.0. My apoligies to the new
+    contributors
+
+  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+    dynamically loadable file
+
+  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+
+  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+
+  * Fix for an issue where os.access() accepted a float for mode
+
+  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+    the last byte of a unicode string sometimes
+
+  * Update built-in cffi_ to the soon-to-be-released 1.8.1 version
+
+  * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
+    translation with a message asking for help porting it 
+
+Thanks to those who reported the issues.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _cffi: https://cffi.readthedocs.io
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
 min=4
-rev=0
+rev=1
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Thu Sep  8 10:39:00 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Sep 2016 07:39:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Align wrapped method compile_module()
 API with compile_module() function
Message-ID: <57d17804.8f081c0a.57fa7.6c49@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r86959:af84bf7ba373
Date: 2016-09-08 15:37 +0100
http://bitbucket.org/pypy/pypy/changeset/af84bf7ba373/

Log:	Align wrapped method compile_module() API with compile_module()
	function

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -307,27 +307,23 @@
     def setup_method(self, func):
         @gateway.unwrap_spec(name=str)
         def compile_module(space, name,
-                           w_separate_module_files=None,
-                           w_separate_module_sources=None):
+                           w_source_files=None,
+                           w_source_strings=None):
             """
             Build an extension module linked against the cpyext api library.
             """
-            if not space.is_none(w_separate_module_files):
-                separate_module_files = space.listview_bytes(
-                    w_separate_module_files)
-                assert separate_module_files is not None
+            if not space.is_none(w_source_files):
+                source_files = space.listview_bytes(w_source_files)
             else:
-                separate_module_files = []
-            if not space.is_none(w_separate_module_sources):
-                separate_module_sources = space.listview_bytes(
-                    w_separate_module_sources)
-                assert separate_module_sources is not None
+                source_files = None
+            if not space.is_none(w_source_strings):
+                source_strings = space.listview_bytes(w_source_strings)
             else:
-                separate_module_sources = []
+                source_strings = None
             pydname = compile_extension_module(
                 self.sys_info, name,
-                source_files=separate_module_files,
-                source_strings=separate_module_sources)
+                source_files=source_files,
+                source_strings=source_strings)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -638,10 +634,10 @@
             skip('record_imported_module not supported in runappdirect mode')
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", separate_module_files=[self.here + 'banana.c'])
+            "apple.banana", source_files=[self.here + 'banana.c'])
         self.record_imported_module("apple.banana")
         date = self.compile_module(
-            "cherry.date", separate_module_files=[self.here + 'date.c'])
+            "cherry.date", source_files=[self.here + 'date.c'])
         self.record_imported_module("cherry.date")
 
         # Set up some package state so that the extensions can actually be

From pypy.commits at gmail.com  Thu Sep  8 11:34:28 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 08 Sep 2016 08:34:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merge in force-virtual-state
Message-ID: <57d18504.8628c20a.f6b0b.4f04@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: 
Changeset: r86960:990479cf9d20
Date: 2016-09-08 11:33 -0400
http://bitbucket.org/pypy/pypy/changeset/990479cf9d20/

Log:	Merge in force-virtual-state

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -365,6 +365,13 @@
     def visitor_dispatch_virtual_type(self, visitor):
         raise NotImplementedError("abstract")
 
+    def make_guards(self, op, short, optimizer):
+        from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
+        op = ResOperation(rop.INT_EQ, [op, CONST_0])
+        short.append(op)
+        op = ResOperation(rop.GUARD_FALSE, [op])
+        short.append(op)
+
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,7 +16,7 @@
 from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
      have_debug_prints
 
-class UnrollableOptimizer(Optimizer):    
+class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             if self.optunroll.short_preamble_producer is None:
@@ -120,7 +120,8 @@
                 assert op.get_forwarded() is None
         if check_newops:
             assert not self.optimizer._newoperations
-    
+
+
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
@@ -156,7 +157,7 @@
         current_vs = self.get_virtual_state(end_jump.getarglist())
         # pick the vs we want to jump to
         assert isinstance(celltoken, JitCellToken)
-        
+
         target_virtual_state = self.pick_virtual_state(current_vs,
                                                        state.virtual_state,
                                                 celltoken.target_tokens)
@@ -180,17 +181,27 @@
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)            
+                    self.optimizer._newoperations)
 
         try:
-            new_virtual_state = self.jump_to_existing_trace(end_jump, label_op,
-                                                            state.runtime_boxes)
+            new_virtual_state = self.jump_to_existing_trace(
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
                     self.optimizer._newoperations)
+
+        if new_virtual_state is not None:
+            # Attempt to force virtual boxes in order to avoid jumping
+            # to the preamble.
+            try:
+                new_virtual_state = self.jump_to_existing_trace(
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+            except InvalidLoop:
+                pass
+
         if new_virtual_state is not None:
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
@@ -199,7 +210,7 @@
 
         self.disable_retracing_if_max_retrace_guards(
             self.optimizer._newoperations, target_token)
-        
+
         return (UnrollInfo(target_token, label_op, extra_same_as,
                            self.optimizer.quasi_immutable_deps),
                 self.optimizer._newoperations)
@@ -241,7 +252,8 @@
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
         try:
-            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes)
+            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                             force_boxes=False)
         except InvalidLoop:
             return self.jump_to_preamble(cell_token, jump_op, info)
         if vs is None:
@@ -252,6 +264,14 @@
             cell_token.retraced_count += 1
             debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
         else:
+            # Try forcing boxes to avoid jumping to the preamble
+            try:
+                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                                 force_boxes=True)
+            except InvalidLoop:
+                pass
+            if vs is None:
+                return info, self.optimizer._newoperations[:]
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
@@ -288,7 +308,7 @@
         return info, self.optimizer._newoperations[:]
 
 
-    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes):
+    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False):
         jitcelltoken = jump_op.getdescr()
         assert isinstance(jitcelltoken, JitCellToken)
         virtual_state = self.get_virtual_state(jump_op.getarglist())
@@ -299,7 +319,8 @@
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, args, runtime_boxes, self.optimizer)
+                    virtual_state, args, runtime_boxes, self.optimizer,
+                    force_boxes=force_boxes)
                 patchguardop = self.optimizer.patchguardop
                 for guard in extra_guards.extra_guards:
                     if isinstance(guard, GuardResOp):
@@ -308,8 +329,18 @@
                     self.send_extra_operation(guard)
             except VirtualStatesCantMatch:
                 continue
-            args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
-                args, self.optimizer)
+
+            # When force_boxes == True, creating the virtual args can fail when
+            # components of the virtual state alias. If this occurs, we must
+            # recompute the virtual state as boxes will have been forced.
+            try:
+                args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
+                    args, self.optimizer, force_boxes=force_boxes)
+            except VirtualStatesCantMatch:
+                assert force_boxes
+                virtual_state = self.get_virtual_state(args)
+                continue
+
             short_preamble = target_token.short_preamble
             try:
                 extra = self.inline_short_preamble(args + virtuals, args,
@@ -452,7 +483,7 @@
         # by short preamble
         label_args = exported_state.virtual_state.make_inputargs(
             targetargs, self.optimizer)
-        
+
         self.short_preamble_producer = ShortPreambleBuilder(
             label_args, exported_state.short_boxes,
             exported_state.short_inputargs, exported_state.exported_infos,
@@ -497,7 +528,7 @@
     * runtime_boxes - runtime values for boxes, necessary when generating
                       guards to jump to
     """
-    
+
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
                  short_inputargs, runtime_boxes, memo):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
      ArrayStructInfo, AbstractStructPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound, IntLowerBound
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation, \
      InputArgInt, InputArgRef, InputArgFloat
 from rpython.rlib.debug import debug_print
 
@@ -20,7 +20,7 @@
 
 
 class GenerateGuardState(object):
-    def __init__(self, optimizer=None, guards=None, renum=None, bad=None):
+    def __init__(self, optimizer=None, guards=None, renum=None, bad=None, force_boxes=False):
         self.optimizer = optimizer
         self.cpu = optimizer.cpu
         if guards is None:
@@ -32,6 +32,7 @@
         if bad is None:
             bad = {}
         self.bad = bad
+        self.force_boxes = force_boxes
 
     def get_runtime_item(self, box, descr, i):
         array = box.getref_base()
@@ -303,7 +304,7 @@
             opinfo = state.optimizer.getptrinfo(box)
             assert isinstance(opinfo, ArrayPtrInfo)
         else:
-            opinfo = None            
+            opinfo = None
         for i in range(self.length):
             for descr in self.fielddescrs:
                 index = i * len(self.fielddescrs) + descr.get_index()
@@ -514,6 +515,8 @@
         NotVirtualStateInfo.__init__(self, cpu, type, info)
 
     def _generate_guards(self, other, box, runtime_box, state):
+        if state.force_boxes and isinstance(other, VirtualStateInfo):
+            return self._generate_virtual_guards(other, box, runtime_box, state)
         if not isinstance(other, NotVirtualStateInfoPtr):
             raise VirtualStatesCantMatch(
                     'The VirtualStates does not match as a ' +
@@ -545,6 +548,23 @@
     # to an existing compiled loop or retracing the loop. Both alternatives
     # will always generate correct behaviour, but performance will differ.
 
+    def _generate_virtual_guards(self, other, box, runtime_box, state):
+        """
+        Generate the guards and add state information for unifying a virtual
+        object with a non-virtual. This involves forcing the object in the
+        event that unification can succeed. Since virtual objects cannot be null,
+        this method need only check that the virtual object has the expected type.
+        """
+        assert state.force_boxes and isinstance(other, VirtualStateInfo)
+
+        if self.level == LEVEL_CONSTANT:
+            raise VirtualStatesCantMatch(
+                    "cannot unify a constant value with a virtual object")
+
+        if self.level == LEVEL_KNOWNCLASS:
+            if not self.known_class.same_constant(other.known_class):
+                raise VirtualStatesCantMatch("classes don't match")
+
     def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
                                  state):
         if not isinstance(other, NotVirtualStateInfoPtr):
@@ -617,10 +637,10 @@
             return False
         return True
 
-    def generate_guards(self, other, boxes, runtime_boxes, optimizer):
+    def generate_guards(self, other, boxes, runtime_boxes, optimizer, force_boxes=False):
         assert (len(self.state) == len(other.state) == len(boxes) ==
                 len(runtime_boxes))
-        state = GenerateGuardState(optimizer)
+        state = GenerateGuardState(optimizer, force_boxes=force_boxes)
         for i in range(len(self.state)):
             self.state[i].generate_guards(other.state[i], boxes[i],
                                           runtime_boxes[i], state)
@@ -644,8 +664,8 @@
 
         return boxes
 
-    def make_inputargs_and_virtuals(self, inputargs, optimizer):
-        inpargs = self.make_inputargs(inputargs, optimizer)
+    def make_inputargs_and_virtuals(self, inputargs, optimizer, force_boxes=False):
+        inpargs = self.make_inputargs(inputargs, optimizer, force_boxes)
         # we append the virtuals here in case some stuff is proven
         # to be not a virtual and there are getfields in the short preamble
         # that will read items out of there
@@ -653,7 +673,7 @@
         for i in range(len(inputargs)):
             if not isinstance(self.state[i], NotVirtualStateInfo):
                 virtuals.append(inputargs[i])
-            
+
         return inpargs, virtuals
 
     def debug_print(self, hdr='', bad=None, metainterp_sd=None):
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4507,3 +4507,54 @@
                 i += 1
             return i
         self.meta_interp(f, [])
+
+    def test_round_trip_raw_pointer(self):
+        # The goal of this test to to get a raw pointer op into the short preamble
+        # so we can check that the proper guards are generated
+        # In this case, the resulting short preamble contains
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # i2 = int_eq(i1, 0)
+        # guard_false(i2)
+        #
+        # as opposed to what the JIT used to produce
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # guard_nonnull(i1)
+        #
+        # Which will probably generate correct assembly, but the optimization
+        # pipline expects guard_nonnull arguments to be pointer ops and may crash
+        # and may crash on other input types.
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+
+        class Box(object):
+            _ptr = lltype.nullptr(rffi.CCHARP.TO)
+
+        def new_int_buffer(value):
+            data = lltype.malloc(rffi.CCHARP.TO, rffi.sizeof(rffi.INT), flavor='raw')
+            rffi.cast(rffi.INTP, data)[0] = rffi.cast(rffi.INT, value)
+            return data
+
+        def read_int_buffer(buf):
+            return rffi.cast(rffi.INTP, buf)[0]
+
+        def f():
+            i = 0
+            val = Box()
+            val._ptr = new_int_buffer(1)
+
+            set_param(None, 'retrace_limit', -1)
+            while i < 100:
+                driver.jit_merge_point(i=i, val=val)
+                driver.can_enter_jit(i=i, val=val)
+                # Just to produce a side exit
+                if i & 0b100:
+                    i += 1
+                i += int(read_int_buffer(val._ptr))
+                lltype.free(val._ptr, flavor='raw')
+                val._ptr = new_int_buffer(1)
+            lltype.free(val._ptr, flavor='raw')
+
+        self.meta_interp(f, [])
+        self.check_resops(guard_nonnull=0)
+
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1,8 +1,8 @@
 import py
-from rpython.rlib.jit import JitDriver, promote, dont_look_inside
+from rpython.rlib.jit import JitDriver, promote, dont_look_inside, set_param
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.jit.codewriter.policy import StopAtXPolicy
-from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.test.support import LLJitMixin, get_stats
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper import rclass
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -965,6 +965,82 @@
         self.check_aborted_count(0)
         self.check_target_token_count(4)
 
+    def test_avoid_preamble(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        class Box(object):
+            def __init__(self, v):
+                self.unbox = v
+
+        mask = -2
+        const = Box(X(5))
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            val   = X(0)
+            i     = 0
+            const.unbox = X(5)
+            while i < 17:
+                driver.can_enter_jit(i=i, val=val)
+                driver.jit_merge_point(i=i, val=val)
+                # Logical & rather than comparison to confuse range analysis.
+                # Test only succeeds on the first 2 iterations
+                if i & -2 == 0:
+                    val = const.unbox
+                else:
+                    val = X(i)
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
+
+        # With retracing disable, there will be one optimized loop expecting a
+        # non-virtual X object. The side exit creates a virtual object which must
+        # be allocated to jump to the optimized trace.
+        self.check_resops(jump=3, label=2, new_with_vtable=2)
+        self.check_target_token_count(2)
+        self.check_trace_count(3)
+
+    def test_conflated_virtual_states(self):
+        # All cases are covered when forcing one component of the virtual state
+        # also forces an as yet unseen component.
+        # i.e. expect [NotVirtual, Virtual] and given a pair of aliasing virtual
+        # objects
+        driver = JitDriver(greens=[], reds=['i', 'v1', 'v2'])
+        class Box(object):
+            def __init__(self, v):
+                self.v = v
+
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        const = Box(X(0))
+        def f():
+            set_param(None, 'retrace_limit', -1)
+            set_param(None, 'threshold', 1)
+            i = 0
+            v1 = X(0)
+            v2 = X(0)
+            const.v = X(0)
+            while i < 17:
+                driver.jit_merge_point(i=i, v1=v1, v2=v2)
+                driver.can_enter_jit(i=i, v1=v1, v2=v2)
+                if i & 1 == 0:
+                    v1 = const.v
+                    v2 = X(i)
+                else:
+                    v1 = v2 = X(i)
+                i += 1
+            return None
+        self.meta_interp(f, [])
+        # assert did not crash
+
 
 class VirtualMiscTests:
     def test_multiple_equal_virtuals(self):

From pypy.commits at gmail.com  Thu Sep  8 11:34:49 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Sep 2016 08:34:49 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Make compile_extension_module() a
 method of sys_info
Message-ID: <57d18519.436ec20a.434cf.46bb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r86961:395605edacba
Date: 2016-09-08 16:29 +0100
http://bitbucket.org/pypy/pypy/changeset/395605edacba/

Log:	Make compile_extension_module() a method of sys_info

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -66,6 +66,30 @@
         self.extra_libs = extra_libs
         self.ext = ext
 
+    def compile_extension_module(self, modname, include_dirs=[],
+            source_files=None, source_strings=None):
+        """
+        Build an extension module and return the filename of the resulting native
+        code file.
+
+        modname is the name of the module, possibly including dots if it is a module
+        inside a package.
+
+        Any extra keyword arguments are passed on to ExternalCompilationInfo to
+        build the module (so specify your source with one of those).
+        """
+        modname = modname.split('.')[-1]
+        soname = create_so(modname,
+            include_dirs=self.include_extra + include_dirs,
+            source_files=source_files,
+            source_strings=source_strings,
+            compile_extra=self.compile_extra,
+            link_extra=self.link_extra,
+            libraries=self.extra_libs)
+        pydname = soname.new(purebasename=modname, ext=self.ext)
+        soname.rename(pydname)
+        return str(pydname)
+
 def get_cpyext_info(space):
     from pypy.module.imp.importing import get_so_extension
     state = space.fromcache(State)
@@ -94,30 +118,6 @@
         ext=get_so_extension(space))
 
 
-def compile_extension_module(sys_info, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-        include_dirs=sys_info.include_extra + include_dirs,
-        source_files=source_files,
-        source_strings=source_strings,
-        compile_extra=sys_info.compile_extra,
-        link_extra=sys_info.link_extra,
-        libraries=sys_info.extra_libs)
-    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
-    soname.rename(pydname)
-    return str(pydname)
-
 def get_so_suffix():
     from imp import get_suffixes, C_EXTENSION
     for suffix, mode, typ in get_suffixes():
@@ -320,8 +320,8 @@
                 source_strings = space.listview_bytes(w_source_strings)
             else:
                 source_strings = None
-            pydname = compile_extension_module(
-                self.sys_info, name,
+            pydname = self.sys_info.compile_extension_module(
+                name,
                 source_files=source_files,
                 source_strings=source_strings)
             return space.wrap(pydname)
@@ -376,8 +376,8 @@
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
                 kwds = dict(source_files=[filename])
-            mod = compile_extension_module(self.sys_info, name,
-                    include_dirs=include_dirs, **kwds)
+            mod = self.sys_info.compile_extension_module(
+                name, include_dirs=include_dirs, **kwds)
 
             if load_it:
                 if self.runappdirect:
@@ -469,11 +469,12 @@
             def wrap(func):
                 return func
             self.sys_info = get_sys_info_app()
+            self.compile_module = self.sys_info.compile_extension_module
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
             self.sys_info = get_cpyext_info(self.space)
-        self.w_compile_module = wrap(interp2app(compile_module))
+            self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
         self.w_reimport_module = wrap(interp2app(reimport_module))
         self.w_import_extension = wrap(interp2app(import_extension))

From pypy.commits at gmail.com  Thu Sep  8 11:48:16 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Sep 2016 08:48:16 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Inline create_so()
Message-ID: <57d18840.811f1c0a.8ba16.55a3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r86962:0ab65e47f082
Date: 2016-09-08 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/0ab65e47f082/

Log:	Inline create_so()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -40,19 +40,6 @@
         files.append(filename)
     return files
 
-def create_so(modname, include_dirs, source_strings=None, source_files=None,
-        compile_extra=None, link_extra=None, libraries=None):
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    if source_strings:
-        assert not source_files
-        files = convert_sources_to_files(source_strings, dirname)
-        source_files = files
-    soname = c_compile(source_files, outputfilename=str(dirname/modname),
-        compile_extra=compile_extra, link_extra=link_extra,
-        include_dirs=include_dirs,
-        libraries=libraries)
-    return soname
-
 class SystemCompilationInfo(object):
     """Bundles all the generic information required to compile extensions.
 
@@ -69,22 +56,25 @@
     def compile_extension_module(self, modname, include_dirs=[],
             source_files=None, source_strings=None):
         """
-        Build an extension module and return the filename of the resulting native
-        code file.
+        Build an extension module and return the filename of the resulting
+        native code file.
 
-        modname is the name of the module, possibly including dots if it is a module
-        inside a package.
+        modname is the name of the module, possibly including dots if it is a
+        module inside a package.
 
         Any extra keyword arguments are passed on to ExternalCompilationInfo to
         build the module (so specify your source with one of those).
         """
         modname = modname.split('.')[-1]
-        soname = create_so(modname,
-            include_dirs=self.include_extra + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
+        dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+        if source_strings:
+            assert not source_files
+            files = convert_sources_to_files(source_strings, dirname)
+            source_files = files
+        soname = c_compile(source_files, outputfilename=str(dirname/modname),
             compile_extra=self.compile_extra,
             link_extra=self.link_extra,
+            include_dirs=self.include_extra + include_dirs,
             libraries=self.extra_libs)
         pydname = soname.new(purebasename=modname, ext=self.ext)
         soname.rename(pydname)

From pypy.commits at gmail.com  Thu Sep  8 12:56:30 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 08 Sep 2016 09:56:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: Update whatsnew-head for the
 force-virtual-state
Message-ID: <57d1983e.a717c20a.dd7b5.817b@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: 
Changeset: r86963:b21c171b5632
Date: 2016-09-08 12:55 -0400
http://bitbucket.org/pypy/pypy/changeset/b21c171b5632/

Log:	Update whatsnew-head for the force-virtual-state

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -12,4 +12,7 @@
 Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
 in numpypy
 
-
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.

From pypy.commits at gmail.com  Thu Sep  8 13:16:25 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Sep 2016 10:16:25 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Simplify import_extension() by
 extracting make_methods()
Message-ID: <57d19ce9.4cc51c0a.85fe4.eb6a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r86964:8ed4d2de2eed
Date: 2016-09-08 17:57 +0100
http://bitbucket.org/pypy/pypy/changeset/8ed4d2de2eed/

Log:	Simplify import_extension() by extracting make_methods()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -136,6 +136,28 @@
         link_extra=link_extra,
         ext=get_so_suffix())
 
+def make_methods(functions, modname):
+    methods_table = []
+    codes = []
+    for funcname, flags, code in functions:
+        cfuncname = "%s_%s" % (modname, funcname)
+        methods_table.append(
+            "{\"%s\", %s, %s}," % (funcname, cfuncname, flags))
+        func_code = """
+        static PyObject* %s(PyObject* self, PyObject* args)
+        {
+        %s
+        }
+        """ % (cfuncname, code)
+        codes.append(func_code)
+
+    body = "\n".join(codes) + """
+    static PyMethodDef methods[] = {
+    %s
+    { NULL }
+    };
+    """ % ('\n'.join(methods_table),)
+    return body
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -402,26 +424,7 @@
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
-            methods_table = []
-            codes = []
-            for funcname, flags, code in functions:
-                cfuncname = "%s_%s" % (modname, funcname)
-                methods_table.append("{\"%s\", %s, %s}," %
-                                     (funcname, cfuncname, flags))
-                func_code = """
-                static PyObject* %s(PyObject* self, PyObject* args)
-                {
-                %s
-                }
-                """ % (cfuncname, code)
-                codes.append(func_code)
-
-            body = prologue + "\n".join(codes) + """
-            static PyMethodDef methods[] = {
-            %s
-            { NULL }
-            };
-            """ % ('\n'.join(methods_table),)
+            body = prologue + make_methods(functions, modname)
             init = """Py_InitModule("%s", methods);""" % (modname,)
             if more_init:
                 init += more_init

From pypy.commits at gmail.com  Thu Sep  8 13:16:27 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Sep 2016 10:16:27 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Simplify load_it=False logic in
 import_module()
Message-ID: <57d19ceb.c70a1c0a.5ad4f.a291@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r86965:2224e01830ae
Date: 2016-09-08 18:15 +0100
http://bitbucket.org/pypy/pypy/changeset/2224e01830ae/

Log:	Simplify load_it=False logic in import_module()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -178,6 +178,7 @@
         return arg
     listview = passthrough
     str_w = passthrough
+    wrap = passthrough
 
     def unwrap(self, args):
         try:
@@ -391,22 +392,18 @@
             mod = self.sys_info.compile_extension_module(
                 name, include_dirs=include_dirs, **kwds)
 
-            if load_it:
-                if self.runappdirect:
-                    import imp
-                    return imp.load_dynamic(name, mod)
-                else:
-                    api.load_extension_module(space, mod, name)
-                    self.imported_module_names.append(name)
-                    return space.getitem(
-                        space.sys.get('modules'),
-                        space.wrap(name))
+            if not load_it:
+                return space.wrap(mod)
+            if self.runappdirect:
+                import imp
+                return imp.load_dynamic(name, mod)
             else:
-                path = os.path.dirname(mod)
-                if self.runappdirect:
-                    return path
-                else:
-                    return space.wrap(path)
+                api.load_extension_module(space, mod, name)
+                self.imported_module_names.append(name)
+                return space.getitem(
+                    space.sys.get('modules'),
+                    space.wrap(name))
+
 
         @gateway.unwrap_spec(mod=str, name=str)
         def reimport_module(space, mod, name):
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 
 class TestImport(BaseApiTest):
     def test_import(self, space, api):
@@ -39,9 +39,8 @@
 
 class AppTestImportLogic(AppTestCpythonExtensionBase):
     def test_import_logic(self):
+        import sys, os
         path = self.import_module(name='test_import_module', load_it=False)
-        import sys
-        sys.path.append(path)
+        sys.path.append(os.path.dirname(path))
         import test_import_module
         assert test_import_module.TEST is None
-

From pypy.commits at gmail.com  Thu Sep  8 16:37:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 08 Sep 2016 13:37:36 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Remove the runtime part,
 move it to http://bitbucket.org/pypy/revdb/
Message-ID: <57d1cc10.a4fdc20a.adb8f.6e06@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86966:9f8dd66da2a9
Date: 2016-09-08 18:01 +0200
http://bitbucket.org/pypy/pypy/changeset/9f8dd66da2a9/

Log:	Remove the runtime part, move it to http://bitbucket.org/pypy/revdb/

diff --git a/rpython/translator/revdb/ancillary.py b/rpython/translator/revdb/ancillary.py
deleted file mode 100644
--- a/rpython/translator/revdb/ancillary.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import py
-import os, sys
-
-
-def build(tmpdir):
-    import cffi
-    ffibuilder = cffi.FFI()
-
-    ffibuilder.cdef("""
-        int ancil_send_fds(int, const int *, unsigned);
-        int ancil_recv_fds(int, int *, unsigned);
-    """)
-
-    local_dir = os.path.dirname(os.path.abspath(__file__))
-    src_dir = os.path.join(local_dir, 'src-revdb')
-
-    ffibuilder.set_source("_ancillary_cffi", """
-        #include <ancillary.h>
-    """, include_dirs=[src_dir],
-         sources=[os.path.join(src_dir, 'fd_send.c'),
-                  os.path.join(src_dir, 'fd_recv.c')])
-
-    ffibuilder.compile(tmpdir=tmpdir, verbose=True)
-
-def import_(verbose=False):
-    import rpython
-    basedir = py.path.local(rpython.__file__).dirpath()
-    tmpdir = str(basedir.ensure('_cache', 'ancillary', dir=1))
-    if verbose:
-        print tmpdir
-    old_sys_path = sys.path[:]
-    sys.path.insert(0, tmpdir)
-    try:
-        import _ancillary_cffi
-    except ImportError:
-        build(tmpdir)
-        import _ancillary_cffi
-    sys.path[:] = old_sys_path
-    return _ancillary_cffi.ffi, _ancillary_cffi.lib
-
-
-def send_fds(pipe_num, fd_list):
-    ffi, lib = import_()
-    if lib.ancil_send_fds(pipe_num, fd_list, len(fd_list)) < 0:
-        raise OSError(ffi.errno, "ancil_send_fds() failed")
-
-def recv_fds(pipe_num, fd_count):
-    ffi, lib = import_()
-    p = ffi.new("int[]", fd_count)
-    result = lib.ancil_recv_fds(pipe_num, p, fd_count)
-    if result < 0:
-        raise OSError(ffi.errno, "ancil_recv_fds() failed")
-    return [p[i] for i in xrange(result)]
-
-
-if __name__ == '__main__':
-    import_(verbose=True)
diff --git a/rpython/translator/revdb/interact.py b/rpython/translator/revdb/interact.py
deleted file mode 100644
--- a/rpython/translator/revdb/interact.py
+++ /dev/null
@@ -1,492 +0,0 @@
-import sys, os, re
-import subprocess, socket
-import traceback, linecache
-from contextlib import contextmanager
-try:
-    import readline
-except ImportError:
-    pass
-
-from rpython.translator.revdb.process import ReplayProcessGroup
-from rpython.translator.revdb.process import Breakpoint
-
-r_cmdline = re.compile(r"([a-zA-Z0-9_]\S*|.)\s*(.*)")
-r_dollar_num = re.compile(r"\$(\d+)\b")
-
-
-class RevDebugControl(object):
-
-    def __init__(self, revdb_log_filename, executable=None,
-                 pygments_background=None):
-        with open(revdb_log_filename, 'rb') as f:
-            header = f.readline()
-        assert header.endswith('\n')
-        fields = header[:-1].split('\t')
-        if len(fields) < 2 or fields[0] != 'RevDB:':
-            raise ValueError("file %r is not a RevDB log" % (
-                revdb_log_filename,))
-        if executable is None:
-            executable = fields[1]
-        if not os.path.isfile(executable):
-            raise ValueError("executable %r not found" % (executable,))
-        linecacheoutput = self.getlinecacheoutput(pygments_background)
-        self.pgroup = ReplayProcessGroup(executable, revdb_log_filename,
-                                         linecacheoutput)
-        self.print_extra_pending_info = None
-
-    def interact(self):
-        self.last_command = 'help'
-        self.previous_time = None
-        self.previous_thread = 0
-        while True:
-            prompt = self.print_lines_before_prompt()
-            try:
-                while True:
-                    cmdline = self.display_prompt(prompt)
-                    self.run_command(cmdline)
-                    prompt = self.print_lines_before_prompt()
-            except KeyboardInterrupt:
-                rtime = self.previous_time or 1
-                print
-                print 'KeyboardInterrupt: restoring state at time %d...' % (
-                    rtime,)
-                self.pgroup.recreate_subprocess(rtime)
-                print "(type 'q' or Ctrl-D to quit)"
-                self.last_command = ''
-                self.previous_thread = '?'
-                self.previous_time = '?'
-
-    def print_lines_before_prompt(self):
-        last_time = self.pgroup.get_current_time()
-        if last_time != self.previous_time:
-            print
-            if self.pgroup.get_current_thread() != self.previous_thread:
-                self.previous_thread = self.pgroup.get_current_thread()
-                if self.previous_thread == 0:
-                    print ('-------------------- in main thread #0 '
-                           '--------------------')
-                else:
-                    print ('-------------------- in non-main thread '
-                           '#%d --------------------' % (self.previous_thread,))
-            self.pgroup.update_watch_values()
-            last_time = self.pgroup.get_current_time()
-        if self.print_extra_pending_info:
-            print self.print_extra_pending_info
-            self.print_extra_pending_info = None
-        if last_time != self.previous_time:
-            self.pgroup.show_backtrace(complete=0)
-            self.previous_time = last_time
-        prompt = '(%d)$ ' % last_time
-        return prompt
-
-    def display_prompt(self, prompt):
-        try:
-            cmdline = raw_input(prompt).strip()
-        except EOFError:
-            print
-            cmdline = 'quit'
-        if not cmdline:
-            cmdline = self.last_command
-        return cmdline
-
-    def run_command(self, cmdline):
-        match = r_cmdline.match(cmdline)
-        if not match:
-            return
-        self.last_command = cmdline
-        command, argument = match.groups()
-        try:
-            runner = getattr(self, 'command_' + command)
-        except AttributeError:
-            print >> sys.stderr, "no command '%s', try 'help'" % (command,)
-        else:
-            try:
-                runner(argument)
-            except KeyboardInterrupt:
-                raise
-            except Exception as e:
-                traceback.print_exc()
-                print >> sys.stderr
-                print >> sys.stderr, 'Something went wrong.  You are now',
-                print >> sys.stderr, 'in a pdb; press Ctrl-D to continue.'
-                import pdb; pdb.post_mortem(sys.exc_info()[2])
-                print >> sys.stderr
-                print >> sys.stderr, 'You are back running %s.' % (
-                    sys.argv[0],)
-
-    def command_help(self, argument):
-        """Display commands summary"""
-        print 'Available commands:'
-        lst = dir(self)
-        commands = [(name[len('command_'):], getattr(self, name))
-                    for name in lst
-                        if name.startswith('command_')]
-        seen = {}
-        for name, func in commands:
-            seen.setdefault(func, []).append(name)
-        for _, func in commands:
-            if func in seen:
-                names = seen.pop(func)
-                names.sort(key=len, reverse=True)
-                docstring = func.__doc__ or 'undocumented'
-                print '\t%-16s %s' % (', '.join(names), docstring)
-
-    def command_quit(self, argument):
-        """Exit the debugger"""
-        self.pgroup.close()
-        sys.exit(0)
-    command_q = command_quit
-
-    def command_go(self, argument):
-        """Jump to time ARG"""
-        arg = int(argument or self.pgroup.get_current_time())
-        self.pgroup.jump_in_time(arg)
-
-    def command_info(self, argument):
-        """Display various info ('info help' for more)"""
-        display = getattr(self, 'cmd_info_' + argument, self.cmd_info_help)
-        return display()
-
-    def cmd_info_help(self):
-        """Display info topics summary"""
-        print 'Available info topics:'
-        for name in dir(self):
-            if name.startswith('cmd_info_'):
-                command = name[len('cmd_info_'):]
-                docstring = getattr(self, name).__doc__ or 'undocumented'
-                print '\tinfo %-12s %s' % (command, docstring)
-
-    def cmd_info_paused(self):
-        """List current paused subprocesses"""
-        lst = [str(n) for n in sorted(self.pgroup.paused)]
-        print ', '.join(lst)
-
-    def _bp_kind(self, num):
-        break_at = self.pgroup.all_breakpoints.num2break.get(num, '??')
-        if break_at[0] == 'B':
-            kind = 'breakpoint'
-            name = break_at[4:]
-        elif break_at[0] == 'W':
-            kind = 'watchpoint'
-            name = self.pgroup.all_breakpoints.sources.get(num, '??')
-        elif num == -3:
-            kind = 'stoppoint'
-            name = 'explicit stop'
-        elif num == -4:
-            kind = 'switchpoint'
-            name = 'thread switch'
-        else:
-            kind = '?????point'
-            name = repr(break_at)
-        return kind, name
-
-    def _bp_new(self, source_expr, break_code, break_at, nids=None):
-        b = self.pgroup.edit_breakpoints()
-        new = 1
-        while new in b.num2break:
-            new += 1
-        b.set_num2break(new, break_code, break_at)
-        b.sources[new] = source_expr
-        if break_code == 'W':
-            b.watchvalues[new] = ''
-            if nids:
-                b.watchuids[new] = self.pgroup.nids_to_uids(nids)
-        return new
-
-    def cmd_info_breakpoints(self):
-        """List current breakpoints and watchpoints"""
-        lst = self.pgroup.all_breakpoints.num2break.keys()
-        if lst:
-            for num in sorted(lst):
-                kind, name = self._bp_kind(num)
-                print '\t%s %d: %s' % (kind, num, name)
-        else:
-            print 'no breakpoints/watchpoints.'
-    cmd_info_watchpoints = cmd_info_breakpoints
-
-    def move_forward(self, steps):
-        self.remove_tainting()
-        try:
-            self.pgroup.go_forward(steps)
-            return None
-        except Breakpoint as b:
-            self.hit_breakpoints(b)
-            return b
-
-    def move_backward(self, steps):
-        try:
-            self.pgroup.go_backward(steps)
-            return None
-        except Breakpoint as b:
-            self.hit_breakpoints(b, backward=True)
-            return b
-
-    def hit_breakpoints(self, b, backward=False):
-        printing = []
-        for num in b.regular_breakpoint_nums():
-            kind, name = self._bp_kind(num)
-            printing.append('%s %s%s: %s' % (
-                'Reverse-hit' if backward else 'Hit',
-                kind,
-                '' if kind == 'stoppoint' else ' %d' % (num,),
-                name))
-        self.print_extra_pending_info = '\n'.join(printing)
-        if self.pgroup.get_current_time() != b.time:
-            target_time = b.time
-            if backward and any(self._bp_kind(num)[0] == 'watchpoint'
-                                for num in b.regular_breakpoint_nums()):
-                target_time += 1
-            self.pgroup.jump_in_time(target_time)
-
-    def remove_tainting(self):
-        if self.pgroup.is_tainted():
-            self.pgroup.jump_in_time(self.pgroup.get_current_time())
-            assert not self.pgroup.is_tainted()
-
-    def command_step(self, argument):
-        """Run forward ARG steps (default 1)"""
-        arg = int(argument or '1')
-        self.move_forward(arg)
-    command_s = command_step
-
-    def command_bstep(self, argument):
-        """Run backward ARG steps (default 1)"""
-        arg = int(argument or '1')
-        self.move_backward(arg)
-    command_bs = command_bstep
-
-    @contextmanager
-    def _stack_id_break(self, stack_id):
-        # add temporarily a breakpoint that hits when we enter/leave
-        # a frame from/to the frame identified by 'stack_id'
-        b = self.pgroup.edit_breakpoints()
-        b.stack_id = stack_id
-        try:
-            yield
-        finally:
-            b.stack_id = 0
-
-    @contextmanager
-    def _thread_num_break(self, thread_num):
-        # add temporarily a breakpoint that hits when we enter/leave
-        # the given thread
-        b = self.pgroup.edit_breakpoints()
-        b.thread_num = thread_num
-        try:
-            yield
-        finally:
-            b.thread_num = -1
-
-    def command_next(self, argument):
-        """Run forward for one step, skipping calls"""
-        while True:
-            stack_id = self.pgroup.get_stack_id(is_parent=False)
-            with self._stack_id_break(stack_id):
-                b = self.move_forward(1)
-            while b is not None:
-                # if we hit a regular breakpoint, stop
-                if any(b.regular_breakpoint_nums()):
-                    return
-                # we hit only calls and returns inside stack_id.  If the
-                # last one of these is a "return", then we're now back inside
-                # stack_id, so stop
-                if b.nums[-1] == -2:
-                    break
-                # else, the last one is a "call", so we entered another frame.
-                # Continue running until the next call/return event occurs
-                # inside stack_id
-                with self._stack_id_break(stack_id):
-                    b = self.move_forward(self.pgroup.get_max_time() -
-                                          self.pgroup.get_current_time())
-                # and then look at that 'b' again (closes the loop)
-
-            # we might be at a "<<" position on the same line as before,
-            # which returns a get_hiddenpos_level() value of 1.  Continue
-            # until we reach a get_hiddenpos_level() value of 0.
-            if b is None or self.pgroup.get_hiddenpos_level() == 0:
-                break
-    command_n = command_next
-
-    def command_bnext(self, argument):
-        """Run backward for one step, skipping calls"""
-        while True:
-            stack_id = self.pgroup.get_stack_id(is_parent=False)
-            with self._stack_id_break(stack_id):
-                b = self.move_backward(1)
-            while b is not None:
-                # if we hit a regular breakpoint, stop
-                if any(b.regular_breakpoint_nums()):
-                    return
-                # we hit only calls and returns inside stack_id.  If the
-                # first one of these is a "call", then we're now back inside
-                # stack_id, so stop
-                if b.nums[0] == -1:
-                    break
-                # else, the first one is a "return", so before, we were
-                # inside a different frame.  Continue running until the next
-                # call/return event occurs inside stack_id
-                with self._stack_id_break(stack_id):
-                    b = self.move_backward(self.pgroup.get_current_time() - 1)
-                # and then look at that 'b' again (closes the loop)
-
-            # we might be at a "<<" position on the same line as before,
-            # which returns a get_hiddenpos_level() value of 1.  Continue
-            # until we reach a get_hiddenpos_level() value of 0.
-            if self.pgroup.get_hiddenpos_level() == 0:
-                break
-    command_bn = command_bnext
-
-    def command_finish(self, argument):
-        """Run forward until the current function finishes"""
-        stack_id = self.pgroup.get_stack_id(is_parent=True)
-        if stack_id == 0:
-            print 'No caller.'
-        else:
-            with self._stack_id_break(stack_id):
-                self.command_continue('')
-
-    def command_bfinish(self, argument):
-        """Run backward until the current function is called"""
-        stack_id = self.pgroup.get_stack_id(is_parent=True)
-        if stack_id == 0:
-            print 'No caller.'
-        else:
-            with self._stack_id_break(stack_id):
-                self.command_bcontinue('')
-
-    def command_continue(self, argument):
-        """Run forward"""
-        self.move_forward(self.pgroup.get_max_time() -
-                          self.pgroup.get_current_time())
-    command_c = command_continue
-
-    def command_bcontinue(self, argument):
-        """Run backward"""
-        self.move_backward(self.pgroup.get_current_time() - 1)
-    command_bc = command_bcontinue
-
-    def _cmd_thread(self, argument, cmd_continue):
-        argument = argument.lstrip('#')
-        if argument:
-            arg = int(argument)
-            if arg == self.pgroup.get_current_thread():
-                print 'Thread #%d is already the current one.' % (arg,)
-                return
-        else:
-            # use the current thread number to detect switches to any
-            # other thread (this works because revdb.c issues a
-            # breakpoint whenever there is a switch FROM or TO the
-            # thread '#arg').
-            arg = self.pgroup.get_current_thread()
-        #
-        with self._thread_num_break(arg):
-            cmd_continue('')
-
-    def command_nthread(self, argument):
-        """Run forward until thread switch (optionally to #ARG)"""
-        self._cmd_thread(argument, self.command_continue)
-
-    def command_bthread(self, argument):
-        """Run backward until thread switch (optionally to #ARG)"""
-        self._cmd_thread(argument, self.command_bcontinue)
-
-    def command_print(self, argument):
-        """Print an expression or execute a line of code"""
-        # locate which $NUM appear used in the expression
-        nids = map(int, r_dollar_num.findall(argument))
-        self.pgroup.print_cmd(argument, nids=nids)
-    command_p = command_print
-    locals()['command_!'] = command_print
-
-    def command_backtrace(self, argument):
-        """Show the backtrace"""
-        self.pgroup.show_backtrace(complete=1)
-    command_bt = command_backtrace
-
-    def command_list(self, argument):
-        """Show the current function"""
-        self.pgroup.show_backtrace(complete=2)
-
-    def command_locals(self, argument):
-        """Show the locals"""
-        self.pgroup.show_locals()
-
-    def command_break(self, argument):
-        """Add a breakpoint"""
-        if not argument:
-            print "Break where?"
-            return
-        num = self._bp_new(argument, 'B', argument)
-        self.pgroup.update_breakpoints()
-        b = self.pgroup.edit_breakpoints()
-        if num not in b.num2break:
-            print "Breakpoint not added"
-        else:
-            kind, name = self._bp_kind(num)
-            print "Breakpoint %d added: %s" % (num, name)
-    command_b = command_break
-
-    def command_delete(self, argument):
-        """Delete a breakpoint/watchpoint"""
-        b = self.pgroup.edit_breakpoints()
-        try:
-            arg = int(argument)
-        except ValueError:
-            for arg in b.num2break:
-                if self._bp_kind(arg)[1] == argument:
-                    break
-            else:
-                print "No such breakpoint/watchpoint: %s" % (argument,)
-                return
-        if arg not in b.num2break:
-            print "No breakpoint/watchpoint number %d" % (arg,)
-        else:
-            kind, name = self._bp_kind(arg)
-            b.num2break.pop(arg, '')
-            b.sources.pop(arg, '')
-            b.watchvalues.pop(arg, '')
-            b.watchuids.pop(arg, '')
-            print "%s %d deleted: %s" % (kind.capitalize(), arg, name)
-    command_del = command_delete
-
-    def command_watch(self, argument):
-        """Add a watchpoint (use $NUM in the expression to watch)"""
-        if not argument:
-            print "Watch what?"
-            return
-        #
-        ok_flag, compiled_code = self.pgroup.compile_watchpoint_expr(argument)
-        if not ok_flag:
-            print compiled_code     # the error message
-            print 'Watchpoint not added'
-            return
-        #
-        nids = map(int, r_dollar_num.findall(argument))
-        ok_flag, text = self.pgroup.check_watchpoint_expr(compiled_code, nids)
-        if not ok_flag:
-            print text
-            print 'Watchpoint not added'
-            return
-        #
-        new = self._bp_new(argument, 'W', compiled_code, nids=nids)
-        self.pgroup.update_watch_values()
-        print "Watchpoint %d added" % (new,)
-
-    def getlinecacheoutput(self, pygments_background):
-        if not pygments_background or pygments_background == 'off':
-            return
-        try:
-            from pygments import highlight
-            from pygments.lexers import PythonLexer
-            from pygments.formatters import TerminalFormatter
-        except ImportError as e:
-            print >> sys.stderr, 'ImportError: %s' % (e,)
-            return None
-        #
-        lexer = PythonLexer()
-        fmt = TerminalFormatter(bg=pygments_background)
-        #
-        def linecacheoutput(filename, lineno):
-            line = linecache.getline(filename, lineno)
-            return highlight(line, lexer, fmt)
-        return linecacheoutput
diff --git a/rpython/translator/revdb/message.py b/rpython/translator/revdb/message.py
deleted file mode 100644
--- a/rpython/translator/revdb/message.py
+++ /dev/null
@@ -1,120 +0,0 @@
-
-INIT_VERSION_NUMBER = 0xd80100
-
-
-# See the corresponding answers for details about messages.
-
-CMD_FORK      = -1     # Message(CMD_FORK)
-CMD_QUIT      = -2     # Message(CMD_QUIT)
-CMD_FORWARD   = -3     # Message(CMD_FORWARD, steps, breakpoint_mode)
-CMD_FUTUREIDS = -4     # Message(CMD_FUTUREIDS, extra=list-of-8bytes-uids)
-CMD_PING      = -5     # Message(CMD_PING)
-# extra commands which are not handled by revdb.c, but
-# by revdb.register_debug_command()
-CMD_PRINT       = 1    # Message(CMD_PRINT, extra=expression)
-CMD_BACKTRACE   = 2    # Message(CMD_BACKTRACE)
-CMD_LOCALS      = 3    # Message(CMD_LOCALS)
-CMD_BREAKPOINTS = 4    # Message(CMD_BREAKPOINTS, stack_id,
-                       #         extra="\0-separated names")
-CMD_STACKID     = 5    # Message(CMD_STACKID, parent-flag)
-CMD_ATTACHID    = 6    # Message(CMD_ATTACHID, small-num, unique-id)
-CMD_COMPILEWATCH= 7    # Message(CMD_COMPILEWATCH, extra=expression)
-CMD_CHECKWATCH  = 8    # Message(CMD_CHECKWATCH, extra=compiled_code)
-CMD_WATCHVALUES = 9    # Message(CMD_WATCHVALUES, extra=texts)
-
-
-# the first message sent by the first child:
-#    Message(ANSWER_INIT, INIT_VERSION_NUMBER, total_stop_points)
-ANSWER_INIT       = -20
-
-# sent when the child is done and waiting for the next command:
-#    Message(ANSWER_READY, current_time, currently_created_objects)
-ANSWER_READY      = -21
-
-# sent after CMD_FORK:
-#    Message(ANSWER_FORKED, child_pid)
-ANSWER_FORKED     = -22
-
-# sent when a child reaches the end (should not occur with process.py)
-#    Message(ANSWER_AT_END)           (the child exits afterwards)
-ANSWER_AT_END     = -23
-
-# breakpoint detected in CMD_FORWARD:
-#    Message(ANSWER_BREAKPOINT, break_time, break_created_objects, bpkt_num)
-# if breakpoint_mode=='b': sent immediately when seeing a breakpoint,
-#                          followed by ANSWER_STD with the same time
-# if breakpoint_mode=='r': sent when we're done going forward, about
-#                          the most recently seen breakpoint
-# if breakpoint_mode=='i': ignored, never sent
-ANSWER_BREAKPOINT = -24
-
-# sent after an Attempted to do I/O or access raw memory, as the last message
-ANSWER_ATTEMPT_IO = -25
-
-
-# print one line of a file to the console, for CMD_PRINT
-#    Message(ANSWER_LINECACHE, linenum, extra=filename)
-ANSWER_LINECACHE  = 19
-
-# print text to the console, for CMD_PRINT and others
-#    Message(ANSWER_TEXT, extra=text)
-ANSWER_TEXT       = 20
-
-# CMD_STACKID returns the id of the current or parent frame (depending
-# on the 'parent-flag' passed in), or 0 if not found.  The id can be just
-# the stack depth, or it can be the unique id of the frame object.  When
-# used in CMD_BREAKPOINTS, it means "break if we are entering/leaving a
-# frame from/to the given frame".
-#    Message(ANSWER_STACKID, stack-id)
-ANSWER_STACKID    = 21
-
-# sent from CMD_PRINT to record the existence of a recallable object
-#    Message(ANSWER_NEXTNID, unique-id)
-ANSWER_NEXTNID    = 22
-
-# sent after CMD_COMPILEWATCH:
-#    Message(ANSWER_WATCH, ok_flag, extra=marshalled_code)
-# sent after CMD_CHECKWATCH:
-#    Message(ANSWER_WATCH, ok_flag, extra=result_of_expr)
-ANSWER_WATCH      = 23
-
-# sent sometimes after CMD_BREAKPOINTS:
-#    Message(ANSWER_CHBKPT, bkpt_num, extra=new_breakpoint_text)
-ANSWER_CHBKPT     = 24
-
-
-# ____________________________________________________________
-
-
-class Message(object):
-    """Represent messages sent and received to subprocesses
-    started with --revdb-replay.
-    """
-
-    def __init__(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
-        self.cmd = cmd
-        self.arg1 = arg1
-        self.arg2 = arg2
-        self.arg3 = arg3
-        self.extra = extra
-
-    def __repr__(self):
-        cmd = self.cmd
-        for key, value in globals().items():
-            if (key.startswith('CMD_') or key.startswith('ANSWER_')) and (
-                    value == cmd):
-                cmd = key
-                break
-        return 'Message(%s, %d, %d, %d, %r)' % (cmd, self.arg1,
-                                                self.arg2, self.arg3,
-                                                self.extra)
-
-    def __eq__(self, other):
-        return (self.cmd == other.cmd and
-                self.arg1 == other.arg1 and
-                self.arg2 == other.arg2 and
-                self.arg3 == other.arg3 and
-                self.extra == other.extra)
-
-    def __ne__(self, other):
-        return not (self == other)
diff --git a/rpython/translator/revdb/process.py b/rpython/translator/revdb/process.py
deleted file mode 100644
--- a/rpython/translator/revdb/process.py
+++ /dev/null
@@ -1,637 +0,0 @@
-import sys, os, struct, socket, errno, subprocess
-import linecache
-from rpython.translator.revdb import ancillary
-from rpython.translator.revdb.message import *
-
-
-class Breakpoint(Exception):
-    def __init__(self, time):
-        self.time = time   # time of the previous stop_point
-        self.nums = []     # list of breakpoint numbers that occurred, in order
-
-    def record_num(self, num):
-        self.nums.append(num)
-
-    def regular_breakpoint_nums(self):
-        for num in self.nums:
-            if num != -1 and num != -2:
-                yield num
-
-    def __repr__(self):
-        return 'Breakpoint(%d, %r)' % (self.time, self.nums)
-    __str__ = __repr__
-
-
-class AllBreakpoints(object):
-
-    def __init__(self):
-        self.num2break = {}    # {small number: encoded break/watchpoint}
-        self.sources = {}      # {small number: src text}
-        self.watchvalues = {}  # {small number: resulting text}
-        self.watchuids = {}    # {small number: [uid...]}
-        self.stack_id = 0      # breaks when leaving/entering a frame from/to
-                               # the frame identified by 'stack_id'
-        self.thread_num = -1   # breaks when leaving/entering the thread_num
-
-    def __repr__(self):
-        return 'AllBreakpoints(%r, %r, %r, %r, %r)' % (
-            self.num2break, self.watchvalues, self.watchuids,
-            self.stack_id, self.thread_num)
-
-    def compare(self, other):
-        if (self.num2break == other.num2break and
-            self.stack_id == other.stack_id and
-            self.thread_num == other.thread_num):
-            if self.watchvalues == other.watchvalues:
-                return 2     # completely equal
-            else:
-                return 1     # equal, but watchvalues out-of-date
-        else:
-            return 0     # different
-
-    def is_empty(self):
-        return (len(self.num2break) == 0 and self.stack_id == 0
-                                         and self.thread_num == -1)
-
-    def duplicate(self):
-        a = AllBreakpoints()
-        a.num2break.update(self.num2break)
-        a.stack_id = self.stack_id
-        a.thread_num = self.thread_num
-        return a
-
-    def set_num2break(self, new, break_code, break_at):
-        if len(break_at) > 0xFFFFFF:
-            raise OverflowError("break/watchpoint too complex")
-        self.num2break[new] = (break_code +
-                               chr(len(break_at) & 0xFF) +
-                               chr((len(break_at) >> 8) & 0xFF) +
-                               chr(len(break_at) >> 16) +
-                               break_at)
-
-
-class RecreateSubprocess(Exception):
-    pass
-
-
-class ReplayProcess(object):
-    """Represent one replaying subprocess.
-
-    It can be either the one started with --revdb-replay, or a fork.
-    """
-
-    def __init__(self, pid, control_socket,
-                 breakpoints_cache=AllBreakpoints(),
-                 printed_objects=frozenset(),
-                 linecacheoutput=None):
-        self.pid = pid
-        self.control_socket = control_socket
-        self.tainted = False
-        self.breakpoints_cache = breakpoints_cache    # don't mutate this
-        self.printed_objects = printed_objects        # don't mutate this
-        # ^^^ frozenset containing the uids of the objects that are
-        #     either already discovered in this child
-        #     (if uid < currently_created_objects), or that will
-        #     automatically be discovered when we move forward
-        self.linecacheoutput = linecacheoutput or linecache.getline
-
-    def _recv_all(self, size):
-        pieces = []
-        while size > 0:
-            data = self.control_socket.recv(size)
-            if not data:
-                raise EOFError
-            size -= len(data)
-            pieces.append(data)
-        return ''.join(pieces)
-
-    def send(self, msg):
-        #print 'SENT:', self.pid, msg
-        binary = struct.pack("iIqqq", msg.cmd, len(msg.extra),
-                             msg.arg1, msg.arg2, msg.arg3)
-        self.control_socket.sendall(binary + msg.extra)
-
-    def recv(self):
-        binary = self._recv_all(struct.calcsize("iIqqq"))
-        cmd, size, arg1, arg2, arg3 = struct.unpack("iIqqq", binary)
-        extra = self._recv_all(size)
-        msg = Message(cmd, arg1, arg2, arg3, extra)
-        #print 'RECV:', self.pid, msg
-        return msg
-
-    def expect(self, cmd, arg1=0, arg2=0, arg3=0, extra=""):
-        msg = self.recv()
-        assert msg.cmd == cmd, msg
-        if arg1 is not Ellipsis:
-            assert msg.arg1 == arg1, msg
-        if arg2 is not Ellipsis:
-            assert msg.arg2 == arg2, msg
-        if arg3 is not Ellipsis:
-            assert msg.arg3 == arg3, msg
-        if extra is not Ellipsis:
-            assert msg.extra == extra, msg
-        return msg
-
-    def expect_ready(self):
-        msg = self.expect(ANSWER_READY, Ellipsis, Ellipsis, Ellipsis)
-        self.update_times(msg)
-
-    def update_times(self, msg):
-        self.current_time = msg.arg1
-        self.currently_created_objects = msg.arg2
-        self.current_thread = msg.arg3
-
-    def clone(self, activate=False):
-        """Fork this subprocess.  Returns a new ReplayProcess() that is
-        an identical copy.
-        """
-        self.send(Message(CMD_FORK, int(activate)))
-        s1, s2 = socket.socketpair()
-        ancillary.send_fds(self.control_socket.fileno(), [s2.fileno()])
-        s2.close()
-        msg = self.expect(ANSWER_FORKED, Ellipsis)
-        child_pid = msg.arg1
-        self.expect_ready()
-        other = ReplayProcess(child_pid, s1,
-                              breakpoints_cache=self.breakpoints_cache,
-                              printed_objects=self.printed_objects,
-                              linecacheoutput=self.linecacheoutput)
-        other.expect_ready()
-        #print >> sys.stderr, 'CLONED', self.current_time
-        return other
-
-    def close(self):
-        """Close this subprocess."""
-        try:
-            self.send(Message(CMD_QUIT))
-        except socket.error:
-            pass
-
-    def forward(self, steps, breakpoint_mode):
-        """Move this subprocess forward in time.
-        Returns the Breakpoint or None.
-        """
-        assert not self.tainted
-
-        # <DEBUGGING ONLY>
-        currents = self.current_time, self.currently_created_objects
-        self.send(Message(CMD_PING))
-        self.expect_ready()
-        assert currents == (self.current_time, self.currently_created_objects)
-        # </DEBUGGING ONLY>
-
-        self.send(Message(CMD_FORWARD, steps, ord(breakpoint_mode)))
-        #
-        # record all breakpoints that occur together during the *last* step
-        bkpt = None
-        while True:
-            msg = self.recv()
-            if msg.cmd != ANSWER_BREAKPOINT:
-                break
-            if bkpt is None or bkpt.time != msg.arg1:
-                bkpt = Breakpoint(msg.arg1)
-            bkpt.record_num(msg.arg3)
-        assert msg.cmd == ANSWER_READY, msg
-        self.update_times(msg)
-        return bkpt
-
-    def print_text_answer(self, pgroup=None):
-        while True:
-            msg = self.recv()
-            if msg.cmd == ANSWER_TEXT:
-                sys.stdout.write(msg.extra)
-                sys.stdout.flush()
-            elif msg.cmd == ANSWER_READY:
-                self.update_times(msg)
-                break
-            elif msg.cmd == ANSWER_LINECACHE:
-                line = self.linecacheoutput(msg.extra, msg.arg1)
-                if line == '':
-                    line = '?'
-                if msg.arg2:    # strip?
-                    line = line.strip()
-                else:
-                    line = line.rstrip('\r\n')
-                sys.stdout.write(line + '\n')
-                sys.stdout.flush()
-            elif msg.cmd == ANSWER_NEXTNID and pgroup is not None:
-                uid = msg.arg1
-                if uid < pgroup.initial_uid:
-                    continue     # created before the first stop point, ignore
-                self.printed_objects = self.printed_objects.union([uid])
-                new_nid = len(pgroup.all_printed_objects_lst)
-                nid = pgroup.all_printed_objects.setdefault(uid, new_nid)
-                if nid == new_nid:
-                    pgroup.all_printed_objects_lst.append(uid)
-                sys.stdout.write('$%d = ' % nid)
-                sys.stdout.flush()
-            elif msg.cmd == ANSWER_ATTEMPT_IO:
-                raise RecreateSubprocess
-            elif msg.cmd == ANSWER_CHBKPT and pgroup is not None:
-                # change the breakpoint definition.  Needed for
-                # ":linenum" breakpoints which must be expanded to the
-                # current file only once
-                b = pgroup.edit_breakpoints()
-                assert b.num2break[msg.arg1][0] == 'B'
-                if msg.extra:
-                    b.set_num2break(msg.arg1, 'B', msg.extra)
-                else:
-                    del b.num2break[msg.arg1]
-            else:
-                print >> sys.stderr, "unexpected %r" % (msg,)
-
-
-class ReplayProcessGroup(object):
-    """Handle a family of subprocesses.
-    """
-    MAX_SUBPROCESSES = 31       # maximum number of subprocesses
-    STEP_RATIO = 0.25           # subprocess n is between subprocess n-1
-                                #   and the end, at this fraction of interval
-
-    def __init__(self, executable, revdb_log_filename, linecacheoutput=None):
-        s1, s2 = socket.socketpair()
-        initial_subproc = subprocess.Popen(
-            [executable, '--revdb-replay', revdb_log_filename,
-             str(s2.fileno())], preexec_fn=s1.close)
-        s2.close()
-        child = ReplayProcess(initial_subproc.pid, s1,
-                              linecacheoutput=linecacheoutput)
-        msg = child.expect(ANSWER_INIT, INIT_VERSION_NUMBER, Ellipsis)
-        self.total_stop_points = msg.arg2
-        if self.total_stop_points == 0:
-            raise ValueError("%r does not contain any stop point" %
-                             (revdb_log_filename,))
-        child.expect_ready()
-        self.initial_uid = child.currently_created_objects
-
-        self.active = child
-        self.paused = {1: child.clone()}     # {time: subprocess}
-        self.all_breakpoints = AllBreakpoints()
-        self.all_printed_objects = {}
-        self.all_printed_objects_lst = []
-
-    def get_current_time(self):
-        return self.active.current_time
-
-    def get_currently_created_objects(self):
-        return self.active.currently_created_objects
-
-    def get_current_thread(self):
-        return self.active.current_thread
-
-    def _check_current_time(self, time):
-        assert self.get_current_time() == time
-        self.active.send(Message(CMD_FORWARD, 0))
-        return self.active.expect(ANSWER_READY, time, Ellipsis, Ellipsis)
-
-    def get_max_time(self):
-        return self.total_stop_points
-
-    def get_next_clone_time(self):
-        # if 'active' has more printed_objects than the next process
-        # already in 'paused', then we re-clone 'active'.
-        cur_time = self.get_current_time()
-        future = [time for time in self.paused if time > cur_time]
-        if future:
-            for futime in sorted(future):
-                if (self.paused[futime].printed_objects !=
-                        frozenset(self.all_printed_objects_lst)):
-                    # 'futime' is the time of the first "future" childs
-                    # with an incomplete 'printed_objects'.  This will
-                    # be re-cloned.
-                    return futime
-        #
-        if len(self.paused) >= self.MAX_SUBPROCESSES:
-            next_time = self.total_stop_points + 1
-        else:
-            latest_done = max(self.paused)
-            range_not_done = self.total_stop_points - latest_done
-            next_time = latest_done + int(self.STEP_RATIO * range_not_done) + 1
-        return next_time
-
-    def is_tainted(self):
-        return self.active.tainted
-
-    def go_forward(self, steps, breakpoint_mode='b'):
-        """Go forward, for the given number of 'steps' of time.
-
-        If needed, it will leave clones at intermediate times.
-        Does not close the active subprocess.  Note that
-        is_tainted() must return false in order to use this.
-
-        breakpoint_mode:
-          'b' = regular mode where hitting a breakpoint stops
-          'i' = ignore breakpoints
-          'r' = record the occurrence of a breakpoint but continue
-        """
-        assert steps >= 0
-        if breakpoint_mode != 'i':
-            self.update_breakpoints()
-        latest_bkpt = None
-        while True:
-            cur_time = self.get_current_time()
-            if cur_time + steps > self.total_stop_points:
-                steps = self.total_stop_points - cur_time
-            next_clone = self.get_next_clone_time()
-            rel_next_clone = next_clone - cur_time
-            if rel_next_clone > steps:
-                break
-            assert rel_next_clone >= 0
-            if rel_next_clone > 0:
-                bkpt = self.active.forward(rel_next_clone, breakpoint_mode)
-                if breakpoint_mode == 'r':
-                    latest_bkpt = bkpt or latest_bkpt
-                elif bkpt:
-                    raise bkpt
-                steps -= rel_next_clone
-            if self.active.current_time in self.paused:
-                self.paused[self.active.current_time].close()
-            clone = self.active.clone()
-            self.paused[clone.current_time] = clone
-        bkpt = self.active.forward(steps, breakpoint_mode)
-        if breakpoint_mode == 'r':
-            bkpt = bkpt or latest_bkpt
-        if bkpt:
-            raise bkpt
-
-    def go_backward(self, steps, ignore_breakpoints=False):
-        """Go backward, for the given number of 'steps' of time.
-
-        Closes the active process.  Implemented as jump_in_time()
-        and then forward-searching for breakpoint locations (if any).
-        """
-        assert steps >= 0
-        initial_time = self.get_current_time()
-        if self.all_breakpoints.is_empty() or ignore_breakpoints:
-            self.jump_in_time(initial_time - steps)
-        else:
-            if self.all_breakpoints.watchvalues:
-                first_steps = 97    # use smaller steps, because that's costly
-            else:
-                first_steps = 957
-            self._backward_search_forward(
-                search_start_time       = initial_time - first_steps,
-                search_stop_time        = initial_time,
-                search_go_on_until_time = initial_time - steps)
-
-    def _backward_search_forward(self, search_start_time, search_stop_time,
-                                 search_go_on_until_time=1):
-        while True:
-            self.jump_in_time(max(search_start_time, search_go_on_until_time))
-            search_start_time = self.get_current_time()
-            time_range_to_search = search_stop_time - search_start_time
-            if time_range_to_search <= 0:
-                print "[search end]"
-                return
-            print "[searching %d..%d]" % (search_start_time,
-                                          search_stop_time)
-            self.go_forward(time_range_to_search, breakpoint_mode='r')
-            # If at least one breakpoint was found, the Breakpoint
-            # exception is raised with the *last* such breakpoint.
-            # Otherwise, we continue here.  Search farther along a
-            # 3-times-bigger range.
-            search_stop_time = search_start_time
-            search_start_time -= time_range_to_search * 3
-
-    def _update_watchpoints_uids(self):
-        if self.all_breakpoints.watchuids:
-            uids = set()
-            uids.update(*self.all_breakpoints.watchuids.values())
-            #print self.all_breakpoints
-            #print '\t===>', uids
-            self.attach_printed_objects(uids, watch_env=True)
-
-    def update_breakpoints(self):
-        self._update_watchpoints_uids()
-        cmp = self.all_breakpoints.compare(self.active.breakpoints_cache)
-        #print 'compare:', cmp, self.all_breakpoints.watchvalues
-        if cmp == 2:
-            return      # up-to-date
-
-        # update the breakpoints/watchpoints
-        self.active.breakpoints_cache = None
-        num2break = self.all_breakpoints.num2break
-        N = (max(num2break) + 1) if num2break else 0
-        if cmp == 0:
-            flat = [num2break.get(n, '\x00') for n in range(N)]
-            arg1 = self.all_breakpoints.stack_id
-            arg2 = self.all_breakpoints.thread_num
-            extra = ''.join(flat)
-            self.active.send(Message(CMD_BREAKPOINTS, arg1, arg2, extra=extra))
-            self.active.print_text_answer(pgroup=self)
-        else:
-            assert cmp == 1
-
-        # update the watchpoint values
-        if any(name.startswith('W') for name in num2break.values()):
-            watchvalues = self.all_breakpoints.watchvalues
-            flat = []
-            for n in range(N):
-                text = ''
-                name = num2break.get(n, '')
-                if name.startswith('W'):
-                    text = watchvalues[n]
-                flat.append(text)
-            extra = '\x00'.join(flat)
-            self.active.send(Message(CMD_WATCHVALUES, extra=extra))
-            self.active.expect_ready()
-
-        self.active.breakpoints_cache = self.all_breakpoints.duplicate()
-
-    def update_watch_values(self):
-        try:
-            self._update_watchpoints_uids()
-        except socket.error as e:
-            print >> sys.stderr, "socket.error: %s" % (e,)
-            print >> sys.stderr, "restarting at position 1"
-            self.jump_in_time(1)
-            self._update_watchpoints_uids()
-        seen = set()
-        for num, name in self.all_breakpoints.num2break.items():
-            if name.startswith('W'):
-                _, text = self.check_watchpoint_expr(name[4:])
-                if text != self.all_breakpoints.watchvalues[num]:
-                    #print self.active.pid
-                    print 'updating watchpoint value: %s => %s' % (
-                        self.all_breakpoints.sources[num], text)
-                    self.all_breakpoints.watchvalues[num] = text
-                seen.add(num)
-        assert set(self.all_breakpoints.watchvalues) == seen
-
-    def compile_watchpoint_expr(self, expr):
-        self.active.send(Message(CMD_COMPILEWATCH, extra=expr))
-        msg = self.active.expect(ANSWER_WATCH, Ellipsis, extra=Ellipsis)
-        self.active.expect_ready()
-        return msg.arg1, msg.extra
-
-    def check_watchpoint_expr(self, compiled_code, nids=None):
-        if nids:
-            self.ensure_nids_to_uids(nids)
-            uids = self.nids_to_uids(nids)
-            self.attach_printed_objects(uids, watch_env=True)
-        self.active.send(Message(CMD_CHECKWATCH, extra=compiled_code))
-        msg = self.active.expect(ANSWER_WATCH, Ellipsis, extra=Ellipsis)
-        self.active.expect_ready()
-        return msg.arg1, msg.extra
-
-    def _resume(self, from_time):
-        clone_me = self.paused[from_time]
-        if self.active is not None:
-            self.active.close()
-        self.active = clone_me.clone(activate=True)
-
-    def jump_in_time(self, target_time):
-        """Jump in time at the given 'target_time'.
-
-        This function always closes the active subprocess.
-        """
-        if target_time < 1:
-            target_time = 1
-        if target_time > self.total_stop_points:
-            target_time = self.total_stop_points
-        uids = set()
-        uids.update(*self.all_breakpoints.watchuids.values())
-        self.ensure_printed_objects(uids, forced_time = target_time)
-
-    def close(self):
-        """Close all subprocesses.
-        """
-        for subp in [self.active] + self.paused.values():
-            subp.close()
-
-    def ensure_printed_objects(self, uids, forced_time=None):
-        """Ensure that all the given unique_ids are loaded in the active
-        child, if necessary by forking another child from earlier.
-        """
-        if forced_time is None:
-            initial_time = self.get_current_time()
-            child = self.active
-        else:
-            initial_time = forced_time
-            stop_time = max(time for time in self.paused
-                                 if time <= initial_time)
-            child = self.paused[stop_time]
-
-        while True:
-            uid_limit = child.currently_created_objects
-            missing_uids = [uid for uid in uids
-                                if uid < uid_limit
-                                   and uid not in child.printed_objects]
-            if not missing_uids:
-                break
-            # pick the earlier fork
-            start_time = child.current_time
-            stop_time = max(time for time in self.paused if time < start_time)
-            child = self.paused[stop_time]
-
-        # No missing_uids left: all uids are either already in
-        # self.active.printed_objects, or in the future.
-        future_uids = [uid for uid in uids if uid >= uid_limit]
-        if child is self.active:
-            assert not future_uids
-        else:
-            self._resume(stop_time)
-            if future_uids:
-                future_uids.sort()
-                pack_uids = [struct.pack('q', uid) for uid in future_uids]
-                pack_uids = ''.join(pack_uids)
-                #print '%d: from %d: CMD_FUTUREIDS %r' % (
-                #    self.active.pid,
-                #    self.active.current_time,
-                #    future_uids)
-                self.active.send(Message(CMD_FUTUREIDS, extra=pack_uids))
-                self.active.expect_ready()
-                self.active.printed_objects = (
-                    self.active.printed_objects.union(future_uids))
-            self.go_forward(initial_time - self.get_current_time(),
-                            breakpoint_mode='i')
-        assert self.active.printed_objects.issuperset(uids)
-
-    def nids_to_uids(self, nids, skip_futures=False):
-        uids = []
-        for nid in set(nids):
-            try:
-                uid = self.all_printed_objects_lst[nid]
-            except IndexError:
-                continue
-            if skip_futures and uid >= self.get_currently_created_objects():
-                #print >> sys.stderr, (
-                #    "note: '$%d' refers to an object that is "
-                #    "only created later in time" % nid)
-                continue
-            uids.append(uid)
-        return uids
-
-    def ensure_nids_to_uids(self, nids):
-        # Take the objects listed in nids which are alive at the
-        # current time, and return a list of uids of them.  This
-        # might require some replaying.
-        uids = []
-        if nids:
-            uids = self.nids_to_uids(nids, skip_futures=True)
-            self.ensure_printed_objects(uids)
-        return uids
-
-    def attach_printed_objects(self, uids, watch_env):
-        for uid in uids:
-            nid = self.all_printed_objects[uid]
-            #print '%d: %s => %s (watch_env=%d)' % (self.active.pid, nid, uid,
-            #                                       watch_env)
-            self.active.send(Message(CMD_ATTACHID, nid, uid, int(watch_env)))
-            self.active.expect_ready()
-
-    def recreate_subprocess(self, target_time=None):
-        # recreate a subprocess at the given time, or by default the
-        # current time
-        if target_time is None:
-            target_time = self.get_current_time()
-        self.active = None
-        self.jump_in_time(target_time)
-
-    def print_cmd(self, expression, nids=[]):
-        """Print an expression.
-        """
-        uids = self.ensure_nids_to_uids(nids)
-        self.active.tainted = True
-        self.attach_printed_objects(uids, watch_env=False)
-        self.active.send(Message(CMD_PRINT, extra=expression))
-        try:
-            self.active.print_text_answer(pgroup=self)
-        except RecreateSubprocess:
-            self.recreate_subprocess()
-
-    def show_backtrace(self, complete=1):
-        """Show the backtrace.
-        """
-        if complete:
-            self.active.tainted = True
-        self.active.send(Message(CMD_BACKTRACE, complete))
-        try:
-            self.active.print_text_answer()
-        except RecreateSubprocess:
-            self.recreate_subprocess()
-
-    def show_locals(self):
-        """Show the locals.
-        """
-        self.active.tainted = True
-        self.active.send(Message(CMD_LOCALS))
-        try:
-            self.active.print_text_answer()
-        except RecreateSubprocess:
-            self.recreate_subprocess()
-
-    def edit_breakpoints(self):
-        return self.all_breakpoints
-
-    def _stack_id(self, is_parent=0):
-        self.active.send(Message(CMD_STACKID, is_parent))
-        msg = self.active.expect(ANSWER_STACKID, Ellipsis, Ellipsis)
-        self.active.expect_ready()
-        return msg
-
-    def get_stack_id(self, is_parent):
-        return self._stack_id(is_parent).arg1
-
-    def get_hiddenpos_level(self):
-        return self._stack_id().arg2
diff --git a/rpython/translator/revdb/revdb.py b/rpython/translator/revdb/revdb.py
deleted file mode 100755
--- a/rpython/translator/revdb/revdb.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python2
-
-import sys, os
-
-
-if __name__ == '__main__':
-    import argparse
-    parser = argparse.ArgumentParser(description='Reverse debugger')
-    parser.add_argument('log', metavar='LOG', help='log file name')
-    parser.add_argument('-x', '--executable', dest='executable',
-                        help='name of the executable file '
-                             'that recorded the log')
-    parser.add_argument('-c', '--color', dest='color',
-                        help='colorize source code (dark,light,off)')
-    options = parser.parse_args()
-
-    sys.path.insert(0, os.path.abspath(
-        os.path.join(__file__, '..', '..', '..', '..')))
-
-    from rpython.translator.revdb.interact import RevDebugControl
-    ctrl = RevDebugControl(options.log, executable=options.executable,
-                           pygments_background=options.color)
-    ctrl.interact()
diff --git a/rpython/translator/revdb/src-revdb/fd_send.c b/rpython/translator/revdb/src-revdb/fd_send.c
deleted file mode 100644
--- a/rpython/translator/revdb/src-revdb/fd_send.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/***************************************************************************
- * libancillary - black magic on Unix domain sockets
- * (C) Nicolas George
- * fd_send.c - sending file descriptors
- ***************************************************************************/
-
-/*
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * 
- *  1. Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *  2. Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *  3. The name of the author may not be used to endorse or promote products
- *     derived from this software without specific prior written permission.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
- * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _XPG4_2 /* Solaris sucks */
-# define _XPG4_2
-#endif
-
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/uio.h>
-#include <assert.h>
-#if defined(__FreeBSD__)
-# include <sys/param.h> /* FreeBSD sucks */
-#endif
-
-#include "ancillary.h"
-
-int
-ancil_send_fds_with_buffer(int sock, const int *fds, unsigned n_fds, void *buffer)
-{
-    struct msghdr msghdr;
-    char nothing = '!';
-    struct iovec nothing_ptr;
-    struct cmsghdr *cmsg;
-    int i;
-
-    nothing_ptr.iov_base = &nothing;
-    nothing_ptr.iov_len = 1;
-    msghdr.msg_name = NULL;
-    msghdr.msg_namelen = 0;
-    msghdr.msg_iov = &nothing_ptr;
-    msghdr.msg_iovlen = 1;
-    msghdr.msg_flags = 0;
-    msghdr.msg_control = buffer;
-    msghdr.msg_controllen = sizeof(struct cmsghdr) + sizeof(int) * n_fds;
-    cmsg = CMSG_FIRSTHDR(&msghdr);
-    cmsg->cmsg_len = msghdr.msg_controllen;
-    cmsg->cmsg_level = SOL_SOCKET;
-    cmsg->cmsg_type = SCM_RIGHTS;
-    for(i = 0; i < n_fds; i++)
-	((int *)CMSG_DATA(cmsg))[i] = fds[i];
-    return(sendmsg(sock, &msghdr, 0) >= 0 ? 0 : -1);
-}
-
-#ifndef SPARE_SEND_FDS
-int
-ancil_send_fds(int sock, const int *fds, unsigned n_fds)
-{
-    ANCIL_FD_BUFFER(ANCIL_MAX_N_FDS) buffer;
-
-    assert(n_fds <= ANCIL_MAX_N_FDS);
-    return(ancil_send_fds_with_buffer(sock, fds, n_fds, &buffer));
-}
-#endif /* SPARE_SEND_FDS */
-
-#ifndef SPARE_SEND_FD
-int
-ancil_send_fd(int sock, int fd)
-{
-    ANCIL_FD_BUFFER(1) buffer;
-
-    return(ancil_send_fds_with_buffer(sock, &fd, 1, &buffer));
-}
-#endif /* SPARE_SEND_FD */

From pypy.commits at gmail.com  Thu Sep  8 18:13:55 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Sep 2016 15:13:55 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: kill w_record_imported_module and
 unskip test_recursive_package_import() on -A
Message-ID: <57d1e2a3.ccd11c0a.e2d2e.0e87@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r86968:9374f605480e
Date: 2016-09-08 23:13 +0100
http://bitbucket.org/pypy/pypy/changeset/9374f605480e/

Log:	kill w_record_imported_module and unskip
	test_recursive_package_import() on -A

diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -61,6 +61,7 @@
         py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.array._charbuf_start())
         py_buf.c_b_size = buf.getlength()
     else:
+        raise RuntimeError
         raise oefmt(space.w_NotImplementedError, "buffer flavor not supported")
 
 
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -317,6 +317,15 @@
         if not cls.runappdirect:
             cls.w_runappdirect = space.wrap(cls.runappdirect)
 
+    def record_imported_module(self, name):
+        """
+        Record a module imported in a test so that it can be cleaned up in
+        teardown before the check for leaks is done.
+
+        name gives the name of the module in the space's sys.modules.
+        """
+        self.imported_module_names.append(name)
+
     def setup_method(self, func):
         @gateway.unwrap_spec(name=str)
         def compile_module(space, name,
@@ -337,6 +346,11 @@
                 name,
                 source_files=source_files,
                 source_strings=source_strings)
+
+            # hackish, but tests calling compile_module() always end up
+            # importing the result
+            self.record_imported_module(name)
+
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -399,7 +413,7 @@
                 return imp.load_dynamic(name, mod)
             else:
                 api.load_extension_module(space, mod, name)
-                self.imported_module_names.append(name)
+                self.record_imported_module(name)
                 return space.getitem(
                     space.sys.get('modules'),
                     space.wrap(name))
@@ -429,16 +443,6 @@
                                  w_include_dirs=w_include_dirs,
                                  PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
 
-        @gateway.unwrap_spec(name=str)
-        def record_imported_module(name):
-            """
-            Record a module imported in a test so that it can be cleaned up in
-            teardown before the check for leaks is done.
-
-            name gives the name of the module in the space's sys.modules.
-            """
-            self.imported_module_names.append(name)
-
         def debug_collect(space):
             rawrefcount._collect()
 
@@ -468,7 +472,6 @@
         self.w_import_module = wrap(interp2app(import_module))
         self.w_reimport_module = wrap(interp2app(reimport_module))
         self.w_import_extension = wrap(interp2app(import_extension))
-        self.w_record_imported_module = wrap(interp2app(record_imported_module))
         self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/')
         self.w_debug_collect = wrap(interp2app(debug_collect))
 
@@ -621,15 +624,11 @@
         If `cherry.date` is an extension module which imports `apple.banana`,
         the latter is added to `sys.modules` for the `"apple.banana"` key.
         """
-        if self.runappdirect:
-            skip('record_imported_module not supported in runappdirect mode')
         # Build the extensions.
         banana = self.compile_module(
             "apple.banana", source_files=[self.here + 'banana.c'])
-        self.record_imported_module("apple.banana")
         date = self.compile_module(
             "cherry.date", source_files=[self.here + 'date.c'])
-        self.record_imported_module("cherry.date")
 
         # Set up some package state so that the extensions can actually be
         # imported.

From pypy.commits at gmail.com  Thu Sep  8 18:13:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Sep 2016 15:13:54 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Fix test_recursive_package_import
 to actually check what it claims to
Message-ID: <57d1e2a2.262ec20a.37090.10ed@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r86967:3b5fd17a3cac
Date: 2016-09-08 19:28 +0100
http://bitbucket.org/pypy/pypy/changeset/3b5fd17a3cac/

Log:	Fix test_recursive_package_import to actually check what it claims
	to

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -641,7 +641,6 @@
         apple.__path__ = [os.path.dirname(banana)]
 
         import cherry.date
-        import apple.banana
 
         assert sys.modules['apple.banana'].__name__ == 'apple.banana'
         assert sys.modules['cherry.date'].__name__ == 'cherry.date'

From pypy.commits at gmail.com  Fri Sep  9 04:54:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 09 Sep 2016 01:54:49 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Move the tests to the external
 repo
Message-ID: <57d278d9.2472c20a.5693d.b402@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86969:6acec5a7e9ed
Date: 2016-09-09 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/6acec5a7e9ed/

Log:	Move the tests to the external repo

diff --git a/rpython/rlib/src/boehm-rawrefcount.h b/rpython/rlib/src/boehm-rawrefcount.h
--- a/rpython/rlib/src/boehm-rawrefcount.h
+++ b/rpython/rlib/src/boehm-rawrefcount.h
@@ -3,6 +3,7 @@
    OP_GC_RAWREFCOUNT_INIT(callback, r): the callback is not supported here
    OP_GC_RAWREFCOUNT_CREATE_LINK_PYOBJ(): not implemented, maybe not needed
 */
+#define RPY_USES_RAWREFCOUNT
 
 #ifdef RPY_REVERSE_DEBUGGER
 /* these macros are defined in src-revdb/revdb_include.h */
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1783,6 +1783,8 @@
 }
 
 
+#ifdef RPY_USES_RAWREFCOUNT
+
 static void *rawrefcount_tree;    /* {pyobj: gcobj} */
 
 struct rawrefcount_link2_s {
@@ -1940,6 +1942,8 @@
     }
 }
 
+#endif  /* RPY_USES_RAWREFCOUNT */
+
 
 /* ------------------------------------------------------------ */
 
diff --git a/rpython/translator/revdb/test/README b/rpython/translator/revdb/test/README
new file mode 100644
--- /dev/null
+++ b/rpython/translator/revdb/test/README
@@ -0,0 +1,3 @@
+The tests are located in the external repository:
+
+    https://bitbucket.org/pypy/revdb/
diff --git a/rpython/translator/revdb/test/__init__.py b/rpython/translator/revdb/test/__init__.py
deleted file mode 100644
diff --git a/rpython/translator/revdb/test/ctrl_c.py b/rpython/translator/revdb/test/ctrl_c.py
deleted file mode 100644
--- a/rpython/translator/revdb/test/ctrl_c.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import sys, os, thread, time, signal
-
-os.setpgid(0, 0)
-assert os.getpgrp() == os.getpid()
-
-
-sys.path[:] = sys.argv[1].split('\x7f')
-from rpython.translator.revdb.process import ReplayProcessGroup
-
-exename, rdbname = sys.argv[2:]
-group = ReplayProcessGroup(exename, rdbname)
-
-
-class MyInterrupt(Exception):
-    pass
-def my_signal(*args):
-    raise MyInterrupt
-prev_signal = signal.signal(signal.SIGINT, my_signal)
-
-def enable_timer():
-    def my_kill():
-        time.sleep(0.8)
-        print >> sys.stderr, "--<<< Sending CTRL-C >>>--"
-        os.killpg(os.getpid(), signal.SIGINT)
-    thread.start_new_thread(my_kill, ())
-
-all_ok = False
-try:
-    # this runs for ~9 seconds if uninterrupted
-    enable_timer()
-    group.print_cmd('very-long-loop')
-except MyInterrupt:
-    print >> sys.stderr, "very-long-loop interrupted, trying again"
-    group.recreate_subprocess(1)
-    try:
-        enable_timer()
-        group.print_cmd('very-long-loop')
-    except MyInterrupt:
-        print >> sys.stderr, "second interruption ok"
-        all_ok = True
-
-assert all_ok, "expected very-long-loop to be killed by SIGINT"
-print "all ok"
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
deleted file mode 100644
--- a/rpython/translator/revdb/test/test_basic.py
+++ /dev/null
@@ -1,462 +0,0 @@
-import py
-import os, sys, subprocess, socket
-import re, array, struct
-from rpython.tool.udir import udir
-from rpython.translator.interactive import Translation
-from rpython.rlib.rarithmetic import LONG_BIT, intmask
-from rpython.rlib import objectmodel, revdb
-from rpython.rlib.debug import debug_print
-from rpython.rtyper.annlowlevel import cast_gcref_to_instance
-from rpython.rtyper.lltypesystem import lltype, llmemory
-
-from rpython.translator.revdb.message import *
-from rpython.translator.revdb.process import ReplayProcess
-
-
-ASYNC_THREAD_SWITCH = 0xff54 - 2**16
-
-
-class RDB(object):
-    def __init__(self, filename, expected_argv):
-        with open(filename, 'rb') as f:
-            self.buffer = f.read()
-        self.cur = self.buffer.index('\x00') + 1
-        header = self.buffer[:self.cur]
-        assert header == 'RevDB:\t' + '\t'.join(expected_argv) + '\n\x00'
-        #
-        x = self.read1('P'); assert x == 0x00FF0003
-        x = self.read1('P'); self.main_thread_id = x
-        x = self.read1('P'); assert x == 0
-        x = self.read1('P'); #assert x == &rpy_reverse_db_stop_point
-        x = self.read1('P'); #assert x == &rpy_revdb
-        x = self.read1('i'); assert x == 0
-        self.argc = self.read1('i')
-        self.argv = self.read1('P')
-        self.current_packet_end = self.cur
-        self.read_check_argv(expected_argv)
-
-    def read1(self, mode):
-        p = self.cur
-        self.cur = p + struct.calcsize(mode)
-        return struct.unpack_from(mode, self.buffer, p)[0]
-
-    def next(self, mode='P'):
-        if self.current_packet_end == self.cur:
-            packet_size = self.read1('h')
-            assert packet_size > 0
-            self.current_packet_end = self.cur + packet_size
-        result = self.read1(mode)
-        assert self.cur <= self.current_packet_end
-        return result
-
-    def is_special_packet(self):
-        if self.current_packet_end != self.cur:
-            assert self.current_packet_end > self.cur
-            return False
-        next_header = struct.unpack_from('h', self.buffer, self.cur)[0]
-        return (next_header & 0xFF00) == 0xFF00
-
-    def special_packet(self, expected, fmt):
-        assert self.current_packet_end == self.cur
-        next_id = self.read1('h')
-        assert next_id == expected
-        p = self.cur
-        self.cur = self.current_packet_end = p + struct.calcsize(fmt)
-        return struct.unpack_from(fmt, self.buffer, p)
-
-    def read_check_argv(self, expected):
-        assert self.argc == len(expected)
-        for i in range(self.argc):
-            self.next()    # this is from "p = argv[i]"
-            s = []
-            # first we determine the length of the "char *p"
-            while True:
-                c = self.next('c')
-                if c == '\x00':
-                    break
-                s.append(c)
-            # then we really read the "char *" and copy it into a rpy string
-            # (that's why this time we don't read the final \0)
-            for c1 in s:
-                c2 = self.next('c')
-                assert c2 == c1
-            assert ''.join(s) == expected[i]
-
-    def number_of_stop_points(self):
-        return struct.unpack_from("q", self.buffer, len(self.buffer) - 8)[0]
-
-    def done(self):
-        return self.cur == len(self.buffer)
-
-    def write_call(self, expected_string):
-        x = self.next()     # raw_malloc: the pointer we got
-        self.gil_release()
-        self.same_stack()   # write
-        x = self.next(); assert x == len(expected_string)
-        self.same_stack()   # errno
-        x = self.next('i'); assert x == 0      # errno
-        self.gil_acquire()
-
-    def same_stack(self):
-        x = self.next('c'); assert x == '\xFC'
-
-    def gil_acquire(self):
-        x = self.next('c'); assert x == '\xFD'
-
-    def gil_release(self):
-        x = self.next('c'); assert x == '\xFE'
-
-    def switch_thread(self, expected=None):
-        th, = self.special_packet(ASYNC_THREAD_SWITCH, 'q')
-        if expected is not None:
-            assert th == expected
-        return th
-
-
-def compile(self, entry_point, backendopt=True,
-            withsmallfuncsets=None, shared=False, thread=False):
-    t = Translation(entry_point, None, gc="boehm")
-    self.t = t
-    t.set_backend_extra_options(c_debug_defines=True)
-    t.config.translation.reverse_debugger = True
-    t.config.translation.lldebug0 = True
-    t.config.translation.shared = shared
-    t.config.translation.thread = thread
-    if withsmallfuncsets is not None:
-        t.config.translation.withsmallfuncsets = withsmallfuncsets
-    if not backendopt:
-        t.disable(["backendopt_lltype"])
-    t.annotate()
-    t.rtype()
-    if t.backendopt:
-        t.backendopt()
-    self.exename = t.compile_c()
-    self.rdbname = os.path.join(os.path.dirname(str(self.exename)),
-                                'log.rdb')
-
-def run(self, *argv):
-    env = os.environ.copy()
-    env['PYPYRDB'] = self.rdbname
-    t = self.t
-    stdout, stderr = t.driver.cbuilder.cmdexec(' '.join(argv), env=env,
-                                               expect_crash=9)
-    print >> sys.stderr, stderr
-    return stdout
-
-def fetch_rdb(self, expected_argv):
-    return RDB(self.rdbname, map(str, expected_argv))
-
-
-class BaseRecordingTests(object):
-    compile = compile
-    run = run
-    fetch_rdb = fetch_rdb
-
-
-class TestRecording(BaseRecordingTests):
-
-    def test_simple(self):
-        def main(argv):
-            print argv[1:]
-            return 9
-        self.compile(main, backendopt=False)
-        assert self.run('abc d') == '[abc, d]\n'
-        rdb = self.fetch_rdb([self.exename, 'abc', 'd'])
-        rdb.write_call('[abc, d]\n')
-        x = rdb.next('q'); assert x == 0      # number of stop points
-        # that's all we should get from this simple example
-        assert rdb.done()
-
-    def test_identityhash(self):
-        def main(argv):
-            print [objectmodel.compute_identity_hash(argv),
-                   objectmodel.compute_identity_hash(argv),
-                   objectmodel.compute_identity_hash(argv)]
-            return 9
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        match = re.match(r'\[(-?\d+), \1, \1]\n', out)
-        assert match
-        hash_value = int(match.group(1))
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        # compute_identity_hash() doesn't record anything
-        rdb.write_call(out)
-        # done
-        x = rdb.next('q'); assert x == 0      # number of stop points
-        assert rdb.done()
-
-    def test_dont_record_vtable_reads(self):
-        class A(object):
-            x = 42
-        class B(A):
-            x = 43
-        lst = [A(), B()]
-        def main(argv):
-            print lst[len(argv) & 1].x
-            return 9
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        assert out == '42\n'
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        # write() call (it used to be the case that vtable reads where
-        # recorded too; the single byte fetched from the vtable from
-        # the '.x' in main() would appear here)
-        rdb.write_call(out)
-        # done
-        x = rdb.next('q'); assert x == 0      # number of stop points
-        assert rdb.done()
-
-    def test_dont_record_pbc_reads(self):
-        class MyPBC:
-            def _freeze_(self):
-                return True
-        pbc1 = MyPBC(); pbc1.x = 41
-        pbc2 = MyPBC(); pbc2.x = 42
-        lst = [pbc1, pbc2]
-        def main(argv):
-            print lst[len(argv) & 1].x
-            return 9
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        assert out == '41\n'
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        # write() call
-        rdb.write_call(out)
-        # done
-        x = rdb.next('q'); assert x == 0      # number of stop points
-        assert rdb.done()
-
-    @py.test.mark.parametrize('limit', [3, 5])
-    def test_dont_record_small_funcset_conversions(self, limit):
-        def f1():
-            return 111
-        def f2():
-            return 222
-        def f3():
-            return 333
-        def g(n):
-            if n & 1:
-                return f1
-            else:
-                return f2
-        def main(argv):
-            x = g(len(argv))    # can be f1 or f2
-            if len(argv) > 5:
-                x = f3  # now can be f1 or f2 or f3
-            print x()
-            return 9
-        self.compile(main, backendopt=False, withsmallfuncsets=limit)
-        for input, expected_output in [
-                ('2 3', '111\n'),
-                ('2 3 4', '222\n'),
-                ('2 3 4 5 6 7', '333\n'),
-                ]:
-            out = self.run(input)
-            assert out == expected_output
-            rdb = self.fetch_rdb([self.exename] + input.split())
-            # write() call
-            rdb.write_call(out)
-            x = rdb.next('q'); assert x == 0      # number of stop points
-            assert rdb.done()
-
-
-class InteractiveTests(object):
-
-    def replay(self, **kwds):
-        s1, s2 = socket.socketpair()
-        subproc = subprocess.Popen(
-            [str(self.exename), '--revdb-replay', str(self.rdbname),
-             str(s2.fileno())], preexec_fn=s1.close, **kwds)
-        s2.close()
-        self.subproc = subproc
-        child = ReplayProcess(subproc.pid, s1)
-        child.expect(ANSWER_INIT, INIT_VERSION_NUMBER,
-                     self.expected_stop_points)
-        child.expect(ANSWER_READY, 1, Ellipsis)
-        return child
-
-
-class TestSimpleInterpreter(InteractiveTests):
-    expected_stop_points = 3
-
-    def setup_class(cls):
-        def main(argv):
-            lst = [argv[0], 'prebuilt']
-            for op in argv[1:]:
-                revdb.stop_point()
-                print op
-                lst.append(op + '??')   # create a new string here
-            for x in lst:
-                print revdb.get_unique_id(x)
-            return 9
-        compile(cls, main, backendopt=False)
-        assert run(cls, 'abc d ef') == ('abc\nd\nef\n'
-                                        '3\n0\n12\n15\n17\n')
-        rdb = fetch_rdb(cls, [cls.exename, 'abc', 'd', 'ef'])
-        assert rdb.number_of_stop_points() == 3
-
-    def test_go(self):
-        child = self.replay()
-        child.send(Message(CMD_FORWARD, 2))
-        child.expect(ANSWER_READY, 3, Ellipsis)
-        child.send(Message(CMD_FORWARD, 2))
-        child.expect(ANSWER_AT_END)
-
-    def test_quit(self):
-        child = self.replay()
-        child.send(Message(CMD_QUIT))
-        assert self.subproc.wait() == 0
-
-    def test_fork(self):
-        child = self.replay()
-        child2 = child.clone()
-        child.send(Message(CMD_FORWARD, 2))
-        child.expect(ANSWER_READY, 3, Ellipsis)
-        child2.send(Message(CMD_FORWARD, 1))
-        child2.expect(ANSWER_READY, 2, Ellipsis)
-        #
-        child.close()
-        child2.close()
-
-
-class TestDebugCommands(InteractiveTests):
-    expected_stop_points = 3
-
-    def setup_class(cls):
-        #
-        class Stuff:
-            pass
-        #
-        def g(cmdline):
-            if len(cmdline) > 5:
-                raise ValueError
-        g._dont_inline_ = True
-        #
-        def went_fw():
-            revdb.send_answer(120, revdb.current_time())
-            if revdb.current_time() != revdb.total_time():
-                revdb.go_forward(1, went_fw)
-        #
-        def _nothing(arg):
-            pass
-        #
-        def callback_track_obj(gcref):
-            revdb.send_output("callback_track_obj\n")
-            dbstate.gcref = gcref
-        #
-        def blip(cmd, extra):
-            debug_print('<<<', cmd.c_cmd, cmd.c_arg1,
-                               cmd.c_arg2, cmd.c_arg3, extra, '>>>')
-            if extra == 'oops':
-                print 42     # I/O not permitted
-            if extra == 'raise-and-catch':
-                try:
-                    g(extra)
-                except ValueError:
-                    pass
-            if extra == 'crash':
-                raise ValueError
-            if extra == 'get-value':
-                revdb.send_answer(100, revdb.current_time(),
-                                       revdb.total_time())
-            if extra == 'current-place':
-                revdb.send_answer(200, revdb.current_place())
-            ## if extra == 'go-fw':
-            ##     revdb.go_forward(1, went_fw)
-            ## if cmdline == 'set-break-after-0':
-            ##     dbstate.break_after = 0
-            ## if cmdline == 'print-id':
-            ##     revdb.send_output('obj.x=%d %d %d\n' % (
-            ##         dbstate.stuff.x,
-            ##         revdb.get_unique_id(dbstate.stuff),
-            ##         revdb.currently_created_objects()))
-            ## if cmdline.startswith('track-object '):
-            ##     uid = int(cmdline[len('track-object '):])
-            ##     dbstate.gcref = lltype.nullptr(llmemory.GCREF.TO)
-            ##     revdb.track_object(uid, callback_track_obj)
-            ## if cmdline == 'get-tracked-object':
-            ##     if dbstate.gcref:
-            ##         revdb.send_output('got obj.x=%d\n' % (
-            ##             cast_gcref_to_instance(Stuff, dbstate.gcref).x,))
-            ##     else:
-            ##         revdb.send_output('none\n')
-            ## if cmdline == 'first-created-uid':
-            ##     revdb.send_output('first-created-uid=%d\n' % (
-            ##         revdb.first_created_object_uid(),))
-            revdb.send_answer(42, cmd.c_cmd, -43, -44, extra)
-        lambda_blip = lambda: blip
-        #
-        class DBState:
-            pass
-        dbstate = DBState()
-        #
-        def main(argv):
-            revdb.register_debug_command(1, lambda_blip)
-            for i, op in enumerate(argv[1:]):
-                dbstate.stuff = Stuff()
-                dbstate.stuff.x = i + 1000
-                revdb.stop_point(i * 10)
-                print op
-                if i == 1:
-                    if os.fork() == 0:    # child
-                        os.write(2, "this line is from the fork child.\n")
-                        return 0
-            return 9
-        compile(cls, main, backendopt=False)
-        assert run(cls, 'abc d ef') == 'abc\nd\nef\n'
-
-    def test_run_blip(self):
-        child = self.replay()
-        child.send(Message(1, extra='foo'))
-        child.expect(42, 1, -43, -44, 'foo')
-
-    def test_io_not_permitted(self):
-        child = self.replay(stderr=subprocess.PIPE)
-        child.send(Message(1, extra='oops'))
-        child.expect(ANSWER_ATTEMPT_IO)
-        child.close()
-        err = self.subproc.stderr.read()
-        assert err.endswith(': Attempted to do I/O or access raw memory\n')
-
-    def test_interaction_with_forward(self):
-        child = self.replay()
-        child.send(Message(CMD_FORWARD, 50))
-        child.expect(ANSWER_AT_END)
-
-    def test_raise_and_catch(self):
-        child = self.replay()
-        child.send(Message(1, extra='raise-and-catch'))
-        child.expect(42, 1, -43, -44, 'raise-and-catch')
-
-    def test_crash(self):
-        child = self.replay(stderr=subprocess.PIPE)
-        child.send(Message(1, extra='crash'))
-        child.close()
-        err = self.subproc.stderr.read()
-        assert err.endswith('Command crashed with ValueError\n')
-
-    def test_get_value(self):
-        child = self.replay()
-        child.send(Message(1, extra='get-value'))
-        child.expect(100, 1, 3)
-
-    def test_current_place(self):
-        child = self.replay()
-        child.send(Message(1, extra='current-place'))
-        child.expect(200, 0)
-        child.expect(42, 1, -43, -44, 'current-place')
-        child.expect(ANSWER_READY, 1, Ellipsis)
-        child.send(Message(CMD_FORWARD, 2))
-        child.expect(ANSWER_READY, 3, Ellipsis)
-        child.send(Message(1, extra='current-place'))
-        child.expect(200, 20)
-        child.expect(42, 1, -43, -44, 'current-place')
-
-    ## def test_go_fw(self):
-    ##     child = self.replay()
-    ##     child.send(Message(1, extra='go-fw'))
-    ##     child.expect(42, 1, -43, -44, 'go-fw')
-    ##     child.expect(120, 2)
-    ##     child.expect(120, 3)
-    ##     child.send(Message(CMD_FORWARD, 0))
-    ##     child.expect(ANSWER_READY, 3, Ellipsis)
diff --git a/rpython/translator/revdb/test/test_callback.py b/rpython/translator/revdb/test/test_callback.py
deleted file mode 100644
--- a/rpython/translator/revdb/test/test_callback.py
+++ /dev/null
@@ -1,123 +0,0 @@
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import revdb
-from rpython.translator.revdb.test.test_basic import BaseRecordingTests
-from rpython.translator.revdb.test.test_basic import InteractiveTests
-
-from rpython.translator.revdb.message import *
-
-
-def get_callback_demo():
-    eci = ExternalCompilationInfo(separate_module_sources=['''
-        int callme(int(*cb)(int)) {
-            return cb(40) * cb(3);
-        }
-    '''], post_include_bits=['''
-        int callme(int(*)(int));
-    '''])
-    FUNCPTR = lltype.Ptr(lltype.FuncType([rffi.INT], rffi.INT))
-    callme = rffi.llexternal('callme', [FUNCPTR], rffi.INT,
-                             compilation_info=eci)
-
-    def callback(n):
-        print intmask(n)
-        return n
-
-    def main(argv):
-        revdb.stop_point()
-        print intmask(callme(callback))
-        revdb.stop_point()
-        return 9
-
-    return main
-
-
-class TestRecording(BaseRecordingTests):
-
-    def test_callback_simple(self):
-        eci = ExternalCompilationInfo(separate_module_sources=['''
-            int callme(int(*cb)(int)) {
-                return cb(40) * cb(3);
-            }
-            int callmesimple(void) {
-                return 55555;
-            }
-        '''], post_include_bits=['''
-            int callme(int(*)(int));
-            int callmesimple(void);
-        '''])
-        FUNCPTR = lltype.Ptr(lltype.FuncType([rffi.INT], rffi.INT))
-        callme = rffi.llexternal('callme', [FUNCPTR], rffi.INT,
-                                 compilation_info=eci)
-        callmesimple = rffi.llexternal('callmesimple', [], rffi.INT,
-                                       compilation_info=eci)
-
-        def callback(n):
-            return intmask(n) * 100
-
-        def main(argv):
-            print intmask(callmesimple())
-            print intmask(callme(callback))
-            return 9
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        rdb.gil_release()
-        rdb.same_stack()                        # callmesimple()
-        x = rdb.next('i'); assert x == 55555
-        rdb.gil_acquire()
-        rdb.write_call('55555\n')
-        rdb.gil_release()
-        b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
-        x = rdb.next('i'); assert x == 40       # arg n
-        rdb.gil_acquire()
-        rdb.gil_release()
-        x = rdb.next('!h'); assert x == b       # -> callback
-        x = rdb.next('i'); assert x == 3        # arg n
-        rdb.gil_acquire()
-        rdb.gil_release()
-        rdb.same_stack()                        # <- return in main thread
-        x = rdb.next('i'); assert x == 4000 * 300   # return from callme()
-        rdb.gil_acquire()
-        rdb.write_call('%s\n' % (4000 * 300,))
-        x = rdb.next('q'); assert x == 0      # number of stop points
-        assert rdb.done()
-
-    def test_callback_with_effects(self):
-        main = get_callback_demo()
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        rdb.gil_release()
-        b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
-        x = rdb.next('i'); assert x == 40       # arg n
-        rdb.gil_acquire()
-        rdb.write_call('40\n')
-        rdb.gil_release()
-        x = rdb.next('!h'); assert x == b       # -> callback again
-        x = rdb.next('i'); assert x == 3        # arg n
-        rdb.gil_acquire()
-        rdb.write_call('3\n')
-        rdb.gil_release()
-        rdb.same_stack()                        # -> return in main thread
-        x = rdb.next('i'); assert x == 120      # <- return from callme()
-        rdb.gil_acquire()
-        rdb.write_call('120\n')
-        x = rdb.next('q'); assert x == 2        # number of stop points
-        assert rdb.done()
-
-
-class TestReplayingCallback(InteractiveTests):
-    expected_stop_points = 2
-
-    def setup_class(cls):
-        from rpython.translator.revdb.test.test_basic import compile, run
-        main = get_callback_demo()
-        compile(cls, main, backendopt=False)
-        run(cls, '')
-
-    def test_replaying_callback(self):
-        child = self.replay()
-        child.send(Message(CMD_FORWARD, 3))
-        child.expect(ANSWER_AT_END)
diff --git a/rpython/translator/revdb/test/test_process.py b/rpython/translator/revdb/test/test_process.py
deleted file mode 100644
--- a/rpython/translator/revdb/test/test_process.py
+++ /dev/null
@@ -1,237 +0,0 @@
-import py, sys, math, os, subprocess, time
-from cStringIO import StringIO
-from rpython.rlib import revdb, rdtoa
-from rpython.rlib.debug import debug_print, ll_assert
-from rpython.rtyper.annlowlevel import cast_gcref_to_instance
-from rpython.translator.revdb.message import *
-from rpython.translator.revdb.process import ReplayProcessGroup, Breakpoint
-
-from hypothesis import given, strategies
-
-
-class stdout_capture(object):
-    def __enter__(self):
-        self.old_stdout = sys.stdout
-        sys.stdout = self.buffer = StringIO()
-        return self.buffer
-    def __exit__(self, *args):
-        sys.stdout = self.old_stdout
-
-
-class TestReplayProcessGroup:
-
-    def setup_class(cls):
-        from rpython.translator.revdb.test.test_basic import compile, run
-
-        class Stuff:
-            pass
-
-        class DBState:
-            break_loop = -2
-            stuff = None
-            metavar = None
-            printed_stuff = None
-            watch_future = -1
-        dbstate = DBState()
-
-        def blip(cmd, extra):
-            debug_print('<<<', cmd.c_cmd, cmd.c_arg1,
-                               cmd.c_arg2, cmd.c_arg3, extra, '>>>')
-            if extra == 'set-breakpoint':
-                dbstate.break_loop = cmd.c_arg1
-            revdb.send_answer(42, cmd.c_cmd, -43, -44, extra)
-        lambda_blip = lambda: blip
-
-        def command_print(cmd, extra):
-            if extra == 'print-me':
-                stuff = dbstate.stuff
-            elif extra == '$0':
-                stuff = dbstate.metavar
-            elif extra == '2.35':
-                val = rdtoa.strtod('2.35')
-                valx, valy = math.modf(val)
-                revdb.send_output(rdtoa.dtoa(valx) + '\n')
-                revdb.send_output(rdtoa.dtoa(valy) + '\n')
-                xx, yy = math.frexp(val)
-                revdb.send_output(rdtoa.dtoa(xx) + '\n')
-                revdb.send_output('%d\n' % yy)
-                return
-            elif extra == 'very-long-loop':
-                i = 0
-                total = 0
-                while i < 2000000000:
-                    total += revdb.flag_io_disabled()
-                    i += 1
-                revdb.send_output(str(total))
-                return
-            else:
-                assert False
-            uid = revdb.get_unique_id(stuff)
-            ll_assert(uid > 0, "uid == 0")
-            revdb.send_nextnid(uid)   # outputs '$NUM = '
-            revdb.send_output('stuff\n')
-            dbstate.printed_stuff = stuff
-        lambda_print = lambda: command_print
-
-        def command_attachid(cmd, extra):
-            index_metavar = cmd.c_arg1
-            uid = cmd.c_arg2
-            ll_assert(index_metavar == 0, "index_metavar != 0")  # in this test
-            dbstate.metavar = dbstate.printed_stuff
-            if dbstate.metavar is None:
-                # uid not found, probably a future object
-                dbstate.watch_future = uid
-        lambda_attachid = lambda: command_attachid
-
-        def command_allocating(uid, gcref):
-            stuff = cast_gcref_to_instance(Stuff, gcref)
-            # 'stuff' is just allocated; 'stuff.x' is not yet initialized
-            dbstate.printed_stuff = stuff
-            if dbstate.watch_future != -1:
-                ll_assert(dbstate.watch_future == uid,
-                          "watch_future out of sync")
-                dbstate.watch_future = -1
-                dbstate.metavar = stuff
-        lambda_allocating = lambda: command_allocating
-
-        def command_compilewatch(cmd, expression):
-            revdb.send_watch("marshalled_code", ok_flag=1)
-        lambda_compilewatch = lambda: command_compilewatch
-
-        def command_checkwatch(cmd, marshalled_code):
-            assert marshalled_code == "marshalled_code"
-            # check that $0 exists
-            if dbstate.metavar is not None:
-                revdb.send_watch("ok, stuff exists\n", ok_flag=1)
-            else:
-                revdb.send_watch("stuff does not exist!\n", ok_flag=0)
-        lambda_checkwatch = lambda: command_checkwatch
-
-        def main(argv):
-            revdb.register_debug_command(100, lambda_blip)
-            revdb.register_debug_command(CMD_PRINT, lambda_print)
-            revdb.register_debug_command(CMD_ATTACHID, lambda_attachid)
-            revdb.register_debug_command("ALLOCATING", lambda_allocating)
-            revdb.register_debug_command(revdb.CMD_COMPILEWATCH,
-                                         lambda_compilewatch)
-            revdb.register_debug_command(revdb.CMD_CHECKWATCH,
-                                         lambda_checkwatch)
-            for i, op in enumerate(argv[1:]):
-                dbstate.stuff = Stuff()
-                dbstate.stuff.x = i + 1000
-                if i == dbstate.break_loop or i == dbstate.break_loop + 1:
-                    revdb.breakpoint(99)
-                revdb.stop_point()
-                print op
-            return 9
-        compile(cls, main, backendopt=False)
-        assert run(cls, 'abc d ef g h i j k l m') == (
-            'abc\nd\nef\ng\nh\ni\nj\nk\nl\nm\n')
-
-
-    def test_init(self):
-        group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        assert group.get_max_time() == 10
-        assert group.get_next_clone_time() == 4
-
-    def test_forward(self):
-        group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.go_forward(100)
-        assert group.get_current_time() == 10
-        assert sorted(group.paused) == [1, 4, 6, 8, 9, 10]
-        assert group._check_current_time(10)
-
-    @given(strategies.lists(strategies.integers(min_value=1, max_value=10)))
-    def test_jump_in_time(self, target_times):
-        group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        for target_time in target_times:
-            group.jump_in_time(target_time)
-            group._check_current_time(target_time)
-
-    def test_breakpoint_b(self):
-        group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.active.send(Message(100, 6, extra='set-breakpoint'))
-        group.active.expect(42, 100, -43, -44, 'set-breakpoint')
-        group.active.expect(ANSWER_READY, 1, Ellipsis)
-        e = py.test.raises(Breakpoint, group.go_forward, 10, 'b')
-        assert e.value.time == 7
-        assert e.value.nums == [99]
-        group._check_current_time(7)
-
-    def test_breakpoint_r(self):
-        group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.active.send(Message(100, 6, extra='set-breakpoint'))
-        group.active.expect(42, 100, -43, -44, 'set-breakpoint')
-        group.active.expect(ANSWER_READY, 1, Ellipsis)
-        e = py.test.raises(Breakpoint, group.go_forward, 10, 'r')
-        assert e.value.time == 7
-        assert e.value.nums == [99]
-        group._check_current_time(10)
-
-    def test_breakpoint_i(self):
-        group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.active.send(Message(100, 6, extra='set-breakpoint'))
-        group.active.expect(42, 100, -43, -44, 'set-breakpoint')
-        group.active.expect(ANSWER_READY, 1, Ellipsis)
-        group.go_forward(10, 'i')    # does not raise Breakpoint
-
-    def test_print_cmd(self):
-        group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        group.go_forward(1)
-        assert group.get_current_time() == 2
-        with stdout_capture() as buf:
-            group.print_cmd('print-me')
-        assert buf.getvalue() == "$0 = stuff\n"
-        return group
-
-    def _print_metavar(self, group):
-        with stdout_capture() as buf:
-            group.print_cmd('$0', nids=[0])
-        assert buf.getvalue() == "$0 = stuff\n"
-
-    def test_print_metavar(self):
-        group = self.test_print_cmd()
-        self._print_metavar(group)
-
-    def test_jump_and_print_metavar(self):
-        group = self.test_print_cmd()
-        assert group.is_tainted()
-        group.jump_in_time(2)
-        self._print_metavar(group)
-
-    def _check_watchpoint_expr(self, group, must_exist):
-        ok_flag, compiled_code = group.compile_watchpoint_expr("$0")
-        assert ok_flag == 1
-        assert compiled_code == "marshalled_code"
-        nids = [0]
-        ok_flag, text = group.check_watchpoint_expr(compiled_code, nids)
-        print text
-        assert ok_flag == must_exist
-
-    def test_check_watchpoint_expr(self):
-        group = self.test_print_cmd()
-        self._check_watchpoint_expr(group, must_exist=1)
-
-    def test_jump_and_check_watchpoint_expr(self):
-        group = self.test_print_cmd()
-        group.jump_in_time(2)
-        self._check_watchpoint_expr(group, must_exist=1)
-
-    def test_rdtoa(self):
-        group = ReplayProcessGroup(str(self.exename), self.rdbname)
-        with stdout_capture() as buf:
-            group.print_cmd('2.35')
-        assert buf.getvalue() == "0.35\n2.0\n0.5875\n2\n"
-
-    def test_ctrl_c(self):
-        localdir = os.path.dirname(__file__)
-        args = [sys.executable, os.path.join(localdir, 'ctrl_c.py'),
-                '\x7f'.join(sys.path),
-                str(self.exename), self.rdbname]
-        t1 = time.time()
-        result = subprocess.check_output(args)
-        t2 = time.time()
-        print 'subprocess returned with captured stdout:\n%r' % (result,)
-        assert result == 'all ok\n'
-        # should take two times ~0.8 seconds if correctly interrupted
-        assert t2 - t1 < 3.0
diff --git a/rpython/translator/revdb/test/test_raw.py b/rpython/translator/revdb/test/test_raw.py
deleted file mode 100644
--- a/rpython/translator/revdb/test/test_raw.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import os, subprocess
-from rpython.rlib import revdb
-from rpython.rtyper.lltypesystem import lltype
-from rpython.translator.revdb.test.test_basic import InteractiveTests
-
-from rpython.translator.revdb.message import *
-
-
-class TestReplayingRaw(InteractiveTests):
-    expected_stop_points = 1
-
-    def setup_class(cls):
-        from rpython.translator.revdb.test.test_basic import compile, run
-        from rpython.translator.revdb.test.test_basic import fetch_rdb
-
-        FOO = lltype.Struct('FOO')
-        foo = lltype.malloc(FOO, flavor='raw', immortal=True)
-
-        BAR = lltype.Struct('BAR', ('p', lltype.Ptr(FOO)))
-        bar = lltype.malloc(BAR, flavor='raw', immortal=True)
-        bar.p = foo
-
-        BAZ = lltype.Struct('BAZ', ('p', lltype.Ptr(FOO)), ('q', lltype.Signed),
-                            hints={'union': True})
-        baz = lltype.malloc(BAZ, flavor='raw', immortal=True)
-        baz.p = foo
-
-        VBAR = lltype.Array(lltype.Ptr(FOO))
-        vbar = lltype.malloc(VBAR, 3, flavor='raw', immortal=True)
-        vbar[0] = vbar[1] = vbar[2] = foo
-
-        RECBAR = lltype.Struct('RECBAR', ('super', BAR), ('q', lltype.Ptr(FOO)))
-        recbar = lltype.malloc(RECBAR, flavor='raw', immortal=True)
-        recbar.q = foo
-        recbar.super.p = foo
-
-        IBAR = lltype.Struct('IBAR', ('p', lltype.Ptr(FOO)),
-                             hints={'static_immutable': True})
-        ibar = lltype.malloc(IBAR, flavor='raw', immortal=True)
-        ibar.p = foo
-
-        BARI = lltype.Struct('BARI', ('b', lltype.Ptr(IBAR)))
-        bari = lltype.malloc(BARI, flavor='raw', immortal=True)
-        bari.b = ibar
-
-        class X:
-            pass
-        x = X()
-        x.foo = foo
-        x.ibar = ibar
-        x.bari = bari
-
-        def main(argv):
-            assert bar.p == foo
-            assert baz.p == foo
-            for i in range(3):
-                assert vbar[i] == foo
-            assert recbar.q == foo
-            assert recbar.super.p == foo
-            assert ibar.p == foo
-            assert bari.b == ibar
-            assert x.foo == foo
-            assert x.ibar == ibar
-            assert x.bari == bari
-            revdb.stop_point()
-            return 9
-
-        compile(cls, main, backendopt=False, shared=True)
-        run(cls, '')
-        rdb = fetch_rdb(cls, [cls.exename])
-        #assert len(rdb.rdb_struct) >= 4
-
-    def test_replaying_raw(self):
-        # This tiny test seems to always have foo at the same address
-        # in multiple runs.  Here we recompile with different options
-        # just to change that address.
-        #
-        # NOTE: not supported right now!  The executable must be
-        # exactly the same one with the same raw addresses.  This
-        # might be fixed in the future.
-        #subprocess.check_call(["make", "clean"],
-        #                      cwd=os.path.dirname(str(self.exename)))
-        #subprocess.check_call(["make", "lldebug"],
-        #                      cwd=os.path.dirname(str(self.exename)))
-        #
-        child = self.replay()
-        child.send(Message(CMD_FORWARD, 2))
-        child.expect(ANSWER_AT_END)
diff --git a/rpython/translator/revdb/test/test_rawrefcount.py b/rpython/translator/revdb/test/test_rawrefcount.py
deleted file mode 100644
--- a/rpython/translator/revdb/test/test_rawrefcount.py
+++ /dev/null
@@ -1,61 +0,0 @@
-from rpython.rlib import objectmodel, rgc, revdb
-from rpython.rtyper.lltypesystem import lltype
-from rpython.translator.revdb.test.test_basic import InteractiveTests
-from rpython.translator.revdb.test.test_basic import compile, fetch_rdb, run
-from rpython.translator.revdb.message import *
-
-from rpython.rlib import rawrefcount
-
-
-class TestRawRefcount(InteractiveTests):
-    expected_stop_points = 27
-
-    def setup_class(cls):
-        class W_Root(object):
-            def __init__(self, n):
-                self.n = n
-        PyObjectS = lltype.Struct('PyObjectS',
-                                  ('c_ob_refcnt', lltype.Signed),
-                                  ('c_ob_pypy_link', lltype.Signed))
-        PyObject = lltype.Ptr(PyObjectS)
-        w1 = W_Root(-42)
-        ob1 = lltype.malloc(PyObjectS, flavor='raw', zero=True,
-                            immortal=True)
-        ob1.c_ob_refcnt = rawrefcount.REFCNT_FROM_PYPY
-
-        def main(argv):
-            rawrefcount.create_link_pypy(w1, ob1)
-            w = None
-            ob = lltype.nullptr(PyObjectS)
-            oblist = []
-            for op in argv[1:]:
-                revdb.stop_point()
-                w = W_Root(42)
-                ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
-                ob.c_ob_refcnt = rawrefcount.REFCNT_FROM_PYPY
-                rawrefcount.create_link_pypy(w, ob)
-                oblist.append(ob)
-            del oblist[-1]
-            #
-            rgc.collect()
-            assert rawrefcount.from_obj(PyObject, w) == ob
-            assert rawrefcount.to_obj(W_Root, ob) == w
-            while True:
-                ob = rawrefcount.next_dead(PyObject)
-                if not ob:
-                    break
-                assert ob in oblist
-                oblist.remove(ob)
-            objectmodel.keepalive_until_here(w)
-            revdb.stop_point()
-            return 9
-        compile(cls, main, backendopt=False)
-        ARGS26 = 'a b c d e f g h i j k l m n o p q r s t u v w x y z'
-        run(cls, ARGS26)
-        rdb = fetch_rdb(cls, [cls.exename] + ARGS26.split())
-        assert rdb.number_of_stop_points() == cls.expected_stop_points
-
-    def test_go(self):
-        child = self.replay()
-        child.send(Message(CMD_FORWARD, 50))
-        child.expect(ANSWER_AT_END)
diff --git a/rpython/translator/revdb/test/test_thread.py b/rpython/translator/revdb/test/test_thread.py
deleted file mode 100644
--- a/rpython/translator/revdb/test/test_thread.py
+++ /dev/null
@@ -1,213 +0,0 @@
-from rpython.translator.revdb.test.test_basic import BaseRecordingTests
-from rpython.translator.revdb.test.test_basic import InteractiveTests
-from rpython.rtyper.lltypesystem import rffi
-from rpython.rlib import rthread
-from rpython.rlib import revdb
-
-from rpython.translator.revdb.message import *
-
-
-_sleep = rffi.llexternal('sleep', [rffi.UINT], rffi.UINT)
-
-
-class TestThreadRecording(BaseRecordingTests):
-
-    def test_thread_simple(self):
-        def bootstrap():
-            rthread.gc_thread_start()
-            _sleep(1)
-            print "BB"
-            _sleep(2)
-            print "BBB"
-            rthread.gc_thread_die()
-
-        def main(argv):
-            print "A"
-            rthread.start_new_thread(bootstrap, ())
-            for i in range(2):
-                _sleep(2)
-                print "AAAA"
-            return 9
-
-        self.compile(main, backendopt=False, thread=True)
-        out = self.run('Xx')
-        # should have printed A, BB, AAAA, BBB, AAAA
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        th_A = rdb.main_thread_id
-        rdb.write_call("A\n")
-        rdb.same_stack()      # RPyGilAllocate()
-        rdb.gil_release()
-
-        th_B = rdb.switch_thread()
-        assert th_B != th_A
-        b = rdb.next('!h'); assert 300 <= b < 310   # "callback": start thread
-        rdb.gil_acquire()
-        rdb.gil_release()
-
-        rdb.switch_thread(th_A)
-        rdb.same_stack()      # start_new_thread returns
-        x = rdb.next(); assert x == th_B    # result is the 'th_B' id
-        rdb.gil_acquire()
-        rdb.gil_release()
-
-        rdb.switch_thread(th_B)
-        rdb.same_stack()      # sleep() (finishes here)
-        rdb.next('i')         # sleep()
-        rdb.gil_acquire()
-        rdb.write_call("BB\n")
-        rdb.gil_release()
-
-        rdb.switch_thread(th_A)
-        rdb.same_stack()      # sleep()
-        rdb.next('i')         # sleep()
-        rdb.gil_acquire()
-        rdb.write_call("AAAA\n")
-        rdb.gil_release()
-
-        rdb.switch_thread(th_B)
-        rdb.same_stack()      # sleep()
-        rdb.next('i')         # sleep()
-        rdb.gil_acquire()
-        rdb.write_call("BBB\n")
-        rdb.gil_release()
-
-        rdb.switch_thread(th_A)
-        rdb.same_stack()      # sleep()
-        rdb.next('i')         # sleep()
-        rdb.gil_acquire()
-        rdb.write_call("AAAA\n")
-        rdb.done()
-
-    def test_threadlocal(self):
-        class EC(object):
-            def __init__(self, value):
-                self.value = value
-        raw_thread_local = rthread.ThreadLocalReference(EC)
-
-        def bootstrap():
-            rthread.gc_thread_start()
-            _sleep(1)
-            ec = EC(4567)
-            raw_thread_local.set(ec)
-            print raw_thread_local.get().value
-            assert raw_thread_local.get() is ec
-            rthread.gc_thread_die()
-
-        def main(argv):
-            ec = EC(12)
-            raw_thread_local.set(ec)
-            rthread.start_new_thread(bootstrap, ())
-            _sleep(2)
-            print raw_thread_local.get().value
-            assert raw_thread_local.get() is ec
-            return 9
-
-        self.compile(main, backendopt=False, thread=True)
-        out = self.run('Xx')
-        # should have printed 4567 and 12
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        th_A = rdb.main_thread_id
-        rdb.same_stack()      # RPyGilAllocate()
-        rdb.gil_release()
-
-        th_B = rdb.switch_thread()
-        assert th_B != th_A
-        b = rdb.next('!h'); assert 300 <= b < 310   # "callback": start thread
-        rdb.gil_acquire()
-        rdb.gil_release()
-
-        rdb.switch_thread(th_A)
-        rdb.same_stack()      # start_new_thread returns
-        x = rdb.next(); assert x == th_B    # result is the 'th_B' id
-        rdb.gil_acquire()
-        rdb.gil_release()
-
-        rdb.switch_thread(th_B)
-        rdb.same_stack()      # sleep() (finishes here)
-        rdb.next('i')         # sleep()
-        rdb.gil_acquire()
-        rdb.write_call("4567\n")
-        rdb.gil_release()
-
-        rdb.switch_thread(th_A)
-        rdb.same_stack()      # sleep()
-        rdb.next('i')         # sleep()
-        rdb.gil_acquire()
-        rdb.write_call("12\n")
-        rdb.done()
-
-
-class TestThreadInteractive(InteractiveTests):
-    expected_stop_points = 5
-
-    def setup_class(cls):
-        from rpython.translator.revdb.test.test_basic import compile, run
-        def bootstrap():
-            rthread.gc_thread_start()
-            _sleep(1)
-            revdb.stop_point()
-            _sleep(2)
-            revdb.stop_point()
-            rthread.gc_thread_die()
-
-        def main(argv):
-            revdb.stop_point()
-            rthread.start_new_thread(bootstrap, ())
-            for i in range(2):
-                _sleep(2)
-                revdb.stop_point()
-            print "ok"
-            return 9
-
-        compile(cls, main, backendopt=False, thread=True)
-        assert run(cls, '') == 'ok\n'
-
-    def test_go(self):
-        child = self.replay()
-        for i in range(2, 6):
-            child.send(Message(CMD_FORWARD, 1))
-            child.expect(ANSWER_READY, i, Ellipsis,
-                         (i & 1) ^ 1)    # thread number: either 0 or 1 here
-        child.send(Message(CMD_FORWARD, 1))
-        child.expect(ANSWER_AT_END)
-
-
-class TestThreadLocal(InteractiveTests):
-    expected_stop_points = 2
-
-    def setup_class(cls):
-        from rpython.translator.revdb.test.test_basic import compile, run
-        class EC(object):
-            def __init__(self, value):
-                self.value = value
-        raw_thread_local = rthread.ThreadLocalReference(EC)
-
-        def bootstrap():
-            rthread.gc_thread_start()
-            _sleep(1)
-            ec = EC(4567)
-            raw_thread_local.set(ec)
-            revdb.stop_point()
-            print raw_thread_local.get().value
-            assert raw_thread_local.get() is ec
-            rthread.gc_thread_die()
-
-        def main(argv):
-            revdb.stop_point()
-            ec = EC(12)
-            raw_thread_local.set(ec)
-            rthread.start_new_thread(bootstrap, ())
-            _sleep(2)
-            print raw_thread_local.get().value
-            assert raw_thread_local.get() is ec
-            return 9
-
-        compile(cls, main, backendopt=False, thread=True)
-        assert run(cls, '') == '4567\n12\n'
-
-    def test_go_threadlocal(self):
-        child = self.replay()
-        child.send(Message(CMD_FORWARD, 1))
-        child.expect(ANSWER_READY, 2, Ellipsis, 1)
-        child.send(Message(CMD_FORWARD, 1))
-        child.expect(ANSWER_AT_END)
diff --git a/rpython/translator/revdb/test/test_weak.py b/rpython/translator/revdb/test/test_weak.py
deleted file mode 100644
--- a/rpython/translator/revdb/test/test_weak.py
+++ /dev/null
@@ -1,357 +0,0 @@
-import py, weakref
-from rpython.rlib import revdb, rgc
-from rpython.rlib.debug import debug_print
-from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rlib.rarithmetic import intmask
-from rpython.translator.revdb.message import *
-from rpython.translator.revdb.test.test_basic import BaseRecordingTests
-from rpython.translator.revdb.test.test_basic import InteractiveTests
-
-
-# Weakrefs: implemented so that the log file contains one byte
-# "afterwards_alive" or "afterwards_dead" at the point where the
-# weakref is created, and similarly one such byte at every point where
-# the weakref is dereferenced.  At every point, the weakref is _alive_
-# at the moment; but the byte tells whether it should stay alive until
-# the _next_ point or not.  Done by always emitting "afterwards_dead"
-# in the log, and patching that to "afterwards_alive" if later we find
-# a deref() where the weakref is still alive.  (If a deref() finds the
-# weakref dead, it doesn't do any recording or patching; it simply
-# leaves the previous already-written "afterwards_dead" byte.)
-
-
-WEAKREF_AFTERWARDS_DEAD  = chr(0xf2)
-WEAKREF_AFTERWARDS_ALIVE = chr(0xeb)
-
-ASYNC_FINALIZER_TRIGGER  = 0xff46 - 2**16
-
-
-def get_finalizer_queue_main():
-    from rpython.rtyper.lltypesystem import lltype, rffi
-    #
-    from rpython.translator.tool.cbuild import ExternalCompilationInfo
-    eci = ExternalCompilationInfo(
-        pre_include_bits=["#define foobar(x) x\n"])
-    foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed,
-                             compilation_info=eci)
-    class Glob:
-        pass
-    glob = Glob()
-    class X:
-        pass
-    class MyFinalizerQueue(rgc.FinalizerQueue):
-        Class = X
-        def finalizer_trigger(self):
-            glob.ping = True
-    fq = MyFinalizerQueue()
-    #
-    def main(argv):
-        glob.ping = False
-        lst1 = [X() for i in range(256)]
-        lst = [X() for i in range(3000)]
-        for i, x in enumerate(lst):
-            x.baz = i
-            fq.register_finalizer(x)
-        for i in range(3000):
-            lst[i] = None
-            if i % 300 == 150:
-                rgc.collect()
-            revdb.stop_point()
-            j = i + glob.ping * 1000000
-            assert foobar(j) == j
-            if glob.ping:
-                glob.ping = False
-                total = 0
-                while True:
-                    x = fq.next_dead()
-                    if x is None:
-                        break
-                    total = intmask(total * 3 + x.baz)
-                assert foobar(total) == total
-        keepalive_until_here(lst1)
-        return 9
-    return main
-
-def get_old_style_finalizer_main():
-    from rpython.rtyper.lltypesystem import lltype, rffi
-    from rpython.translator.tool.cbuild import ExternalCompilationInfo
-    #
-    eci = ExternalCompilationInfo(
-        pre_include_bits=["#define foobar(x) x\n"])
-    foobar = rffi.llexternal('foobar', [lltype.Signed], lltype.Signed,
-                             compilation_info=eci, _nowrapper=True)
-    class Glob:
-        pass
-    glob = Glob()
-    class X:
-        def __del__(self):
-            assert foobar(-7) == -7
-            glob.count += 1
-    def main(argv):
-        glob.count = 0
-        lst = [X() for i in range(3000)]
-        x = -1
-        for i in range(3000):
-            lst[i] = None
-            if i % 300 == 150:
-                rgc.collect()
-            revdb.stop_point()
-            x = glob.count
-            assert foobar(x) == x
-        print x
-        return 9
-    return main
-
-
-class TestRecording(BaseRecordingTests):
-
-    def test_weakref_create(self):
-        class X:
-            pass
-        class Glob:
-            pass
-        glob = Glob()
-        def main(argv):
-            glob.r1 = weakref.ref(X())
-            glob.r2 = weakref.ref(X())
-            glob.r3 = weakref.ref(X())
-            return 9
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        # find the extra WEAKREF_DEAD
-        x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD
-        x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD
-        x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD
-        x = rdb.next('q'); assert x == 0      # number of stop points
-        assert rdb.done()
-
-    def test_weakref_deref_nondead(self):
-        class X:
-            pass
-        class Glob:
-            pass
-        glob = Glob()
-        def main(argv):
-            x1 = X(); x2 = X()
-            r1 = weakref.ref(x1)     # (*)
-            r2 = weakref.ref(x2)     # (*)
-            for i in range(8500):
-                assert r1() is x1    # (*)
-                assert r2() is x2    # (*)
-            return 9
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        # find the 2 + 16998 first WEAKREF_xxx (all "(*)" but the last two)
-        for i in range(2 + 16998):
-            x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_ALIVE
-        for i in range(2):
-            x = rdb.next('c'); assert x == WEAKREF_AFTERWARDS_DEAD
-        x = rdb.next('q'); assert x == 0      # number of stop points
-        assert rdb.done()
-
-    def test_prebuilt_weakref(self):
-        class X:
-            pass
-        x1 = X()
-        x1.foobar = 9
-        wr = weakref.ref(x1)
-        def main(argv):
-            X().foobar = 43
-            return wr().foobar
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        # the weakref is prebuilt, so doesn't generate any WEAKREF_xxx
-        x = rdb.next('q'); assert x == 0      # number of stop points
-        assert rdb.done()
-
-    def test_finalizer_light_ignored(self):
-        py.test.skip("lightweight finalizers could be skipped, but that "
-                     "requires also skipping (instead of recording) any "
-                     "external call they do")
-        class X:
-            @rgc.must_be_light_finalizer
-            def __del__(self):
-                pass
-        def main(argv):
-            lst = [X() for i in range(3000)]
-            for i in range(3000):
-                lst[i] = None
-                if i % 300 == 150:
-                    rgc.collect()
-                revdb.stop_point()
-            return 9
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        x = rdb.next('q'); assert x == 3000    # number of stop points
-        assert rdb.done()
-
-    def test_finalizer_queue(self):
-        main = get_finalizer_queue_main()
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        uid_seen = set()
-        totals = []
-        for i in range(3000):
-            triggered = False
-            if rdb.is_special_packet():
-                time, = rdb.special_packet(ASYNC_FINALIZER_TRIGGER, 'q')
-                assert time == i + 1
-                y = intmask(rdb.next('q')); assert y == -1
-                triggered = True
-            rdb.gil_release()
-            rdb.same_stack()  #
-            j = rdb.next()    # call to foobar()
-            rdb.gil_acquire()
-            assert j == i + 1000000 * triggered
-            if triggered:
-                lst = []
-                while True:
-                    uid = intmask(rdb.next())
-                    if uid == -1:
-                        break
-                    assert uid > 0 and uid not in uid_seen
-                    uid_seen.add(uid)
-                    lst.append(uid)
-                rdb.gil_release()
-                rdb.same_stack()                           #
-                totals.append((lst, intmask(rdb.next())))  # call to foobar()
-                rdb.gil_acquire()
-        x = rdb.next('q'); assert x == 3000    # number of stop points
-        #
-        assert 1500 <= len(uid_seen) <= 3000
-        d = dict(zip(sorted(uid_seen), range(len(uid_seen))))
-        for lst, expected in totals:
-            total = 0
-            for uid in lst:
-                total = intmask(total * 3 + d[uid])
-            assert total == expected
-
-    def test_old_style_finalizer(self):
-        main = get_old_style_finalizer_main()
-        self.compile(main, backendopt=False)
-        out = self.run('Xx')
-        assert 1500 < int(out) <= 3000
-        rdb = self.fetch_rdb([self.exename, 'Xx'])
-        seen_uids = set()
-        for i in range(3000):
-            triggered = False
-            if rdb.is_special_packet():
-                time, = rdb.special_packet(ASYNC_FINALIZER_TRIGGER, 'q')
-                assert time == i + 1
-                triggered = True
-                x = intmask(rdb.next())
-                while True:
-                    assert x != -1
-                    assert x not in seen_uids
-                    seen_uids.add(x)
-                    rdb.same_stack()
-                    y = intmask(rdb.next())
-                    assert y == -7      # from the __del__
-                    x = intmask(rdb.next())
-                    if x == -1:
-                        break
-            rdb.same_stack()
-            x = rdb.next()
-            assert x == len(seen_uids)
-        assert len(seen_uids) == int(out)
-        rdb.write_call(out)
-        x = rdb.next('q'); assert x == 3000    # number of stop points
-
-
-class TestReplayingWeakref(InteractiveTests):
-    expected_stop_points = 1
-
-    def setup_class(cls):
-        from rpython.translator.revdb.test.test_basic import compile, run
-
-        class X:
-            def __init__(self, s):
-                self.s = s
-        prebuilt = X('prebuilt')
-
-        def make(s):
-            lst = [prebuilt] + [X(c) for c in s]
-            keepalive = lst[-1]
-            return [weakref.ref(x) for x in lst], keepalive
-
-        def main(argv):
-            lst, keepalive = make(argv[0])
-            expected = ['prebuilt'] + [c for c in argv[0]]
-            dead = [False] * len(lst)
-            for j in range(17000):
-                outp = []
-                for i in range(len(lst)):
-                    v = lst[i]()
-                    debug_print(v)
-                    if dead[i]:
-                        assert v is None
-                    elif v is None:
-                        outp.append('<DEAD>')
-                        dead[i] = True
-                    else:
-                        outp.append(v.s)
-                        assert v.s == expected[i]
-                print ''.join(outp)
-                if (j % 1000) == 999:
-                    debug_print('============= COLLECT ===========')
-                    rgc.collect()
-                debug_print('------ done', j, '.')
-            assert not dead[0]
-            assert not dead[-1]
-            keepalive_until_here(keepalive)
-            revdb.stop_point()
-            return 9
-        compile(cls, main, backendopt=False)
-        output = run(cls, '')
-        lines = output.splitlines()
-        assert lines[-1].startswith('prebuilt') and lines[-1].endswith(
-            str(cls.exename)[-1])
-        assert (len(lines[-1]) + output.count('<DEAD>') ==
-                len('prebuilt') + len(str(cls.exename)))
-
-    def test_replaying_weakref(self):
-        child = self.replay()
-        # the asserts are replayed; if we get here it means they passed again
-        child.send(Message(CMD_FORWARD, 1))
-        child.expect(ANSWER_AT_END)
-
-
-class TestReplayingFinalizerQueue(InteractiveTests):
-    expected_stop_points = 3000
-
-    def setup_class(cls):
-        from rpython.translator.revdb.test.test_basic import compile, run
-        main = get_finalizer_queue_main()
-        compile(cls, main, backendopt=False)
-        run(cls, '')
-
-    def test_replaying_finalizer_queue(self):
-        child = self.replay()
-        child.send(Message(CMD_FORWARD, 3001))
-        child.expect(ANSWER_AT_END)
-
-
-class TestReplayingOldStyleFinalizer(InteractiveTests):
-    expected_stop_points = 3000
-
-    def setup_class(cls):
-        from rpython.translator.revdb.test.test_basic import compile, run
-        main = get_old_style_finalizer_main()
-        compile(cls, main, backendopt=False)
-        run(cls, '')
-
-    def test_replaying_old_style_finalizer(self):
-        child = self.replay()
-        child.send(Message(CMD_FORWARD, 3001))
-        child.expect(ANSWER_AT_END)
-
-    def test_bug1(self):
-        child = self.replay()
-        for i in range(50):
-            child.send(Message(CMD_FORWARD, i))
-            child.expect_ready()

From pypy.commits at gmail.com  Fri Sep  9 05:01:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 09 Sep 2016 02:01:59 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Disable micronumpy as well,
 for now
Message-ID: <57d27a87.e440c20a.881ca.b489@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86970:dcba9f2816ab
Date: 2016-09-09 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/dcba9f2816ab/

Log:	Disable micronumpy as well, for now

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -57,6 +57,7 @@
 
 reverse_debugger_disable_modules = set([
     "_continuation", "_vmprof", "_multiprocessing",
+    "micronumpy",
     ])
 
 # XXX this should move somewhere else, maybe to platform ("is this posixish"

From pypy.commits at gmail.com  Fri Sep  9 05:04:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 09 Sep 2016 02:04:13 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Rename the env var REVDB
Message-ID: <57d27b0d.255ac20a.7ccce.b2fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86971:4909c06daf41
Date: 2016-09-09 11:03 +0200
http://bitbucket.org/pypy/pypy/changeset/4909c06daf41/

Log:	Rename the env var REVDB

diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -200,7 +200,7 @@
             skip the caching logic
 
     inside getter methods or properties, to make them usable from
-    watchpoints.  Note that you need to re-run ``PYPYRDB=.. pypy''
+    watchpoints.  Note that you need to re-run ``REVDB=.. pypy''
     after changing the Python code.
     """
     return space.wrap(space._side_effects_ok())
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -91,7 +91,7 @@
 
            Potentially buggy to use argv[0] here, but good enough I
            suppose.  For this reason ensure_fixed_address_space() is
-           not called when running manually without any PYPYRDB
+           not called when running manually without any REVDB
            environment variable set.
         */
         execv(argv[0], argv);
@@ -207,7 +207,7 @@
 
 static void setup_record_mode(int argc, char *argv[])
 {
-    char *filename = getenv("PYPYRDB");
+    char *filename = getenv("REVDB");
     rdb_header_t h;
     int i;
 
@@ -216,11 +216,11 @@
     if (filename && *filename) {
         ensure_fixed_address_space(argv);
 
-        putenv("PYPYRDB=");
+        putenv("REVDB=");
         rpy_rev_fileno = open(filename, O_RDWR | O_CLOEXEC |
                               O_CREAT | O_NOCTTY | O_TRUNC, 0600);
         if (rpy_rev_fileno < 0) {
-            fprintf(stderr, "Fatal error: can't create PYPYRDB file '%s'\n",
+            fprintf(stderr, "Fatal error: can't create REVDB file '%s'\n",
                     filename);
             abort();
         }
@@ -250,7 +250,7 @@
     }
     else {
         fprintf(stderr, "PID %d starting, log file disabled "
-                        "(use PYPYRDB=logfile)\n", (int)getpid());
+                        "(use REVDB=logfile)\n", (int)getpid());
     }
 
     rpy_revdb.buf_p = rpy_rev_buffer + sizeof(int16_t);

From pypy.commits at gmail.com  Fri Sep  9 08:40:35 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 09 Sep 2016 05:40:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: test, fix Py_buffer format,
 which can be a string (issue #2396 and IRC discussion)
Message-ID: <57d2adc3.a4fdc20a.9fe80.12fe@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86972:fe0add22fd7e
Date: 2016-09-09 12:37 +0300
http://bitbucket.org/pypy/pypy/changeset/fe0add22fd7e/

Log:	test, fix Py_buffer format, which can be a string (issue #2396 and
	IRC discussion)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,7 +119,7 @@
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
@@ -645,7 +645,7 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
-        ('_format', rffi.UCHAR),
+        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
         ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -144,6 +144,7 @@
 
 /* Py3k buffer interface, adapted for PyPy */
 #define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -158,7 +159,7 @@
     Py_ssize_t *shape;
     Py_ssize_t *strides;
     Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format;
+    unsigned char _format[Py_MAX_FMT];
     Py_ssize_t _strides[Py_MAX_NDIMS];
     Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,5 +1,5 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -41,10 +41,22 @@
     view.c_len = w_obj.getlength()
     view.c_itemsize = w_obj.buf.getitemsize()
     rffi.setintfield(view, 'c_ndim', ndim)
-    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
     view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
     view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
     view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = w_obj.buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        if ord(fmt[i]) > 255:
+            view.c_format[i] = '*'
+        else:
+            view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'        
     shape = w_obj.buf.getshape()
     strides = w_obj.buf.getstrides()
     for i in range(ndim):
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
 import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.api import PyObject
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,3 +1,4 @@
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
@@ -16,8 +17,12 @@
         w_buf = space.newbuffer(StringBuffer("hello"))
         w_memoryview = api.PyMemoryView_FromObject(w_buf)
         w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
-        ndim = w_view.c_ndim
-        assert ndim == 1
+        assert w_view.c_ndim == 1
+        f = rffi.charp2str(w_view.c_format)
+        assert f == 'B'
+        assert w_view.c_shape[0] == 5
+        assert w_view.c_strides[0] == 1
+        assert w_view.c_len == 5
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):

From pypy.commits at gmail.com  Fri Sep  9 08:40:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 09 Sep 2016 05:40:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: move all Py_buffer to memoryobject,
 create helper function
Message-ID: <57d2adc5.6937c20a.1be9a.14a2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86973:f8f2080418c8
Date: 2016-09-09 13:59 +0300
http://bitbucket.org/pypy/pypy/changeset/f8f2080418c8/

Log:	move all Py_buffer to memoryobject, create helper function

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,9 +1,7 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib.rarithmetic import widen
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
-from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
+from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -14,102 +12,4 @@
         return 1
     return 0  
 
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    flags = widen(flags)
-    buf = space.buffer_w(w_obj, flags)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-    except ValueError:
-        raise BufferError("could not create buffer from object")
-    view.c_len = buf.getlength()
-    view.c_obj = make_ref(space, w_obj)
-    ndim = buf.getndim()
-    view.c_itemsize = buf.getitemsize()
-    rffi.setintfield(view, 'c_readonly', int(buf.readonly))
-    rffi.setintfield(view, 'c_ndim', ndim)
-    view.c_format = rffi.str2charp(buf.getformat())
-    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
-    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
-    shape = buf.getshape()
-    strides = buf.getstrides()
-    for i in range(ndim):
-        view.c_shape[i] = shape[i]
-        view.c_strides[i] = strides[i]
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    return 0
-
-def _IsFortranContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(view.c_ndim):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-
-def _IsCContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(ndim - 1, -1, -1):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-        
-
- at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fort):
-    """Return 1 if the memory defined by the view is C-style (fortran is
-    'C') or Fortran-style (fortran is 'F') contiguous or either one
-    (fortran is 'A').  Return 0 otherwise."""
-    # traverse the strides, checking for consistent stride increases from
-    # right-to-left (c) or left-to-right (fortran). Copied from cpython
-    if not view.c_suboffsets:
-        return 0
-    if (fort == 'C'):
-        return _IsCContiguous(view)
-    elif (fort == 'F'):
-        return _IsFortranContiguous(view)
-    elif (fort == 'A'):
-        return (_IsCContiguous(view) or _IsFortranContiguous(view))
-    return 0
-
     
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -2,10 +2,124 @@
                          Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    return fill_Py_buffer(space, w_obj, view, flags)
+    view.c_obj = make_ref(space, w_obj)
+
+def fill_Py_buffer(space, buf, view):    
+    # c_buf, c_obj have been filled in
+    ndim = buf.getndim()
+    view.c_len = buf.getlength()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        if ord(fmt[i]) > 255:
+            view.c_format[i] = '*'
+        else:
+            view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'        
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+ at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+    """Return 1 if the memory defined by the view is C-style (fortran is
+    'C') or Fortran-style (fortran is 'F') contiguous or either one
+    (fortran is 'A').  Return 0 otherwise."""
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
     return space.call_method(space.builtin, "memoryview", w_obj)
@@ -38,31 +152,6 @@
         view.c_obj = make_ref(space, w_s)
         rffi.setintfield(view, 'c_readonly', 1)
         isstr = True
-    view.c_len = w_obj.getlength()
-    view.c_itemsize = w_obj.buf.getitemsize()
-    rffi.setintfield(view, 'c_ndim', ndim)
-    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
-    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-    fmt = w_obj.buf.getformat()
-    n = Py_MAX_FMT - 1 # NULL terminated buffer
-    if len(fmt) > n:
-        ### WARN?
-        pass
-    else:
-        n = len(fmt)
-    for i in range(n):
-        if ord(fmt[i]) > 255:
-            view.c_format[i] = '*'
-        else:
-            view.c_format[i] = fmt[i]
-    view.c_format[n] = '\x00'        
-    shape = w_obj.buf.getshape()
-    strides = w_obj.buf.getstrides()
-    for i in range(ndim):
-        view.c_shape[i] = shape[i]
-        view.c_strides[i] = strides[i]
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    fill_Py_buffer(space, w_obj.buf, view)     
     return view
 

From pypy.commits at gmail.com  Fri Sep  9 08:40:40 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 09 Sep 2016 05:40:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: make sure test of PyObject_GetBuffer
 runs to completion, fix implementation
Message-ID: <57d2adc8.948e1c0a.a05af.d5e9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86975:dcd57351c263
Date: 2016-09-09 15:13 +0300
http://bitbucket.org/pypy/pypy/changeset/dcd57351c263/

Log:	make sure test of PyObject_GetBuffer runs to completion, fix
	implementation

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -33,8 +33,8 @@
         view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
     except ValueError:
         raise BufferError("could not create buffer from object")
-    return fill_Py_buffer(space, w_obj, view, flags)
     view.c_obj = make_ref(space, w_obj)
+    return fill_Py_buffer(space, buf, view)
 
 def fill_Py_buffer(space, buf, view):    
     # c_buf, c_obj have been filled in
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -43,14 +43,10 @@
         from _numpypy import multiarray as np
         module = self.import_module(name='buffer_test')
         get_buffer_info = module.get_buffer_info
-        # test_export_flags from numpy test_multiarray
         raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
-        # test_relaxed_strides from numpy test_multiarray
-        arr = np.zeros((1, 10))
-        if arr.flags.f_contiguous:
-            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
-            assert strides[0] == 8
-            arr = np.ones((10, 1), order='F')
-            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
-            assert strides[-1] == 8
-
+        arr = np.zeros((1, 10), order='F')
+        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+        assert strides[0] == 8
+        arr = np.zeros((10, 1), order='C')
+        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+        assert strides[-1] == 8

From pypy.commits at gmail.com  Fri Sep  9 08:40:39 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 09 Sep 2016 05:40:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix unintentional skip of untranslated
 tests
Message-ID: <57d2adc7.05d71c0a.ab11f.0efb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r86974:0f729817ac87
Date: 2016-09-09 14:24 +0300
http://bitbucket.org/pypy/pypy/changeset/0f729817ac87/

Log:	fix unintentional skip of untranslated tests

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
 
 from .support import c_compile
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0
@@ -275,11 +277,11 @@
             "the test actually passed in the first place; if it failed "
             "it is likely to reach this place.")
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_only_import(self):
         import cpyext
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_load_error(self):
         import cpyext
         raises(ImportError, cpyext.load_module, "missing.file", "foo")
@@ -894,7 +896,7 @@
             ])
         raises(SystemError, mod.newexc, "name", Exception, {})
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy specific test')
+    @pytest.mark.skipif(only_pypy, reason='pypy specific test')
     def test_hash_pointer(self):
         mod = self.import_extension('foo', [
             ('get_hash', 'METH_NOARGS',
@@ -945,7 +947,7 @@
         print p
         assert 'py' in p
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_version(self):
         mod = self.import_extension('foo', [
             ('get_version', 'METH_NOARGS',
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -1,12 +1,13 @@
 import sys
 
-import py, pytest
+import pytest
 
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 class AppTestThread(AppTestCpythonExtensionBase):
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_thread_ident(self):
         module = self.import_extension('foo', [
             ("get_thread_ident", "METH_NOARGS",
@@ -33,7 +34,7 @@
 
         assert results[0][0] != results[1][0]
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_acquire_lock(self):
         module = self.import_extension('foo', [
             ("test_acquire_lock", "METH_NOARGS",
@@ -57,7 +58,7 @@
             ])
         module.test_acquire_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_release_lock(self):
         module = self.import_extension('foo', [
             ("test_release_lock", "METH_NOARGS",
@@ -79,7 +80,7 @@
             ])
         module.test_release_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_tls(self):
         module = self.import_extension('foo', [
             ("create_key", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,6 +3,7 @@
 import py, pytest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 def test_pragma_version():
     from pypy.module.sys.version import CPYTHON_VERSION
@@ -32,11 +33,9 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);

From pypy.commits at gmail.com  Fri Sep  9 09:47:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 09 Sep 2016 06:47:15 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Added tag RevDB-pypy2.7-v5.4.1
 for changeset 4909c06daf41
Message-ID: <57d2bd63.48811c0a.de509.278c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86976:bd755cdacd05
Date: 2016-09-09 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/bd755cdacd05/

Log:	Added tag RevDB-pypy2.7-v5.4.1 for changeset 4909c06daf41

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1

From pypy.commits at gmail.com  Fri Sep  9 12:02:19 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 09 Sep 2016 09:02:19 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase
Message-ID: <57d2dd0b.915c1c0a.e70f.5c1b@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86977:e2e9fdac5cf3
Date: 2016-09-09 18:01 +0200
http://bitbucket.org/pypy/pypy/changeset/e2e9fdac5cf3/

Log:	Update qcgc codebase

diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
--- a/rpython/translator/c/src/qcgc/allocator.c
+++ b/rpython/translator/c/src/qcgc/allocator.c
@@ -29,6 +29,8 @@
 		qcgc_arena_bag_create(QCGC_ARENA_BAG_INIT_SIZE);
 	qcgc_allocator_state.free_arenas = qcgc_arena_bag_create(4); // XXX
 
+	qcgc_allocator_state.use_bump_allocator = true;
+
 	// Bump Allocator
 	qcgc_allocator_state.bump_state.bump_ptr = NULL;
 	qcgc_allocator_state.bump_state.remaining_cells = 0;
@@ -254,17 +256,22 @@
 #endif
 	cell_t *result = NULL;
 	for ( ; index < QCGC_SMALL_FREE_LISTS; index++) {
-		linear_free_list_t *free_list =
-			qcgc_allocator_state.fit_state.small_free_list[index];
 		size_t list_cell_size = small_index_to_cells(index);
 
-		while (free_list->count > 0) {
-			result = free_list->items[free_list->count - 1];
-			free_list = qcgc_linear_free_list_remove_index(free_list,
-					free_list->count - 1);
+		while (qcgc_allocator_state.fit_state.small_free_list[index]->count
+				> 0) {
+			result = qcgc_allocator_state.fit_state.small_free_list[index]->
+				items[qcgc_allocator_state.fit_state.small_free_list[index]
+				->count - 1];
+			qcgc_allocator_state.fit_state.small_free_list[index] =
+				qcgc_linear_free_list_remove_index(
+						qcgc_allocator_state.fit_state.small_free_list[index],
+						qcgc_allocator_state.fit_state.small_free_list[index]->
+						count - 1);
 
 			// Check whether block is still valid
 			if (valid_block(result, list_cell_size)) {
+				// The next call might invalidate free_list, reload!
 				qcgc_fit_allocator_add(result + cells, list_cell_size - cells);
 				break;
 			} else {
@@ -272,7 +279,6 @@
 			}
 		}
 
-		qcgc_allocator_state.fit_state.small_free_list[index] = free_list;
 		if (result != NULL) {
 			return result;
 		}
@@ -285,24 +291,32 @@
 	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP) <= cells);
 	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP + 1) > cells);
 #endif
-	exp_free_list_t *free_list =
-		qcgc_allocator_state.fit_state.large_free_list[index];
-	size_t best_fit_index = free_list->count;
+	size_t best_fit_index = qcgc_allocator_state.fit_state.
+		large_free_list[index]->count;
 
 	cell_t *result = NULL;
 	size_t best_fit_cells = SIZE_MAX;
 	size_t i = 0;
-	while (i < free_list->count) {
-		if (valid_block(free_list->items[i].ptr, free_list->items[i].size)) {
-			if (free_list->items[i].size >= cells &&
-					free_list->items[i].size < best_fit_cells) {
-				result = free_list->items[i].ptr;
-				best_fit_cells = free_list->items[i].size;
+	while (i < qcgc_allocator_state.fit_state.large_free_list[index]->count) {
+		if (valid_block(qcgc_allocator_state.fit_state.large_free_list[index] ->
+					items[i].ptr,
+					qcgc_allocator_state.fit_state.large_free_list[index]->
+					items[i].size)) {
+			if (qcgc_allocator_state.fit_state.large_free_list[index]->
+					items[i].size >= cells &&
+					qcgc_allocator_state.fit_state.large_free_list[index]->
+					items[i].size < best_fit_cells) {
+				result = qcgc_allocator_state.fit_state.large_free_list[index]->
+					items[i].ptr;
+				best_fit_cells = qcgc_allocator_state.fit_state.
+					large_free_list[index]->items[i].size;
 				best_fit_index = i;
 			}
 			i++;
 		} else {
-			free_list = qcgc_exp_free_list_remove_index(free_list, i);
+			qcgc_allocator_state.fit_state.large_free_list[index] =
+				qcgc_exp_free_list_remove_index(qcgc_allocator_state.fit_state.
+						large_free_list[index], i);
 			// NO i++ !
 		}
 
@@ -313,14 +327,16 @@
 
 	if (result != NULL) {
 		// Best fit was found
-		assert(best_fit_index < free_list->count);
-		free_list = qcgc_exp_free_list_remove_index(free_list, best_fit_index);
+		assert(best_fit_index < qcgc_allocator_state.fit_state.
+				large_free_list[index]->count);
+		qcgc_allocator_state.fit_state.large_free_list[index] =
+			qcgc_exp_free_list_remove_index(qcgc_allocator_state.fit_state.
+					large_free_list[index], best_fit_index);
 		qcgc_fit_allocator_add(result + cells, best_fit_cells - cells);
 	} else {
 		// No best fit, go for first fit
 		result = fit_allocator_large_first_fit(index + 1, cells);
 	}
-	qcgc_allocator_state.fit_state.large_free_list[index] = free_list;
 	return result;
 }
 
@@ -330,13 +346,17 @@
 #endif
 	cell_t *result = NULL;
 	for ( ; index < QCGC_LARGE_FREE_LISTS; index++) {
-		exp_free_list_t *free_list =
-			qcgc_allocator_state.fit_state.large_free_list[index];
-		while(free_list->count > 0) {
+		while(qcgc_allocator_state.fit_state.large_free_list[index]->count
+				> 0) {
 			struct exp_free_list_item_s item =
-				free_list->items[free_list->count - 1];
-			free_list = qcgc_exp_free_list_remove_index(free_list,
-					free_list->count - 1);
+				qcgc_allocator_state.fit_state.large_free_list[index]->items[
+				qcgc_allocator_state.fit_state.large_free_list[index]->count - 1
+				];
+			qcgc_allocator_state.fit_state.large_free_list[index] =
+				qcgc_exp_free_list_remove_index(
+						qcgc_allocator_state.fit_state.large_free_list[index],
+						qcgc_allocator_state.fit_state.large_free_list[index]->
+						count - 1);
 
 			// Check whether block is still valid
 			if (valid_block(item.ptr, item.size)) {
@@ -345,7 +365,6 @@
 				break;
 			}
 		}
-		qcgc_allocator_state.fit_state.large_free_list[index] = free_list;
 		if (result != NULL) {
 			return result;
 		}
diff --git a/rpython/translator/c/src/qcgc/allocator.h b/rpython/translator/c/src/qcgc/allocator.h
--- a/rpython/translator/c/src/qcgc/allocator.h
+++ b/rpython/translator/c/src/qcgc/allocator.h
@@ -48,6 +48,7 @@
 		linear_free_list_t *small_free_list[QCGC_SMALL_FREE_LISTS];
 		exp_free_list_t *large_free_list[QCGC_LARGE_FREE_LISTS];
 	} fit_state;
+	bool use_bump_allocator;
 } qcgc_allocator_state;
 
 /**
diff --git a/rpython/translator/c/src/qcgc/arena.c b/rpython/translator/c/src/qcgc/arena.c
--- a/rpython/translator/c/src/qcgc/arena.c
+++ b/rpython/translator/c/src/qcgc/arena.c
@@ -11,6 +11,7 @@
 
 #include "allocator.h"
 #include "event_logger.h"
+#include "gc_state.h"
 
 /**
  * Internal functions
@@ -178,85 +179,124 @@
 	// No coalescing, collector will do this
 }
 
+bool qcgc_arena_pseudo_sweep(arena_t *arena) {
+#if CHECKED
+	assert(arena != NULL);
+	assert(qcgc_arena_is_coalesced(arena));
+	assert(qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) == arena);
+#endif
+	// XXX: Maybe ignore free cell / largest block counting here?
+	size_t last_free_cell = 0;
+	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
+			cell < QCGC_ARENA_CELLS_COUNT;
+			cell++) {
+		switch (get_blocktype(arena, cell)) {
+			case BLOCK_FREE:
+				last_free_cell = cell;
+			case BLOCK_EXTENT: // Fall through
+				break;
+			case BLOCK_BLACK:
+				set_blocktype(arena, cell, BLOCK_WHITE);
+			case BLOCK_WHITE: // Fall through
+				if (last_free_cell != 0) {
+					qcgc_state.free_cells += cell - last_free_cell;
+					qcgc_state.largest_free_block = MAX(
+							qcgc_state.largest_free_block,
+							cell - last_free_cell);
+					last_free_cell = 0;
+				}
+				break;
+		}
+	}
+	if (last_free_cell != 0) {
+		qcgc_state.free_cells += QCGC_ARENA_CELLS_COUNT - last_free_cell;
+		qcgc_state.largest_free_block = MAX(
+				qcgc_state.largest_free_block,
+				QCGC_ARENA_CELLS_COUNT - last_free_cell);
+		last_free_cell = 0;
+	}
+#if CHECKED
+	assert(qcgc_arena_is_coalesced(arena));
+#endif
+	return false;
+}
+
 bool qcgc_arena_sweep(arena_t *arena) {
 #if CHECKED
 	assert(arena != NULL);
 	assert(qcgc_arena_is_coalesced(arena));
+	//assert(qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) != arena);
 #endif
-#if DEBUG_ZERO_ON_SWEEP
-	bool zero = true;
-#endif
-	bool free = true;
-	bool coalesce = false;
-	bool add_to_free_list = false;
-	size_t last_free_cell = QCGC_ARENA_FIRST_CELL_INDEX;
-
 	if (qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) == arena) {
-		for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
-				cell < QCGC_ARENA_CELLS_COUNT;
-				cell++) {
-			if (get_blocktype(arena, cell) == BLOCK_BLACK) {
-				set_blocktype(arena, cell, BLOCK_WHITE);
-			}
-		}
-		return false;
+		return qcgc_arena_pseudo_sweep(arena);
 	}
 
+	size_t last_free_cell = 0;
+	bool register_free_block = false;
+	bool free = true;
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
 		switch (get_blocktype(arena, cell)) {
 			case BLOCK_EXTENT:
-#if DEBUG_ZERO_ON_SWEEP
-				if (zero) {
-					memset(&arena->cells[cell], 0, sizeof(cell_t));
-				}
-#endif
 				break;
 			case BLOCK_FREE:
-				if (coalesce) {
+				if (last_free_cell != 0) {
+					// Coalesce
 					set_blocktype(arena, cell, BLOCK_EXTENT);
 				} else {
 					last_free_cell = cell;
 				}
-				coalesce = true;
-#if DEBUG_ZERO_ON_SWEEP
-				zero = true;
-				memset(&arena->cells[cell], 0, sizeof(cell_t));
-#endif
+				// ==> last_free_cell != 0
 				break;
 			case BLOCK_WHITE:
-				if (coalesce) {
+				if (last_free_cell != 0) {
+					// Coalesce
 					set_blocktype(arena, cell, BLOCK_EXTENT);
 				} else {
 					set_blocktype(arena, cell, BLOCK_FREE);
 					last_free_cell = cell;
 				}
-				coalesce = true;
-				add_to_free_list = true;
-#if DEBUG_ZERO_ON_SWEEP
-				zero = true;
-				memset(&arena->cells[cell], 0, sizeof(cell_t));
-#endif
+				// ==> last_free_cell != 0
+				register_free_block = true;
 				break;
 			case BLOCK_BLACK:
 				set_blocktype(arena, cell, BLOCK_WHITE);
-				if (add_to_free_list) {
-					qcgc_fit_allocator_add(arena->cells + last_free_cell,
+				if (last_free_cell != 0) {
+					if (register_free_block) {
+						qcgc_fit_allocator_add(arena->cells + last_free_cell,
+								cell - last_free_cell);
+#if DEBUG_ZERO_ON_SWEEP
+						memset(arena->cells + last_free_cell, 0,
+								sizeof(cell_t) * (cell - last_free_cell));
+#endif
+					}
+					qcgc_state.free_cells += cell - last_free_cell;
+					qcgc_state.largest_free_block = MAX(
+							qcgc_state.largest_free_block,
 							cell - last_free_cell);
+					last_free_cell = 0;
 				}
+				register_free_block = false;
 				free = false;
-				coalesce = false;
-				add_to_free_list = false;
-#if DEBUG_ZERO_ON_SWEEP
-				zero = false;
-#endif
+				// ==> last_free_cell == 0
 				break;
 		}
 	}
-	if (add_to_free_list && !free) {
-		qcgc_fit_allocator_add(arena->cells + last_free_cell,
-							QCGC_ARENA_CELLS_COUNT - last_free_cell);
+	if (last_free_cell != 0 && !free) {
+		if (register_free_block) {
+			qcgc_fit_allocator_add(arena->cells + last_free_cell,
+					QCGC_ARENA_CELLS_COUNT - last_free_cell);
+#if DEBUG_ZERO_ON_SWEEP
+			memset(arena->cells + last_free_cell, 0,
+					sizeof(cell_t) * (QCGC_ARENA_CELLS_COUNT - last_free_cell));
+#endif
+		}
+		qcgc_state.free_cells += QCGC_ARENA_CELLS_COUNT - last_free_cell;
+		qcgc_state.largest_free_block = MAX(
+				qcgc_state.largest_free_block,
+				QCGC_ARENA_CELLS_COUNT - last_free_cell);
+		last_free_cell = 0;
 	}
 #if CHECKED
 	assert(qcgc_arena_is_coalesced(arena));
@@ -284,6 +324,7 @@
 	return true;
 }
 
+
 bool qcgc_arena_is_coalesced(arena_t *arena) {
 #if CHECKED
 	assert(arena != NULL);
diff --git a/rpython/translator/c/src/qcgc/arena.h b/rpython/translator/c/src/qcgc/arena.h
--- a/rpython/translator/c/src/qcgc/arena.h
+++ b/rpython/translator/c/src/qcgc/arena.h
@@ -143,6 +143,15 @@
  */
 bool qcgc_arena_sweep(arena_t *arena);
 
+/**
+ * Sweep given arena, but only reset black to white, no white to free
+ *
+ * @param	arena	Arena
+ * @return	Whether arena is empty after sweeping, always false
+ */
+bool qcgc_arena_pseudo_sweep(arena_t *arena);
+
+
 /*******************************************************************************
  * Debug functions                                                             *
  ******************************************************************************/
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -1,14 +1,14 @@
 #pragma once
 
 #define CHECKED 0							// Enable runtime sanity checks
-#define DEBUG_ZERO_ON_SWEEP 1				// Zero memory on sweep (debug only)
+#define DEBUG_ZERO_ON_SWEEP 0				// Zero memory on sweep (debug only)
 
 #define QCGC_INIT_ZERO 1					// Init new objects with zero bytes
 
 /**
  * Event logger
  */
-#define EVENT_LOG 0							// Enable event log
+#define EVENT_LOG 1							// Enable event log
 #define LOGFILE "./qcgc_events.log"			// Default logfile
 #define LOG_ALLOCATION 0					// Enable allocation log (warning:
 											// significant performance impact)
diff --git a/rpython/translator/c/src/qcgc/event_logger.h b/rpython/translator/c/src/qcgc/event_logger.h
--- a/rpython/translator/c/src/qcgc/event_logger.h
+++ b/rpython/translator/c/src/qcgc/event_logger.h
@@ -21,7 +21,6 @@
 	EVENT_NEW_ARENA,
 
 	EVENT_MARK_START,
-	EVENT_INCMARK_START,
 	EVENT_MARK_DONE,
 };
 
diff --git a/rpython/translator/c/src/qcgc/gc_state.h b/rpython/translator/c/src/qcgc/gc_state.h
--- a/rpython/translator/c/src/qcgc/gc_state.h
+++ b/rpython/translator/c/src/qcgc/gc_state.h
@@ -33,4 +33,9 @@
 	gc_phase_t phase;
 	size_t bytes_since_collection;
 	size_t bytes_since_incmark;
+	size_t free_cells;			// Overall amount of free cells without huge
+								// blocks and free areans. Valid right after sweep
+	size_t largest_free_block;	// Size of the largest free block.
+								// (Free arenas don't count as free blocks)
+								// Valid right after sweep
 } qcgc_state;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -38,6 +38,8 @@
 	qcgc_state.phase = GC_PAUSE;
 	qcgc_state.bytes_since_collection = 0;
 	qcgc_state.bytes_since_incmark = 0;
+	qcgc_state.free_cells = 0;
+	qcgc_state.largest_free_block = 0;
 	qcgc_allocator_initialize();
 	qcgc_hbtable_initialize();
 	qcgc_event_logger_initialize();
@@ -145,7 +147,7 @@
 
 	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
 		// Use bump / fit allocator
-		if (true) { // FIXME: Implement reasonable switch
+		if (qcgc_allocator_state.use_bump_allocator) {
 			result = qcgc_bump_allocate(size);
 		} else {
 			result = qcgc_fit_allocate(size);
@@ -203,8 +205,17 @@
 	if (qcgc_state.phase == GC_COLLECT) {
 		return;	// Fast exit when there is nothing to mark
 	}
-	// FIXME: Log some more information
-	qcgc_event_logger_log(EVENT_MARK_START, 0, NULL);
+
+	{
+		struct log_info_s {
+			bool incremental;
+			size_t gray_stack_size;
+		};
+		struct log_info_s log_info = {incremental, qcgc_state.gray_stack_size};
+		qcgc_event_logger_log(EVENT_MARK_START, sizeof(struct log_info_s),
+				(uint8_t *) &log_info);
+	}
+
 	qcgc_state.bytes_since_incmark = 0;
 
 	if (qcgc_state.phase == GC_PAUSE) {
@@ -268,8 +279,15 @@
 		qcgc_state.phase = GC_COLLECT;
 	}
 
-	// FIXME: Log some more information
-	qcgc_event_logger_log(EVENT_MARK_DONE, 0, NULL);
+	{
+		struct log_info_s {
+			bool incremental;
+			size_t gray_stack_size;
+		};
+		struct log_info_s log_info = {incremental, qcgc_state.gray_stack_size};
+		qcgc_event_logger_log(EVENT_MARK_DONE, sizeof(struct log_info_s),
+				(uint8_t *) &log_info);
+	}
 #if CHECKED
 	assert(incremental || (qcgc_state.phase = GC_COLLECT));
 	assert(qcgc_state.phase != GC_PAUSE);
@@ -339,13 +357,17 @@
 #if CHECKED
 	assert(qcgc_state.phase == GC_COLLECT);
 #endif
-	unsigned long arena_count;
-	arena_count = qcgc_allocator_state.arenas->count;
-	qcgc_event_logger_log(EVENT_SWEEP_START, sizeof(arena_count),
-			(uint8_t *) &arena_count);
+	{
+		unsigned long arena_count;
+		arena_count = qcgc_allocator_state.arenas->count;
+		qcgc_event_logger_log(EVENT_SWEEP_START, sizeof(arena_count),
+				(uint8_t *) &arena_count);
+	}
 
 	qcgc_hbtable_sweep();
 	size_t i = 0;
+	qcgc_state.free_cells = 0;
+	qcgc_state.largest_free_block = 0;
 	while (i < qcgc_allocator_state.arenas->count) {
 		arena_t *arena = qcgc_allocator_state.arenas->items[i];
 		// The arena that contains the bump pointer is autmatically skipped
@@ -355,7 +377,6 @@
 					qcgc_allocator_state.arenas, i);
 			qcgc_allocator_state.free_arenas = qcgc_arena_bag_add(
 					qcgc_allocator_state.free_arenas, arena);
-
 			// NO i++
 		} else {
 			// Not free
@@ -364,8 +385,26 @@
 	}
 	qcgc_state.phase = GC_PAUSE;
 
-	qcgc_event_logger_log(EVENT_SWEEP_DONE, 0, NULL);
+	// Determine whether fragmentation is too high
+	// Fragmenation = 1 - (largest block / total free space)
+	// Use bump allocator when fragmentation < 50%
+	qcgc_allocator_state.use_bump_allocator = qcgc_state.free_cells <
+		2 * qcgc_state.largest_free_block;
+
 	update_weakrefs();
+
+	{
+		struct log_info_s {
+			size_t free_cells;
+			size_t largest_free_block;
+		};
+		struct log_info_s log_info = {
+			qcgc_state.free_cells,
+			qcgc_state.largest_free_block
+		};
+		qcgc_event_logger_log(EVENT_SWEEP_DONE, sizeof(struct log_info_s),
+				(uint8_t *) &log_info);
+	}
 }
 
 void qcgc_collect(void) {

From pypy.commits at gmail.com  Fri Sep  9 13:28:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 09 Sep 2016 10:28:37 -0700 (PDT)
Subject: [pypy-commit] pypy release-5.x: Probable fix for issue #2383: have
 'list(S())' call 'S.__getitem__' if S
Message-ID: <57d2f145.442d1c0a.36d62.82d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-5.x
Changeset: r86978:eacdeb58a9ec
Date: 2016-08-29 20:32 +0200
http://bitbucket.org/pypy/pypy/changeset/eacdeb58a9ec/

Log:	Probable fix for issue #2383: have 'list(S())' call 'S.__getitem__'
	if S is a subclass of str with a custom __getitem__. (grafted from
	990f5b2322e124bbbfd7d9ab56d44a77f0085a8a)

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -58,6 +58,20 @@
     return w_iter
 tuple_iter._annspecialcase_ = 'specialize:memo'
 
+def str_getitem(space):
+    "Utility that returns the app-level descriptor str.__getitem__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_str,
+                                               '__getitem__')
+    return w_iter
+str_getitem._annspecialcase_ = 'specialize:memo'
+
+def unicode_getitem(space):
+    "Utility that returns the app-level descriptor unicode.__getitem__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
+                                               '__getitem__')
+    return w_iter
+unicode_getitem._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:
         raise oefmt(space.w_AttributeError,
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -445,7 +445,7 @@
             return w_obj.listview_bytes()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_bytes()
-        if isinstance(w_obj, W_BytesObject) and self._uses_no_iter(w_obj):
+        if isinstance(w_obj, W_BytesObject) and self._str_uses_no_iter(w_obj):
             return w_obj.listview_bytes()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_bytes()
@@ -460,7 +460,7 @@
             return w_obj.listview_unicode()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_unicode()
-        if isinstance(w_obj, W_UnicodeObject) and self._uses_no_iter(w_obj):
+        if isinstance(w_obj, W_UnicodeObject) and self._uni_uses_no_iter(w_obj):
             return w_obj.listview_unicode()
         if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_unicode()
@@ -504,8 +504,15 @@
         from pypy.objspace.descroperation import tuple_iter
         return self.lookup(w_obj, '__iter__') is tuple_iter(self)
 
-    def _uses_no_iter(self, w_obj):
-        return self.lookup(w_obj, '__iter__') is None
+    def _str_uses_no_iter(self, w_obj):
+        from pypy.objspace.descroperation import str_getitem
+        return (self.lookup(w_obj, '__iter__') is None and
+                self.lookup(w_obj, '__getitem__') is str_getitem(self))
+
+    def _uni_uses_no_iter(self, w_obj):
+        from pypy.objspace.descroperation import unicode_getitem
+        return (self.lookup(w_obj, '__iter__') is None and
+                self.lookup(w_obj, '__getitem__') is unicode_getitem(self))
 
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -432,7 +432,7 @@
 
 
 class AppTestListObject(object):
-    spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
+    #spaceconfig = {"objspace.std.withliststrategies": True}  # it's the default
 
     def setup_class(cls):
         import platform
@@ -1518,6 +1518,16 @@
             def __iter__(self):
                 yield "ok"
         assert list(U(u"don't see me")) == ["ok"]
+        #
+        class S(str):
+            def __getitem__(self, index):
+                return str.__getitem__(self, index).upper()
+        assert list(S("abc")) == list("ABC")
+        #
+        class U(unicode):
+            def __getitem__(self, index):
+                return unicode.__getitem__(self, index).upper()
+        assert list(U(u"abc")) == list(u"ABC")
 
     def test_extend_from_nonempty_list_with_subclasses(self):
         l = ["hi!"]
@@ -1543,6 +1553,20 @@
         l.extend(U(u"don't see me"))
         #
         assert l == ["hi!", "okT", "okL", "okL", "okS", "okU"]
+        #
+        class S(str):
+            def __getitem__(self, index):
+                return str.__getitem__(self, index).upper()
+        l = []
+        l.extend(S("abc"))
+        assert l == list("ABC")
+        #
+        class U(unicode):
+            def __getitem__(self, index):
+                return unicode.__getitem__(self, index).upper()
+        l = []
+        l.extend(U(u"abc"))
+        assert l == list(u"ABC")
 
     def test_no_len_on_range_iter(self):
         iterable = range(10)

From pypy.commits at gmail.com  Fri Sep  9 15:26:44 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 09 Sep 2016 12:26:44 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Simplify import_module(): kill
 load_it param and remove duplication with reimport_module()
Message-ID: <57d30cf4.010c1c0a.8043.abfb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r86979:12888babfe97
Date: 2016-09-09 20:25 +0100
http://bitbucket.org/pypy/pypy/changeset/12888babfe97/

Log:	Simplify import_module(): kill load_it param and remove duplication
	with reimport_module()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -53,19 +53,19 @@
         self.extra_libs = extra_libs
         self.ext = ext
 
-    def compile_extension_module(self, modname, include_dirs=[],
+    def compile_extension_module(self, name, include_dirs=[],
             source_files=None, source_strings=None):
         """
         Build an extension module and return the filename of the resulting
         native code file.
 
-        modname is the name of the module, possibly including dots if it is a
+        name is the name of the module, possibly including dots if it is a
         module inside a package.
 
         Any extra keyword arguments are passed on to ExternalCompilationInfo to
         build the module (so specify your source with one of those).
         """
-        modname = modname.split('.')[-1]
+        modname = name.split('.')[-1]
         dirname = (udir/uniquemodulename('module')).ensure(dir=1)
         if source_strings:
             assert not source_files
@@ -354,9 +354,8 @@
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
-                     load_it=bool, filename='str_or_None',
-                     PY_SSIZE_T_CLEAN=bool)
-        def import_module(space, name, init=None, body='', load_it=True,
+                     filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
+        def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
                           PY_SSIZE_T_CLEAN=False):
             """
@@ -405,30 +404,21 @@
                 kwds = dict(source_files=[filename])
             mod = self.sys_info.compile_extension_module(
                 name, include_dirs=include_dirs, **kwds)
+            w_result = load_module(space, mod, name)
+            if not self.runappdirect:
+                self.record_imported_module(name)
+            return w_result
 
-            if not load_it:
-                return space.wrap(mod)
+
+        @gateway.unwrap_spec(mod=str, name=str)
+        def load_module(space, mod, name):
             if self.runappdirect:
                 import imp
                 return imp.load_dynamic(name, mod)
             else:
                 api.load_extension_module(space, mod, name)
-                self.record_imported_module(name)
                 return space.getitem(
-                    space.sys.get('modules'),
-                    space.wrap(name))
-
-
-        @gateway.unwrap_spec(mod=str, name=str)
-        def reimport_module(space, mod, name):
-            if self.runappdirect:
-                import imp
-                return imp.load_dynamic(name, mod)
-            else:
-                api.load_extension_module(space, mod, name)
-            return space.getitem(
-                space.sys.get('modules'),
-                space.wrap(name))
+                    space.sys.get('modules'), space.wrap(name))
 
         @gateway.unwrap_spec(modname=str, prologue=str,
                              more_init=str, PY_SSIZE_T_CLEAN=bool)
@@ -470,7 +460,7 @@
             self.sys_info = get_cpyext_info(self.space)
             self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
-        self.w_reimport_module = wrap(interp2app(reimport_module))
+        self.w_load_module = wrap(interp2app(load_module))
         self.w_import_extension = wrap(interp2app(import_extension))
         self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/')
         self.w_debug_collect = wrap(interp2app(debug_collect))
@@ -976,7 +966,7 @@
             f.write('not again!\n')
             f.close()
             m1 = sys.modules['foo']
-            m2 = self.reimport_module(m1.__file__, name='foo')
+            m2 = self.load_module(m1.__file__, name='foo')
             assert m1 is m2
             assert m1 is sys.modules['foo']
 
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -40,7 +40,8 @@
 class AppTestImportLogic(AppTestCpythonExtensionBase):
     def test_import_logic(self):
         import sys, os
-        path = self.import_module(name='test_import_module', load_it=False)
+        path = self.compile_module('test_import_module',
+            source_files=[os.path.join(self.here, 'test_import_module.c')])
         sys.path.append(os.path.dirname(path))
         import test_import_module
         assert test_import_module.TEST is None

From pypy.commits at gmail.com  Sat Sep 10 03:36:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 00:36:27 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Print full precision doubles
Message-ID: <57d3b7fb.81a2c20a.5e378.564e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86980:d7724c0a5700
Date: 2016-09-10 09:35 +0200
http://bitbucket.org/pypy/pypy/changeset/d7724c0a5700/

Log:	Print full precision doubles

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1768,7 +1768,7 @@
     char buffer[128], *p;
     RPyString *result;
     int size;
-    size = snprintf(buffer, sizeof(buffer) - 3, "%g", d);
+    size = snprintf(buffer, sizeof(buffer) - 3, "%.17g", d);
     if (size < 0)
         size = 0;
     for (p = buffer; '0' <= *p && *p <= '9'; p++) {

From pypy.commits at gmail.com  Sat Sep 10 04:38:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 01:38:26 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Added tag RevDB-pypy2.7-v5.4.1
 for changeset d7724c0a5700
Message-ID: <57d3c682.02d31c0a.ec84b.614a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86981:b6111ee37219
Date: 2016-09-10 10:37 +0200
http://bitbucket.org/pypy/pypy/changeset/b6111ee37219/

Log:	Added tag RevDB-pypy2.7-v5.4.1 for changeset d7724c0a5700

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -34,3 +34,5 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
 4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1

From pypy.commits at gmail.com  Sat Sep 10 08:57:59 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sat, 10 Sep 2016 05:57:59 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase
Message-ID: <57d40357.c398c20a.dfeea.c1ab@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86982:79998830344e
Date: 2016-09-10 14:57 +0200
http://bitbucket.org/pypy/pypy/changeset/79998830344e/

Log:	Update qcgc codebase

diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
--- a/rpython/translator/c/src/qcgc/allocator.c
+++ b/rpython/translator/c/src/qcgc/allocator.c
@@ -23,6 +23,7 @@
 QCGC_STATIC cell_t *fit_allocator_large_first_fit(size_t index, size_t cells);
 
 QCGC_STATIC bool valid_block(cell_t *ptr, size_t cells);
+QCGC_STATIC void free_list_consistency_check(void);
 
 void qcgc_allocator_initialize(void) {
 	qcgc_allocator_state.arenas =
@@ -77,8 +78,13 @@
 	if (cells > 0) {
 		assert((((object_t *)ptr)->flags & QCGC_PREBUILT_OBJECT) == 0);
 		assert((cell_t *) qcgc_arena_addr(ptr) != ptr);
-		assert(qcgc_arena_get_blocktype(ptr) == BLOCK_FREE ||
-				qcgc_arena_get_blocktype(ptr) == BLOCK_EXTENT);
+		assert(qcgc_arena_get_blocktype(ptr) == BLOCK_FREE);
+		if (qcgc_arena_addr(ptr) == qcgc_arena_addr(ptr + cells)) {
+			assert(qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT);
+		}
+		for (size_t i = 1; i < cells; i++) {
+			assert(qcgc_arena_get_blocktype(ptr + i) == BLOCK_EXTENT);
+		}
 	}
 #endif
 	if (cells > 0) {
@@ -208,6 +214,9 @@
 }
 
 object_t *qcgc_fit_allocate(size_t bytes) {
+#if CHECKED
+	free_list_consistency_check();
+#endif
 	size_t cells = bytes_to_cells(bytes);
 	cell_t *mem;
 
@@ -223,7 +232,6 @@
 		return NULL;
 	}
 
-	qcgc_arena_mark_allocated(mem, cells);
 	object_t *result = (object_t *) mem;
 
 #if QCGC_INIT_ZERO
@@ -250,6 +258,16 @@
 	return result;
 }
 
+void qcgc_fit_allocator_empty_lists(void) {
+	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
+		qcgc_allocator_state.fit_state.small_free_list[i]->count = 0;
+	}
+
+	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
+		qcgc_allocator_state.fit_state.large_free_list[i]->count = 0;
+	}
+}
+
 QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells) {
 #if CHECKED
 	assert(small_index_to_cells(index) >= cells);
@@ -272,16 +290,11 @@
 			// Check whether block is still valid
 			if (valid_block(result, list_cell_size)) {
 				// The next call might invalidate free_list, reload!
+				qcgc_arena_mark_allocated(result, cells);
 				qcgc_fit_allocator_add(result + cells, list_cell_size - cells);
-				break;
-			} else {
-				result = NULL;
+				return result;
 			}
 		}
-
-		if (result != NULL) {
-			return result;
-		}
 	}
 	return fit_allocator_large_first_fit(0, cells);
 }
@@ -332,6 +345,7 @@
 		qcgc_allocator_state.fit_state.large_free_list[index] =
 			qcgc_exp_free_list_remove_index(qcgc_allocator_state.fit_state.
 					large_free_list[index], best_fit_index);
+		qcgc_arena_mark_allocated(result, cells);
 		qcgc_fit_allocator_add(result + cells, best_fit_cells - cells);
 	} else {
 		// No best fit, go for first fit
@@ -344,7 +358,6 @@
 #if CHECKED
 	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP) >= cells);
 #endif
-	cell_t *result = NULL;
 	for ( ; index < QCGC_LARGE_FREE_LISTS; index++) {
 		while(qcgc_allocator_state.fit_state.large_free_list[index]->count
 				> 0) {
@@ -360,14 +373,11 @@
 
 			// Check whether block is still valid
 			if (valid_block(item.ptr, item.size)) {
+				qcgc_arena_mark_allocated(item.ptr, cells);
 				qcgc_fit_allocator_add(item.ptr + cells, item.size - cells);
-				result = item.ptr;
-				break;
+				return item.ptr;
 			}
 		}
-		if (result != NULL) {
-			return result;
-		}
 	}
 	return NULL;
 }
@@ -415,3 +425,31 @@
 				((qcgc_arena_addr(ptr + cells)) == (arena_t *) (ptr + cells)) ||
 				qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT));
 }
+
+QCGC_STATIC void free_list_consistency_check(void) {
+	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
+		linear_free_list_t *free_list =
+			qcgc_allocator_state.fit_state.small_free_list[i];
+		for (size_t j = 0; j < free_list->count; j++) {
+			cell_t *item = free_list->items[j];
+			if  (qcgc_arena_get_blocktype(item) == BLOCK_FREE) {
+				for (size_t s = 1; s < small_index_to_cells(i); s++) {
+					assert(qcgc_arena_get_blocktype(item + s) == BLOCK_EXTENT);
+				}
+			}
+		}
+	}
+
+	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
+		exp_free_list_t *free_list =
+			qcgc_allocator_state.fit_state.large_free_list[i];
+		for (size_t j = 0; j < free_list->count; j++) {
+			struct exp_free_list_item_s item = free_list->items[j];
+			if  (qcgc_arena_get_blocktype(item.ptr) == BLOCK_FREE) {
+				for (size_t s = 1; s < item.size; s++) {
+					assert(qcgc_arena_get_blocktype(item.ptr + s) == BLOCK_EXTENT);
+				}
+			}
+		}
+	}
+}
diff --git a/rpython/translator/c/src/qcgc/allocator.h b/rpython/translator/c/src/qcgc/allocator.h
--- a/rpython/translator/c/src/qcgc/allocator.h
+++ b/rpython/translator/c/src/qcgc/allocator.h
@@ -89,6 +89,11 @@
  */
 object_t *qcgc_large_allocate(size_t bytes);
 
+/**
+ * Empty all free lists (used before sweep)
+ */
+void qcgc_fit_allocator_empty_lists(void);
+
 
 /**
  * Add memory to free lists
diff --git a/rpython/translator/c/src/qcgc/arena.c b/rpython/translator/c/src/qcgc/arena.c
--- a/rpython/translator/c/src/qcgc/arena.c
+++ b/rpython/translator/c/src/qcgc/arena.c
@@ -185,36 +185,20 @@
 	assert(qcgc_arena_is_coalesced(arena));
 	assert(qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) == arena);
 #endif
-	// XXX: Maybe ignore free cell / largest block counting here?
-	size_t last_free_cell = 0;
+	// Ignore free cell / largest block counting here, as blocks are not
+	// registerd in free lists as well
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
 		switch (get_blocktype(arena, cell)) {
-			case BLOCK_FREE:
-				last_free_cell = cell;
-			case BLOCK_EXTENT: // Fall through
-				break;
 			case BLOCK_BLACK:
 				set_blocktype(arena, cell, BLOCK_WHITE);
+			case BLOCK_FREE: // Fall through
+			case BLOCK_EXTENT: // Fall through
 			case BLOCK_WHITE: // Fall through
-				if (last_free_cell != 0) {
-					qcgc_state.free_cells += cell - last_free_cell;
-					qcgc_state.largest_free_block = MAX(
-							qcgc_state.largest_free_block,
-							cell - last_free_cell);
-					last_free_cell = 0;
-				}
 				break;
 		}
 	}
-	if (last_free_cell != 0) {
-		qcgc_state.free_cells += QCGC_ARENA_CELLS_COUNT - last_free_cell;
-		qcgc_state.largest_free_block = MAX(
-				qcgc_state.largest_free_block,
-				QCGC_ARENA_CELLS_COUNT - last_free_cell);
-		last_free_cell = 0;
-	}
 #if CHECKED
 	assert(qcgc_arena_is_coalesced(arena));
 #endif
@@ -232,7 +216,6 @@
 	}
 
 	size_t last_free_cell = 0;
-	bool register_free_block = false;
 	bool free = true;
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
@@ -258,40 +241,34 @@
 					last_free_cell = cell;
 				}
 				// ==> last_free_cell != 0
-				register_free_block = true;
 				break;
 			case BLOCK_BLACK:
 				set_blocktype(arena, cell, BLOCK_WHITE);
 				if (last_free_cell != 0) {
-					if (register_free_block) {
-						qcgc_fit_allocator_add(arena->cells + last_free_cell,
-								cell - last_free_cell);
+					qcgc_fit_allocator_add(arena->cells + last_free_cell,
+							cell - last_free_cell);
 #if DEBUG_ZERO_ON_SWEEP
-						memset(arena->cells + last_free_cell, 0,
-								sizeof(cell_t) * (cell - last_free_cell));
+					memset(arena->cells + last_free_cell, 0,
+							sizeof(cell_t) * (cell - last_free_cell));
 #endif
-					}
 					qcgc_state.free_cells += cell - last_free_cell;
 					qcgc_state.largest_free_block = MAX(
 							qcgc_state.largest_free_block,
 							cell - last_free_cell);
 					last_free_cell = 0;
 				}
-				register_free_block = false;
 				free = false;
 				// ==> last_free_cell == 0
 				break;
 		}
 	}
 	if (last_free_cell != 0 && !free) {
-		if (register_free_block) {
-			qcgc_fit_allocator_add(arena->cells + last_free_cell,
-					QCGC_ARENA_CELLS_COUNT - last_free_cell);
+		qcgc_fit_allocator_add(arena->cells + last_free_cell,
+				QCGC_ARENA_CELLS_COUNT - last_free_cell);
 #if DEBUG_ZERO_ON_SWEEP
-			memset(arena->cells + last_free_cell, 0,
-					sizeof(cell_t) * (QCGC_ARENA_CELLS_COUNT - last_free_cell));
+		memset(arena->cells + last_free_cell, 0,
+				sizeof(cell_t) * (QCGC_ARENA_CELLS_COUNT - last_free_cell));
 #endif
-		}
 		qcgc_state.free_cells += QCGC_ARENA_CELLS_COUNT - last_free_cell;
 		qcgc_state.largest_free_block = MAX(
 				qcgc_state.largest_free_block,
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -147,7 +147,8 @@
 
 	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
 		// Use bump / fit allocator
-		if (qcgc_allocator_state.use_bump_allocator) {
+		//if (qcgc_allocator_state.use_bump_allocator) {
+		if (false) {
 			result = qcgc_bump_allocate(size);
 		} else {
 			result = qcgc_fit_allocate(size);
@@ -368,6 +369,8 @@
 	size_t i = 0;
 	qcgc_state.free_cells = 0;
 	qcgc_state.largest_free_block = 0;
+
+	qcgc_fit_allocator_empty_lists();
 	while (i < qcgc_allocator_state.arenas->count) {
 		arena_t *arena = qcgc_allocator_state.arenas->items[i];
 		// The arena that contains the bump pointer is autmatically skipped

From pypy.commits at gmail.com  Sat Sep 10 09:36:23 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Sat, 10 Sep 2016 06:36:23 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Add teardown method to codegen
Message-ID: <57d40c57.217fc20a.479c9.0f83@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r86983:30fd102afbe9
Date: 2016-09-10 15:35 +0200
http://bitbucket.org/pypy/pypy/changeset/30fd102afbe9/

Log:	Add teardown method to codegen

diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -40,6 +40,9 @@
     def gc_startup_code(self):
         return []
 
+    def gc_teardown_code(self):
+        return []
+
     def struct_setup(self, structdefnode, rtti):
         return None
 
@@ -477,6 +480,12 @@
         for i in s:
             yield i
 
+    def gc_teardown_code(self):
+        yield 'qcgc_destroy();'
+        s = list(super(QcgcFrameworkGcPolicy, self).gc_teardown_code())
+        for i in s:
+            yield i
+
 name_to_gcpolicy = {
     'boehm': BoehmGcPolicy,
     'ref': RefcountingGcPolicy,
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -823,6 +823,15 @@
 
     print >> f, '}'
 
+def gen_teardowncode(f, database):
+    # generate tear-down code and put it into a function (empty except for qcgc)
+    print >> f, 'void RPython_TeardownCode(void) {'
+
+    for line in database.gcpolicy.gc_teardown_code():
+        print >> f, "\t" + line
+
+    print >> f, '}'
+
 def commondefs(defines):
     from rpython.rlib.rarithmetic import LONG_BIT, LONGLONG_BIT
     defines['PYPY_LONG_BIT'] = LONG_BIT
@@ -887,6 +896,7 @@
     headers_to_precompile.insert(0, incfilename)
 
     gen_startupcode(f, database)
+    gen_teardowncode(f, database)
     f.close()
 
     if 'PYPY_INSTRUMENT' in defines:
diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c
--- a/rpython/translator/c/src/entrypoint.c
+++ b/rpython/translator/c/src/entrypoint.c
@@ -109,6 +109,7 @@
     RPyGilRelease();
 #endif
 
+	RPython_TeardownCode();
     return exitcode;
 
  memory_out:
diff --git a/rpython/translator/c/src/entrypoint.h b/rpython/translator/c/src/entrypoint.h
--- a/rpython/translator/c/src/entrypoint.h
+++ b/rpython/translator/c/src/entrypoint.h
@@ -9,5 +9,6 @@
 #endif
 
 RPY_EXTERN void RPython_StartupCode(void);
+RPY_EXTERN void RPython_TeardownCode(void);
 RPY_EXPORTED int PYPY_MAIN_FUNCTION(int argc, char *argv[]);
 #endif  /* PYPY_STANDALONE */

From pypy.commits at gmail.com  Sat Sep 10 13:12:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 10:12:29 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: emit the byte "release gil"
 before actually releasing the GIL
Message-ID: <57d43efd.85b81c0a.422a8.dbff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86985:a82acaa0d692
Date: 2016-09-10 19:03 +0200
http://bitbucket.org/pypy/pypy/changeset/a82acaa0d692/

Log:	emit the byte "release gil" before actually releasing the GIL

diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py
--- a/rpython/translator/revdb/gencsupp.py
+++ b/rpython/translator/revdb/gencsupp.py
@@ -92,10 +92,9 @@
         # reading the 0xFD or 0xFE, we switch to a different thread if needed
         # (actually implemented with stacklets).
         if call_code == 'RPyGilAcquire();':
-            byte = '0xFD'
+            return 'RPY_REVDB_CALL_GIL_ACQUIRE();'
         else:
-            byte = '0xFE'
-        return 'RPY_REVDB_CALL_GIL(%s, %s);' % (call_code, byte)
+            return 'RPY_REVDB_CALL_GIL_RELEASE();'
     #
     tp = funcgen.lltypename(v_result)
     if tp == 'void @':
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -1730,10 +1730,10 @@
 }
 
 RPY_EXTERN
-void rpy_reverse_db_bad_acquire_gil(void)
+void rpy_reverse_db_bad_acquire_gil(const char *name)
 {
     fprintf(stderr, "out of sync: unexpected byte in log "
-                    " (at acquire_gil or release_gil)\n");
+                    " (at %s_gil)\n", name);
     exit(1);
 }
 
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -151,23 +151,32 @@
             rpy_reverse_db_invoke_callback(_re);                        \
     }
 
-#define RPY_REVDB_CALL_GIL(call_code, byte)                             \
+#define RPY_REVDB_CALL_GIL_ACQUIRE()                                    \
     if (!RPY_RDB_REPLAY) {                                              \
-        call_code                                                       \
+        RPyGilAcquire();                                                \
         _RPY_REVDB_LOCK();                                              \
-        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, byte)                \
+        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, 0xFD)                \
         _RPY_REVDB_UNLOCK();                                            \
     }                                                                   \
     else {                                                              \
         unsigned char _re;                                              \
         _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re)                   \
-        if (_re != byte)                                                \
-            rpy_reverse_db_bad_acquire_gil();                           \
+        if (_re != 0xFD)                                                \
+            rpy_reverse_db_bad_acquire_gil("acquire");                  \
     }
 
-#define RPY_REVDB_CALL_GILCTRL(call_code)                               \
+#define RPY_REVDB_CALL_GIL_RELEASE()                                    \
     if (!RPY_RDB_REPLAY) {                                              \
-        call_code                                                       \
+        _RPY_REVDB_LOCK();                                              \
+        _RPY_REVDB_EMIT_RECORD_L(unsigned char _e, 0xFE)                \
+        _RPY_REVDB_UNLOCK();                                            \
+        RPyGilRelease();                                                \
+    }                                                                   \
+    else {                                                              \
+        unsigned char _re;                                              \
+        _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re)                   \
+        if (_re != 0xFE)                                                \
+            rpy_reverse_db_bad_acquire_gil("release");                  \
     }
 
 #define RPY_REVDB_CALLBACKLOC(locnum)                                   \
@@ -284,7 +293,7 @@
 RPY_EXTERN void rpy_reverse_db_invoke_callback(unsigned char);
 RPY_EXTERN void rpy_reverse_db_callback_loc(int);
 RPY_EXTERN void rpy_reverse_db_lock_acquire(bool_t lock_contention);
-RPY_EXTERN void rpy_reverse_db_bad_acquire_gil(void);
+RPY_EXTERN void rpy_reverse_db_bad_acquire_gil(const char *name);
 RPY_EXTERN void rpy_reverse_db_set_thread_breakpoint(int64_t tnum);
 RPY_EXTERN double rpy_reverse_db_strtod(RPyString *s);
 RPY_EXTERN RPyString *rpy_reverse_db_dtoa(double d);

From pypy.commits at gmail.com  Sat Sep 10 13:12:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 10:12:27 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Fix for Linux 32-bit
Message-ID: <57d43efb.03121c0a.b279a.0d96@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r86984:9d7562248cdf
Date: 2016-09-10 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/9d7562248cdf/

Log:	Fix for Linux 32-bit

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -848,7 +848,9 @@
         ssize_t rsize = read(rpy_rev_fileno, buf + result, count_max - result);
         if (rsize <= 0) {
             if (rsize == 0)
-                fprintf(stderr, "RevDB file appears truncated\n");
+                fprintf(stderr, "RevDB file appears truncated (cannot read "
+                                "more after offset %lld)\n",
+                        (long long)lseek(rpy_rev_fileno, 0, SEEK_CUR));
             else
                 fprintf(stderr, "RevDB file read error: %m\n");
             exit(1);
@@ -997,11 +999,11 @@
 
     count = lseek(rpy_rev_fileno, 0, SEEK_CUR);
     if (count < 0 ||
-            lseek(rpy_rev_fileno, -sizeof(uint64_t), SEEK_END) < 0 ||
-            read(rpy_rev_fileno, &total_stop_points,
-                 sizeof(uint64_t)) != sizeof(uint64_t) ||
-            lseek(rpy_rev_fileno, count, SEEK_SET) != count) {
-        fprintf(stderr, "%s: %m\n", rpy_rev_filename);
+        lseek(rpy_rev_fileno, -(off_t)sizeof(uint64_t), SEEK_END) < 0 ||
+        (read_all(&total_stop_points, sizeof(uint64_t)),
+         lseek(rpy_rev_fileno, count, SEEK_SET)) != count) {
+        fprintf(stderr, "%s: invalid total_stop_points (%m)\n",
+                rpy_rev_filename);
         exit(1);
     }
 

From pypy.commits at gmail.com  Sat Sep 10 15:26:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 12:26:59 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: PyPy on FreeBSD is maintained
 externally on FreshPorts. Thanks DragonSA!
Message-ID: <57d45e83.058a1c0a.d2b9e.fae4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r790:1d67e372fd65
Date: 2016-09-10 21:26 +0200
http://bitbucket.org/pypy/pypy.org/changeset/1d67e372fd65/

Log:	PyPy on FreeBSD is maintained externally on FreshPorts. Thanks
	DragonSA!

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -124,7 +124,7 @@
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
-<li>FreeBSD 9.2 x86 64 bit <em>(hopefully availabe soon)</em> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li>FreeBSD x86 and x86_64: see <a class="reference external" href="http://www.freshports.org/lang/pypy">FreshPorts</a></li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
 installer <a class="reference external" href="http://www.microsoft.com/en-us/download/details.aspx?id=5582">vcredist_x86.exe</a>.)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -86,7 +86,7 @@
 * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below)
 * `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)`__ (see ``[1]`` below)
 * `Mac OS/X binary (64bit)`__
-* FreeBSD 9.2 x86 64 bit *(hopefully availabe soon)* (see ``[1]`` below)
+* FreeBSD x86 and x86_64: see FreshPorts_
 * `Windows binary (32bit)`__ (you might need the VS 2008 runtime library
   installer `vcredist_x86.exe`_.)
 * `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below)
@@ -111,6 +111,7 @@
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
+.. _FreshPorts: http://www.freshports.org/lang/pypy
 
 Python 3.3.5 compatible PyPy3.3 v5.2
 -------------------------------------

From pypy.commits at gmail.com  Sat Sep 10 15:42:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 12:42:19 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57d4621b.4cc51c0a.57b5f.358d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r791:91c9256c0b81
Date: 2016-09-10 21:42 +0200
http://bitbucket.org/pypy/pypy.org/changeset/91c9256c0b81/

Log:	update the values

diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $58969 of $80000 (73.7%)
+   $58979 of $80000 (73.7%)
    <div id="progressbar">
    </div>
 
@@ -25,7 +25,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
   Current status:
-    we have $48195 left
+    we have $48204 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sat Sep 10 15:52:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 12:52:15 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Close the STM donation button,
 as per the January 22 mail on pypy-z.
Message-ID: <57d4646f.eeb8c20a.ba0e2.3d04@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r792:bdb3d0de7c45
Date: 2016-09-10 21:52 +0200
http://bitbucket.org/pypy/pypy.org/changeset/bdb3d0de7c45/

Log:	Close the STM donation button, as per the January 22 mail on pypy-z.

diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -23,7 +23,11 @@
 
   </li>
   <li>
-  <span id="donate_text">This donation goes towards supporting the Transactional Memory in PyPy.
+  <span id="donate_text">At this time, donations towards supporting the
+  Transactional Memory in PyPy are closed.  We cannot promise that the
+  work will finish soon, sorry.</span>
+  
+  <!--
   Current status:
     we have $48204 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
@@ -42,7 +46,7 @@
   <li>
     <div class="sidepic">
       <a id="main_download" href="download.html">Download PyPy</a>
-    </div>
+    </div-->
   </li>
 </ul>
 
diff --git a/js/script2.js b/js/script2.js
--- a/js/script2.js
+++ b/js/script2.js
@@ -25,5 +25,5 @@
 if (location.href.indexOf("py3donate.html") >= 0)
     f = py3k_donate;
 else
-    f = stm_donate;
+    f = general_donate;
 $(document).ready(f);

From pypy.commits at gmail.com  Sat Sep 10 16:04:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 13:04:54 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Close the call for donation
Message-ID: <57d46766.11331c0a.74120.4666@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r793:c51b7e6f6b85
Date: 2016-09-10 22:04 +0200
http://bitbucket.org/pypy/pypy.org/changeset/c51b7e6f6b85/

Log:	Close the call for donation

diff --git a/source/tmdonate2.txt b/source/tmdonate2.txt
--- a/source/tmdonate2.txt
+++ b/source/tmdonate2.txt
@@ -7,6 +7,24 @@
 Transactional Memory, 2nd Call
 ==============================
 
+UPDATE (September 2016):
+
+    **This call for donation is closed.  Thank you everybody for
+    contributing!**
+
+    *We have actually more money in the pot that we can use in the near
+    future.  As it turns out, STM is a hard researchy topic.  Remi Meier
+    is still actively working on this topic as part of his Ph.D. thesis.
+    Armin Rigo thinks more fundamental work is needed, which will
+    eventually lead us to a next generation attempt.*
+
+========================
+
+.. raw:: html
+
+    <br><br>
+
+
 .. class:: download_menu
 
    * `Preamble`_
diff --git a/tmdonate2.html b/tmdonate2.html
--- a/tmdonate2.html
+++ b/tmdonate2.html
@@ -69,7 +69,18 @@
 <div>
 <div id="main">
 <h1 class="title">2nd Call for donations - Transactional Memory in PyPy</h1>
-<ul class="download-menu simple">
+<p>UPDATE (September 2016):</p>
+<blockquote>
+<p><strong>This call for donation is closed.  Thank you everybody for
+contributing!</strong></p>
+<p><em>We have actually more money in the pot that we can use in the near
+future.  As it turns out, STM is a hard researchy topic.  Remi Meier
+is still actively working on this topic as part of his Ph.D. thesis.
+Armin Rigo thinks more fundamental work is needed, which will
+eventually lead us to a next generation attempt.</em></p>
+</blockquote>
+<hr class="docutils" />
+<br><br><ul class="download-menu simple">
 <li><a class="reference internal" href="#preamble">Preamble</a></li>
 <li><a class="reference internal" href="#introduction">Introduction</a></li>
 <li><a class="reference internal" href="#in-more-detail">In more detail</a><ul>

From pypy.commits at gmail.com  Sat Sep 10 16:09:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 13:09:09 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Mention Mozilla at the start of
 this page
Message-ID: <57d46865.c15e1c0a.bf721.3e86@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r794:9ca72ae2a37e
Date: 2016-09-10 22:08 +0200
http://bitbucket.org/pypy/pypy.org/changeset/9ca72ae2a37e/

Log:	Mention Mozilla at the start of this page

diff --git a/py3donate.html b/py3donate.html
--- a/py3donate.html
+++ b/py3donate.html
@@ -69,6 +69,10 @@
 <div>
 <div id="main">
 <h1 class="title">Call for donations - PyPy to support Python3!</h1>
+<p>UPDATE (August 2016):</p>
+<blockquote>
+<em>See</em> <a class="reference external" href="https://morepypy.blogspot.ch/2016/08/pypy-gets-funding-from-mozilla-for.html">PyPy gets funding from Mozilla for Python 3.5 support</a>.
+<em>Individual donations through here are still welcome, though.</em></blockquote>
 <p>UPDATE (February 2014):</p>
 <blockquote>
 <em>Thanks to our donors, we have raised 45% of
@@ -117,7 +121,7 @@
 and a different payment method to avoid the high Paypal fees.  Please
 contact pypy at sfconservancy.org if you want to know details on how
 to donate via other means.</p>
-<div class="section" id="id1">
+<div class="section" id="id2">
 <span id="planned-stages-of-work"></span><h1>Planned stages of work</h1>
 <p>The goal of this project is to write an interpreter that interprets
 version 3 of the Python language. To be precise we would aim at having
diff --git a/source/py3donate.txt b/source/py3donate.txt
--- a/source/py3donate.txt
+++ b/source/py3donate.txt
@@ -3,6 +3,13 @@
 title: Call for donations - PyPy to support Python3!
 ---
 
+UPDATE (August 2016):
+    
+  *See* `PyPy gets funding from Mozilla for Python 3.5 support`__.
+  *Individual donations through here are still welcome, though.*
+
+.. __: https://morepypy.blogspot.ch/2016/08/pypy-gets-funding-from-mozilla-for.html
+
 UPDATE (February 2014):
 
   *Thanks to our donors, we have raised 45% of

From pypy.commits at gmail.com  Sat Sep 10 16:15:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 13:15:16 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Still link to tmdonate2
Message-ID: <57d469d4.eeb8c20a.ba0e2.4414@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r795:10f6c4729525
Date: 2016-09-10 22:14 +0200
http://bitbucket.org/pypy/pypy.org/changeset/10f6c4729525/

Log:	Still link to tmdonate2

diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -24,8 +24,8 @@
   </li>
   <li>
   <span id="donate_text">At this time, donations towards supporting the
-  Transactional Memory in PyPy are closed.  We cannot promise that the
-  work will finish soon, sorry.</span>
+  Transactional Memory in PyPy are closed.  <a href="tmdonate2.html">Read
+  more...</a></span>
   
   <!--
   Current status:

From pypy.commits at gmail.com  Sun Sep 11 02:07:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 10 Sep 2016 23:07:14 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Expand a bit the update
Message-ID: <57d4f492.812e1c0a.d7c1f.ed27@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r796:0539455c31ed
Date: 2016-09-11 08:07 +0200
http://bitbucket.org/pypy/pypy.org/changeset/0539455c31ed/

Log:	Expand a bit the update

diff --git a/py3donate.html b/py3donate.html
--- a/py3donate.html
+++ b/py3donate.html
@@ -71,8 +71,12 @@
 <h1 class="title">Call for donations - PyPy to support Python3!</h1>
 <p>UPDATE (August 2016):</p>
 <blockquote>
-<em>See</em> <a class="reference external" href="https://morepypy.blogspot.ch/2016/08/pypy-gets-funding-from-mozilla-for.html">PyPy gets funding from Mozilla for Python 3.5 support</a>.
-<em>Individual donations through here are still welcome, though.</em></blockquote>
+<em>We are soon releasing a beta supporting Python 3.3.  For the next
+full year, though, see</em> <a class="reference external" href="https://morepypy.blogspot.ch/2016/08/pypy-gets-funding-from-mozilla-for.html">PyPy gets funding from Mozilla for Python 3.5
+support</a>.  <em>Individual donations through here are still welcome, and
+go towards the same goal, which is upgrading PyPy to support Python
+3.x (which really means 3.3/3.5 by now).  Thanks to all our past
+contributors!  Your money has been put to good use so far.</em></blockquote>
 <p>UPDATE (February 2014):</p>
 <blockquote>
 <em>Thanks to our donors, we have raised 45% of
diff --git a/source/py3donate.txt b/source/py3donate.txt
--- a/source/py3donate.txt
+++ b/source/py3donate.txt
@@ -4,9 +4,13 @@
 ---
 
 UPDATE (August 2016):
-    
-  *See* `PyPy gets funding from Mozilla for Python 3.5 support`__.
-  *Individual donations through here are still welcome, though.*
+
+  *We are soon releasing a beta supporting Python 3.3.  For the next
+  full year, though, see* `PyPy gets funding from Mozilla for Python 3.5
+  support`__.  *Individual donations through here are still welcome, and
+  go towards the same goal, which is upgrading PyPy to support Python
+  3.x (which really means 3.3/3.5 by now).  Thanks to all our past
+  contributors!  Your money has been put to good use so far.*
 
 .. __: https://morepypy.blogspot.ch/2016/08/pypy-gets-funding-from-mozilla-for.html
 

From pypy.commits at gmail.com  Sun Sep 11 03:43:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 00:43:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: LibreSSL uses a higher value of
 OPENSSL_VERSION_NUMBER just because
Message-ID: <57d50b16.217fc20a.479c9.18f9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86986:3c3ef872946b
Date: 2016-09-11 09:42 +0200
http://bitbucket.org/pypy/pypy/changeset/3c3ef872946b/

Log:	LibreSSL uses a higher value of OPENSSL_VERSION_NUMBER just because
	(thanks nimaje1)

diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,7 +97,8 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
-if OPENSSL_VERSION_NUMBER >= 0x10100000:
+if (OPENSSL_VERSION_NUMBER >= 0x10100000 and
+     OPENSSL_VERSION_NUMBER < 0x20000000):    # <= libressl :-(
     eci.pre_include_bits = ()
     eci.post_include_bits = ()
     raise Exception("""OpenSSL version >= 1.1 not supported yet.

From pypy.commits at gmail.com  Sun Sep 11 03:54:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 00:54:38 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Another attempt at
 doing a conditional_call that returns a value.
Message-ID: <57d50dbe.c19d1c0a.8367.d277@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86987:096c62600f5c
Date: 2016-09-11 08:31 +0200
http://bitbucket.org/pypy/pypy/changeset/096c62600f5c/

Log:	Another attempt at doing a conditional_call that returns a value.
	Starting from scratch, will copy parts of the
	'conditional_call_value' branch manually.


From pypy.commits at gmail.com  Sun Sep 11 03:54:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 00:54:41 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Goal
Message-ID: <57d50dc1.814ac20a.9e7a1.072d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86988:0e155e8e286a
Date: 2016-09-11 09:53 +0200
http://bitbucket.org/pypy/pypy/changeset/0e155e8e286a/

Log:	Goal

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1178,33 +1178,47 @@
         hop.exception_is_here()
         return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
-def _jit_conditional_call(condition, function, *args):
-    pass
+def _jit_conditional_call_value(value, special_constant, function, *args):
+    """NOT_RPYTHON"""
 
 @specialize.call_location()
 def conditional_call(condition, function, *args):
     if we_are_jitted():
-        _jit_conditional_call(condition, function, *args)
+        _jit_conditional_call_value(condition, True, function, *args)
     else:
         if condition:
             function(*args)
 conditional_call._always_inline_ = True
 
+ at specialize.call_location()
+def conditional_call_value(value, special_constant, function, *args):
+    if we_are_jitted():
+        return _jit_conditional_call_value(value, special_constant,
+                                           function, *args)
+    else:
+        if value == special_constant:
+            value = function(*args)
+        return value
+conditional_call_value._always_inline_ = True
+
 class ConditionalCallEntry(ExtRegistryEntry):
-    _about_ = _jit_conditional_call
+    _about_ = _jit_conditional_call_value
 
     def compute_result_annotation(self, *args_s):
         self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
-                                         args_s[1], args_s[2:])
+                                         args_s[2], args_s[3:])
+        return args_s[0]
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
 
-        args_v = hop.inputargs(lltype.Bool, lltype.Void, *hop.args_r[2:])
-        args_v[1] = hop.args_r[1].get_concrete_llfn(hop.args_s[1],
-                                                    hop.args_s[2:], hop.spaceop)
+        r_value = hop.args_r[0]
+        args_v = hop.inputargs(r_value, r_value, lltype.Void, *hop.args_r[3:])
+        args_v[2] = hop.args_r[2].get_concrete_llfn(hop.args_s[2],
+                                                    hop.args_s[3:], hop.spaceop)
         hop.exception_is_here()
-        return hop.genop('jit_conditional_call', args_v)
+        return hop.genop('jit_conditional_call_value', args_v,
+                         resulttype=r_value)
 
 def enter_portal_frame(unique_id):
     """call this when starting to interpret a function. calling this is not
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -3,7 +3,7 @@
 from rpython.conftest import option
 from rpython.annotator.model import UnionError
 from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote,
-    JitHintError, oopspec, isconstant, conditional_call,
+    JitHintError, oopspec, isconstant, conditional_call, conditional_call_value,
     elidable, unroll_safe, dont_look_inside,
     enter_portal_frame, leave_portal_frame)
 from rpython.rlib.rarithmetic import r_uint
@@ -309,3 +309,14 @@
             leave_portal_frame()
         t = Translation(g, [])
         t.compile_c() # does not crash
+
+    def test_conditional_call_value(self):
+        def g(m):
+            return m + 42
+        def f(n, m):
+            return conditional_call_value(n, -1, g, m)
+
+        res = self.interpret(f, [10, 200])
+        assert res == 10
+        res = self.interpret(f, [-1, 200])
+        assert res == 242
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -430,7 +430,7 @@
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'jit_record_exact_class'  : LLOp(canrun=True),
     'jit_ffi_save_result':  LLOp(canrun=True),
-    'jit_conditional_call': LLOp(),
+    'jit_conditional_call_value': LLOp(),
     'jit_enter_portal_frame': LLOp(canrun=True),
     'jit_leave_portal_frame': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),

From pypy.commits at gmail.com  Sun Sep 11 04:00:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 01:00:57 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: "Test" and fix. It's
 hard to really test because the test previously
Message-ID: <57d50f39.d32d1c0a.1fb34.d26e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86989:4aa76477570f
Date: 2016-09-11 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/4aa76477570f/

Log:	"Test" and fix. It's hard to really test because the test previously
	passed too, just by emitting gcc warnings

diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -320,3 +320,13 @@
         assert res == 10
         res = self.interpret(f, [-1, 200])
         assert res == 242
+
+    def test_compiled_conditional_call_value(self):
+        from rpython.translator.c.test.test_genc import compile
+        def g(m):
+            return m + 42
+        def f(n, m):
+            return conditional_call_value(n, -1, g, m)
+        fn = compile(f, [int, int], backendopt=False)
+        assert fn(10, 200) == 10
+        assert fn(-1, 200) == 242
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -453,7 +453,7 @@
         fnexpr = '((%s)%s)' % (cdecl(typename, ''), self.expr(fnaddr))
         return self.generic_call(FUNC, fnexpr, op.args[1:], op.result)
 
-    def OP_JIT_CONDITIONAL_CALL(self, op):
+    def OP_JIT_CONDITIONAL_CALL_VALUE(self, op):
         return 'abort();  /* jit_conditional_call */'
 
     # low-level operations

From pypy.commits at gmail.com  Sun Sep 11 04:57:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 01:57:13 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: in-progress
Message-ID: <57d51c69.82cbc20a.7af73.e688@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86990:5701981db5d8
Date: 2016-09-11 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/5701981db5d8/

Log:	in-progress

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -364,7 +364,7 @@
         return getattr(self, 'handle_%s_indirect_call' % kind)(op)
 
     def rewrite_call(self, op, namebase, initialargs, args=None,
-                     calldescr=None):
+                     calldescr=None, forced_ir=False):
         """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)'
            into 'i0 = xxx_call_ir_i(fn, descr, [i1,i2], [ref1,ref2])'.
            The name is one of '{residual,direct}_call_{r,ir,irf}_{i,r,f,v}'."""
@@ -374,7 +374,7 @@
         lst_i, lst_r, lst_f = self.make_three_lists(args)
         reskind = getkind(op.result.concretetype)[0]
         if lst_f or reskind == 'f': kinds = 'irf'
-        elif lst_i: kinds = 'ir'
+        elif lst_i or forced_ir: kinds = 'ir'
         else: kinds = 'r'
         sublists = []
         if 'i' in kinds: sublists.append(lst_i)
@@ -1564,20 +1564,19 @@
             return []
         return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver)
 
-    def rewrite_op_jit_conditional_call(self, op):
-        have_floats = False
+    def rewrite_op_jit_conditional_call_value(self, op):
+        have_floats = (getkind(op.result.concretetype) == 'float')
         for arg in op.args:
             if getkind(arg.concretetype) == 'float':
                 have_floats = True
-                break
-        if len(op.args) > 4 + 2 or have_floats:
+        if len(op.args) > 4 + 3 or have_floats:
             raise Exception("Conditional call does not support floats or more than 4 arguments")
-        callop = SpaceOperation('direct_call', op.args[1:], op.result)
+        callop = SpaceOperation('direct_call', op.args[2:], op.result)
         calldescr = self.callcontrol.getcalldescr(callop)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
         op1 = self.rewrite_call(op, 'conditional_call',
-                                op.args[:2], args=op.args[2:],
-                                calldescr=calldescr)
+                                op.args[:3], args=op.args[3:],
+                                calldescr=calldescr, forced_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1199,29 +1199,26 @@
     def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr):
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
-    # conditional calls - note that they cannot return stuff
-    @arguments("cpu", "i", "i", "I", "d")
-    def bhimpl_conditional_call_i_v(cpu, condition, func, args_i, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, None, None, calldescr)
+    @arguments("cpu", "i", "i", "I", "R", "d", returns="i")
+    def bhimpl_conditional_call_ir_i(cpu, value, special_constant,
+                                     func, args_i, args_r, calldescr):
+        if value == special_constant:
+            value = cpu.bh_call_i(func, args_i, args_r, None, calldescr)
+        return value
 
-    @arguments("cpu", "i", "i", "R", "d")
-    def bhimpl_conditional_call_r_v(cpu, condition, func, args_r, calldescr):
-        if condition:
-            cpu.bh_call_v(func, None, args_r, None, calldescr)
+    @arguments("cpu", "r", "r", "I", "R", "d", returns="r")
+    def bhimpl_conditional_call_ir_r(cpu, value, special_constant,
+                                     func, args_i, args_r, calldescr):
+        if value == special_constant:
+            value = cpu.bh_call_r(func, args_i, args_r, None, calldescr)
+        return value
 
-    @arguments("cpu", "i", "i", "I", "R", "d")
-    def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
-                                     calldescr):
-        if condition:
+    @arguments("cpu", "r", "r", "I", "R", "d")
+    def bhimpl_conditional_call_ir_v(cpu, value, special_constant,
+                                     func, args_i, args_r, calldescr):
+        if value == special_constant:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
-    @arguments("cpu", "i", "i", "I", "R", "F", "d")
-    def bhimpl_conditional_call_irf_v(cpu, condition, func, args_i, args_r,
-                                      args_f, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
-
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -96,10 +96,26 @@
 do_call_may_force_f = do_call_f
 do_call_may_force_n = do_call_n
 
-def do_cond_call(cpu, metainterp, argboxes, descr):
-    condbox = argboxes[0]
-    if condbox.getint():
-        do_call_n(cpu, metainterp, argboxes[1:], descr)
+def do_cond_call_i(cpu, metainterp, argboxes, descr):
+    cond = argboxes[0].getint()
+    specialval = argboxes[1].getint()
+    if cond == specialval:
+        return do_call_i(cpu, metainterp, argboxes[2:], descr)
+    return cond
+
+def do_cond_call_r(cpu, metainterp, argboxes, descr):
+    cond = argboxes[0].getref_base()
+    specialval = argboxes[1].getref_base()
+    if cond == specialval:
+        return do_call_r(cpu, metainterp, argboxes[2:], descr)
+    return cond
+
+def do_cond_call_n(cpu, metainterp, argboxes, descr):
+    cond = argboxes[0].getint()
+    specialval = argboxes[1].getint()
+    assert specialval == 1       # cond_call_n is only used with that case
+    if cond == specialval:
+        do_call_n(cpu, metainterp, argboxes[2:], descr)
 
 def do_getarrayitem_gc_i(cpu, _, arraybox, indexbox, arraydescr):
     array = arraybox.getref_base()
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -271,7 +271,8 @@
             return
         if (OpHelpers.is_plain_call(opnum) or
             OpHelpers.is_call_loopinvariant(opnum) or
-            opnum == rop.COND_CALL):
+            opnum == rop.COND_CALL_I or
+            opnum == rop.COND_CALL_R):
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
             if (ef == effectinfo.EF_LOOPINVARIANT or
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -904,7 +904,8 @@
                                        history.CONST_NULL)
         funcbox = ConstInt(rffi.cast(lltype.Signed, vinfo.clear_vable_ptr))
         calldescr = vinfo.clear_vable_descr
-        self.execute_varargs(rop.COND_CALL, [condbox, funcbox, box],
+        self.execute_varargs(rop.COND_CALL_I, [condbox, history.CONST_TRUE,
+                                               funcbox, box],
                              calldescr, False, False)
 
     def _get_virtualizable_field_index(self, fielddescr):
@@ -1056,22 +1057,26 @@
     opimpl_residual_call_irf_f = _opimpl_residual_call3
     opimpl_residual_call_irf_v = _opimpl_residual_call3
 
-    @arguments("box", "box", "boxes", "descr", "orgpc")
-    def opimpl_conditional_call_i_v(self, condbox, funcbox, argboxes, calldescr,
-                                    pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
-
-    opimpl_conditional_call_r_v = opimpl_conditional_call_i_v
-
-    @arguments("box", "box", "boxes2", "descr", "orgpc")
-    def opimpl_conditional_call_ir_v(self, condbox, funcbox, argboxes,
-                                     calldescr, pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
-
-    @arguments("box", "box", "boxes3", "descr", "orgpc")
-    def opimpl_conditional_call_irf_v(self, condbox, funcbox, argboxes,
-                                      calldescr, pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
+    @arguments("box", "box", "box", "boxes2", "descr", "orgpc")
+    def opimpl_conditional_call_ir_i(self, condbox, specialvalbox,
+                                     funcbox, argboxes, calldescr, pc):
+        return self.do_conditional_call(condbox, specialvalbox,
+                                        funcbox, argboxes, calldescr, pc,
+                                        rop.COND_CALL_I)
+
+    @arguments("box", "box", "box", "boxes2", "descr", "orgpc")
+    def opimpl_conditional_call_ir_r(self, condbox, specialvalbox,
+                                     funcbox, argboxes, calldescr, pc):
+        return self.do_conditional_call(condbox, specialvalbox,
+                                        funcbox, argboxes, calldescr, pc,
+                                        rop.COND_CALL_R)
+
+    @arguments("box", "box", "box", "boxes2", "descr", "orgpc")
+    def opimpl_conditional_call_ir_v(self, condbox, specialvalbox,
+                                     funcbox, argboxes, calldescr, pc):
+        return self.do_conditional_call(condbox, specialvalbox,
+                                        funcbox, argboxes, calldescr, pc,
+                                        rop.COND_CALL_N)
 
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
@@ -1724,16 +1729,19 @@
             else:
                 assert False
 
-    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc):
-        if isinstance(condbox, ConstInt) and condbox.value == 0:
-            return   # so that the heapcache can keep argboxes virtual
+    def do_conditional_call(self, condbox, specialvalbox,
+                            funcbox, argboxes, descr, pc, rop_num):
+        if (isinstance(condbox, Const) and
+                not condbox.same_constant(specialvalbox)):
+            return condbox  # so that the heapcache can keep argboxes virtual
         allboxes = self._build_allboxes(funcbox, argboxes, descr)
         effectinfo = descr.get_extra_info()
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
         pure = effectinfo.check_is_elidable()
-        return self.execute_varargs(rop.COND_CALL, [condbox] + allboxes, descr,
-                                    exc, pure)
+        return self.execute_varargs(rop_num,
+                                    [condbox, specialvalbox] + allboxes,
+                                    descr, exc, pure)
 
     def _do_jit_force_virtual(self, allboxes, descr, pc):
         assert len(allboxes) == 2
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1149,7 +1149,7 @@
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
     'CALL/*d/rfin',
-    'COND_CALL/*d/n',
+    'COND_CALL/*d/rin',
     # a conditional call, with first argument as a condition
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1178,13 +1178,15 @@
         hop.exception_is_here()
         return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
+def _jit_conditional_call(value, ignored, function, *args):
+    """NOT_RPYTHON"""
 def _jit_conditional_call_value(value, special_constant, function, *args):
     """NOT_RPYTHON"""
 
 @specialize.call_location()
 def conditional_call(condition, function, *args):
     if we_are_jitted():
-        _jit_conditional_call_value(condition, True, function, *args)
+        _jit_conditional_call(condition, True, function, *args)
     else:
         if condition:
             function(*args)
@@ -1202,12 +1204,14 @@
 conditional_call_value._always_inline_ = True
 
 class ConditionalCallEntry(ExtRegistryEntry):
-    _about_ = _jit_conditional_call_value
+    _about_ = _jit_conditional_call_value, _jit_conditional_call
 
     def compute_result_annotation(self, *args_s):
+        from rpython.annotator import model as annmodel
         self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
                                          args_s[2], args_s[3:])
-        return args_s[0]
+        if self.instance is _jit_conditional_call_value:
+            return args_s[0]
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
@@ -1218,7 +1222,7 @@
                                                     hop.args_s[3:], hop.spaceop)
         hop.exception_is_here()
         return hop.genop('jit_conditional_call_value', args_v,
-                         resulttype=r_value)
+                         resulttype=hop.r_result)
 
 def enter_portal_frame(unique_id):
     """call this when starting to interpret a function. calling this is not

From pypy.commits at gmail.com  Sun Sep 11 05:00:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 02:00:14 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: in-progress
Message-ID: <57d51d1e.8759c20a.ca591.e591@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86991:b3f2dba6bb50
Date: 2016-09-11 10:59 +0200
http://bitbucket.org/pypy/pypy/changeset/b3f2dba6bb50/

Log:	in-progress

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -1328,11 +1328,21 @@
         x = math.sqrt(y)
         return support.cast_to_floatstorage(x)
 
-    def execute_cond_call(self, calldescr, cond, func, *args):
-        if not cond:
-            return
-        # cond_call can't have a return value
-        self.execute_call_n(calldescr, func, *args)
+    def execute_cond_call_i(self, calldescr, cond, specialval, func, *args):
+        if cond == specialval:
+            cond = self.execute_call_i(calldescr, func, *args)
+        return cond
+
+    def execute_cond_call_r(self, calldescr, cond, specialval, func, *args):
+        if cond == specialval:
+            cond = self.execute_call_r(calldescr, func, *args)
+        return cond
+
+    def execute_cond_call_n(self, calldescr, cond, specialval, func, *args):
+        assert specialval == 1
+        assert cond in (0, 1)
+        if cond == specialval:
+            self.execute_call_n(calldescr, func, *args)
 
     def _execute_call(self, calldescr, func, *args):
         effectinfo = calldescr.get_extra_info()
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1074,9 +1074,9 @@
     @arguments("box", "box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, specialvalbox,
                                      funcbox, argboxes, calldescr, pc):
-        return self.do_conditional_call(condbox, specialvalbox,
-                                        funcbox, argboxes, calldescr, pc,
-                                        rop.COND_CALL_N)
+        self.do_conditional_call(condbox, specialvalbox,
+                                 funcbox, argboxes, calldescr, pc,
+                                 rop.COND_CALL_N)
 
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):

From pypy.commits at gmail.com  Sun Sep 11 05:08:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 02:08:38 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Tests, progress
Message-ID: <57d51f16.11331c0a.74120.ec59@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86992:64fbf35bf49e
Date: 2016-09-11 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/64fbf35bf49e/

Log:	Tests, progress

diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -52,3 +52,31 @@
 
         assert self.meta_interp(main, [10]) == 42
         self.check_resops(guard_no_exception=0)
+
+    def test_cond_call_i(self):
+        def f(l, n):
+            l.append(n)
+            return 1000
+
+        def main(n):
+            l = []
+            x = jit.conditional_call_value(n, 10, f, l, n)
+            return x + len(l)
+
+        assert self.interp_operations(main, [10]) == 1001
+        assert self.interp_operations(main, [5]) == 5
+
+    def test_cond_call_r(self):
+        def f(n):
+            return [n]
+
+        def main(n):
+            if n == 10:
+                l = []
+            else:
+                l = None
+            l = jit.conditional_call_value(l, None, f, n)
+            return len(l)
+
+        assert self.interp_operations(main, [10]) == 0
+        assert self.interp_operations(main, [5]) == 1
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -3,7 +3,8 @@
 import py
 
 from rpython.rlib.nonconst import NonConstant
-from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize
+from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.sourcetools import rpython_wrapper
@@ -1198,8 +1199,12 @@
         return _jit_conditional_call_value(value, special_constant,
                                            function, *args)
     else:
-        if value == special_constant:
-            value = function(*args)
+        if not we_are_translated() or isinstance(value, int):
+            if value == special_constant:
+                value = function(*args)
+        else:
+            if value is special_constant:
+                value = function(*args)
         return value
 conditional_call_value._always_inline_ = True
 

From pypy.commits at gmail.com  Sun Sep 11 05:57:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 02:57:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: This test is meant to test for
 virtualize.optimize_COND_CALL().
Message-ID: <57d52a81.8759c20a.ca591.f5f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r86994:24e26ca99bb1
Date: 2016-09-11 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/24e26ca99bb1/

Log:	This test is meant to test for virtualize.optimize_COND_CALL().
	Previously it was passing for an unrelated reason.

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7592,7 +7592,7 @@
         ops = """
         [i0]
         p1 = new_with_vtable(descr=nodesize)
-        cond_call(1, 123, p1, descr=clear_vable)
+        cond_call(i0, 123, p1, descr=clear_vable)
         jump(i0)
         """
         expected = """

From pypy.commits at gmail.com  Sun Sep 11 05:58:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 02:58:39 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Fix
Message-ID: <57d52acf.07c41c0a.297ec.faa3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86995:f3ea6663fd04
Date: 2016-09-11 11:58 +0200
http://bitbucket.org/pypy/pypy/changeset/f3ea6663fd04/

Log:	Fix

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7592,7 +7592,7 @@
         ops = """
         [i0]
         p1 = new_with_vtable(descr=nodesize)
-        cond_call_n(1, 1, 123, p1, descr=clear_vable)
+        cond_call_n(i0, 1, 123, p1, descr=clear_vable)
         jump(i0)
         """
         expected = """
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -94,11 +94,11 @@
     optimize_CALL_MAY_FORCE_F = optimize_CALL_MAY_FORCE_I
     optimize_CALL_MAY_FORCE_N = optimize_CALL_MAY_FORCE_I
 
-    def XXXoptimize_COND_CALL(self, op):
+    def optimize_COND_CALL_N(self, op):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE:
-            opinfo = self.getptrinfo(op.getarg(2))
+            opinfo = self.getptrinfo(op.getarg(3))
             if opinfo and opinfo.is_virtual():
                 return
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -904,7 +904,7 @@
                                        history.CONST_NULL)
         funcbox = ConstInt(rffi.cast(lltype.Signed, vinfo.clear_vable_ptr))
         calldescr = vinfo.clear_vable_descr
-        self.execute_varargs(rop.COND_CALL_I, [condbox, history.CONST_TRUE,
+        self.execute_varargs(rop.COND_CALL_N, [condbox, history.CONST_TRUE,
                                                funcbox, box],
                              calldescr, False, False)
 

From pypy.commits at gmail.com  Sun Sep 11 06:23:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 03:23:44 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Add support for
 COND_CALL_X in the x86 backend
Message-ID: <57d530b0.ec02c20a.dabde.fc17@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86996:87c41fa2e0f5
Date: 2016-09-11 12:23 +0200
http://bitbucket.org/pypy/pypy/changeset/87c41fa2e0f5/

Log:	Add support for COND_CALL_X in the x86 backend

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -757,17 +757,17 @@
         next_op = operations[i + 1]
         opnum = next_op.getopnum()
         if (opnum != rop.GUARD_TRUE and opnum != rop.GUARD_FALSE
-                                    and opnum != rop.COND_CALL):
+                                    and opnum != rop.COND_CALL_N):
             return False
         if next_op.getarg(0) is not op:
             return False
         if self.longevity[op][1] > i + 1:
             return False
-        if opnum != rop.COND_CALL:
+        if opnum != rop.COND_CALL_N:
             if op in operations[i + 1].getfailargs():
                 return False
         else:
-            if op in operations[i + 1].getarglist()[1:]:
+            if op in operations[i + 1].getarglist()[3:]:
                 return False
         return True
 
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -392,7 +392,7 @@
         opnum = next_op.getopnum()
         if not (opnum == rop.GUARD_TRUE or
                 opnum == rop.GUARD_FALSE or
-                opnum == rop.COND_CALL):
+                opnum == rop.COND_CALL_N):
             return False
         if next_op.getarg(0) is not op:
             return False
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2215,51 +2215,6 @@
         excvalue = self.cpu.grab_exc_value(deadframe)
         assert not excvalue
 
-    def test_cond_call_value(self):
-        def func_int(*args):
-            called.append(args)
-            return len(args) * 100 + 1000
-
-        for i in range(5):
-            called = []
-
-            FUNC = self.FuncType([lltype.Signed] * i, lltype.Signed)
-            func_ptr = llhelper(lltype.Ptr(FUNC), func_int)
-            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-                                             EffectInfo.MOST_GENERAL)
-
-            ops = '''
-            [i0, i1, i2, i3, i4, i5, i6, f0, f1]
-            i15 = cond_call_i(i1, 20, ConstClass(func_ptr), %s)
-            guard_false(i0, descr=faildescr) [i1,i2,i3,i4,i5,i6,i15, f0,f1]
-            finish(i15)
-            ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] +
-                            ["descr=calldescr"])
-            loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
-                                         'func_ptr': func_ptr,
-                                         'calldescr': calldescr})
-            looptoken = JitCellToken()
-            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
-            f1 = longlong.getfloatstorage(1.2)
-            f2 = longlong.getfloatstorage(3.4)
-            frame = self.cpu.execute_token(looptoken, 1, 50, 1, 2, 3, 4, 5,
-                                           f1, f2)
-            assert not called
-            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
-                        50, 1, 2, 3, 4, 5, 50]
-            assert longlong.getrealfloat(
-                        self.cpu.get_float_value(frame, 7)) == 1.2
-            assert longlong.getrealfloat(
-                        self.cpu.get_float_value(frame, 8)) == 3.4
-            #
-            frame = self.cpu.execute_token(looptoken, 1, 20, 1, 2, 3, 4, 5,
-                                           f1, f2)
-            assert called == [(1, 2, 3, 4)[:i]]
-            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
-                        20, 1, 2, 3, 4, 5, i * 100 + 1000]
-            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 1.2
-            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 8)) == 3.4
-
     def test_cond_call_gc_wb(self):
         def func_void(a):
             record.append(rffi.cast(lltype.Signed, a))
@@ -2422,7 +2377,7 @@
 
             ops = '''
             [i0, i1, i2, i3, i4, i5, i6, f0, f1]
-            cond_call(i1, ConstClass(func_ptr), %s)
+            cond_call_n(i1, 1, ConstClass(func_ptr), %s)
             guard_false(i0, descr=faildescr) [i1, i2, i3, i4, i5, i6, f0, f1]
             ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] + ["descr=calldescr"])
             loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
@@ -2434,7 +2389,7 @@
             f2 = longlong.getfloatstorage(3.4)
             frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5, f1, f2)
             assert not called
-            for j in range(5):
+            for j in range(6):
                 assert self.cpu.get_int_value(frame, j) == j
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 6)) == 1.2
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 3.4
@@ -2472,7 +2427,7 @@
             ops = '''
             [%s, %s, i3, i4]
             i2 = %s(%s)
-            cond_call(i2, ConstClass(func_ptr), i3, i4, descr=calldescr)
+            cond_call_n(i2, 1, ConstClass(func_ptr), i3, i4, descr=calldescr)
             guard_no_exception(descr=faildescr) []
             finish()
             ''' % ("i0" if operation.startswith('int') else "f0",
@@ -2492,6 +2447,51 @@
                                            67, 89)
             assert called == [(67, 89)]
 
+    def test_cond_call_value(self):
+        def func_int(*args):
+            called.append(args)
+            return len(args) * 100 + 1000
+
+        for i in range(5):
+            called = []
+
+            FUNC = self.FuncType([lltype.Signed] * i, lltype.Signed)
+            func_ptr = llhelper(lltype.Ptr(FUNC), func_int)
+            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                             EffectInfo.MOST_GENERAL)
+
+            ops = '''
+            [i0, i1, i2, i3, i4, i5, i6, f0, f1]
+            i15 = cond_call_i(i1, 20, ConstClass(func_ptr), %s)
+            guard_false(i0, descr=faildescr) [i1,i2,i3,i4,i5,i6,i15, f0,f1]
+            finish(i15)
+            ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] +
+                            ["descr=calldescr"])
+            loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
+                                         'func_ptr': func_ptr,
+                                         'calldescr': calldescr})
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+            f1 = longlong.getfloatstorage(1.2)
+            f2 = longlong.getfloatstorage(3.4)
+            frame = self.cpu.execute_token(looptoken, 1, 50, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert not called
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        50, 1, 2, 3, 4, 5, 50]
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 8)) == 3.4
+            #
+            frame = self.cpu.execute_token(looptoken, 1, 20, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert called == [(1, 2, 3, 4)[:i]]
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        20, 1, 2, 3, 4, 5, i * 100 + 1000]
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 8)) == 3.4
+
     def test_force_operations_returning_void(self):
         values = []
         def maybe_force(token, flag):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -174,8 +174,8 @@
         # copy registers to the frame, with the exception of the
         # 'cond_call_register_arguments' and eax, because these have already
         # been saved by the caller.  Note that this is not symmetrical:
-        # these 5 registers are saved by the caller but restored here at
-        # the end of this function.
+        # these 5 registers are saved by the caller but 4 of them are
+        # restored here at the end of this function.
         self._push_all_regs_to_frame(mc, cond_call_register_arguments + [eax],
                                      supports_floats, callee_only)
         # the caller already did push_gcmap(store=True)
@@ -198,7 +198,7 @@
             mc.ADD(esp, imm(WORD * 7))
         self.set_extra_stack_depth(mc, 0)
         self.pop_gcmap(mc)   # cancel the push_gcmap(store=True) in the caller
-        self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only)
+        self._pop_all_regs_from_frame(mc, [eax], supports_floats, callee_only)
         mc.RET()
         return mc.materialize(self.cpu, [])
 
@@ -1703,7 +1703,8 @@
         self.implement_guard(guard_token)
         # If the previous operation was a COND_CALL, overwrite its conditional
         # jump to jump over this GUARD_NO_EXCEPTION as well, if we can
-        if self._find_nearby_operation(-1).getopnum() == rop.COND_CALL:
+        if self._find_nearby_operation(-1).getopnum() in (
+                rop.COND_CALL_N, rop.COND_CALL_I, rop.COND_CALL_R):
             jmp_adr = self.previous_cond_call_jcond
             offset = self.mc.get_relative_pos() - jmp_adr
             if offset <= 127:
@@ -2381,7 +2382,7 @@
     def label(self):
         self._check_frame_depth_debug(self.mc)
 
-    def cond_call(self, op, gcmap, imm_func, arglocs):
+    def cond_call(self, op, gcmap, imm_func, arglocs, resloc=None):
         assert self.guard_success_cc >= 0
         self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0)
                                                             # patched later
@@ -2394,11 +2395,14 @@
         # plus the register 'eax'
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         should_be_saved = self._regalloc.rm.reg_bindings.values()
+        pop_eax = False
         for gpr in cond_call_register_arguments + [eax]:
-            if gpr not in should_be_saved:
+            if gpr not in should_be_saved or gpr is resloc:
                 continue
             v = gpr_reg_mgr_cls.all_reg_indexes[gpr.value]
             self.mc.MOV_br(v * WORD + base_ofs, gpr.value)
+            if gpr is eax:
+                pop_eax = True
         #
         # load the 0-to-4 arguments into these registers
         from rpython.jit.backend.x86.jump import remap_frame_layout
@@ -2422,8 +2426,15 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.CALL(imm(follow_jump(cond_call_adr)))
+        # if this is not a COND_CALL_N, we need to move the result in place
+        if resloc is not None and resloc is not eax:
+            self.mc.MOV(resloc, eax)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  We must only pop eax, if needed.
+        if pop_eax:
+            v = gpr_reg_mgr_cls.all_reg_indexes[eax.value]
+            self.mc.MOV_rb(eax.value, v * WORD + base_ofs)
+        #
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -928,7 +928,7 @@
 
     consider_cond_call_gc_wb_array = consider_cond_call_gc_wb
 
-    def XXXconsider_cond_call(self, op):
+    def consider_cond_call_n(self, op):
         # A 32-bit-only, asmgcc-only issue: 'cond_call_register_arguments'
         # contains edi and esi, which are also in asmgcroot.py:ASM_FRAMEDATA.
         # We must make sure that edi and esi do not contain GC pointers.
@@ -938,16 +938,27 @@
                     self.rm.force_spill_var(box)
                     assert box not in self.rm.reg_bindings
         #
-        assert op.type == 'v'
+        #assert op.type == 'v'
         args = op.getarglist()
-        assert 2 <= len(args) <= 4 + 2     # maximum 4 arguments
-        v = args[1]
-        assert isinstance(v, Const)
-        imm_func = self.rm.convert_to_imm(v)
-        arglocs = [self.loc(args[i]) for i in range(2, len(args))]
+        assert 3 <= len(args) <= 4 + 3     # maximum 4 arguments
+        v_func = args[2]
+        assert isinstance(v_func, Const)
+        imm_func = self.rm.convert_to_imm(v_func)
+        arglocs = [self.loc(args[i]) for i in range(3, len(args))]
         gcmap = self.get_gcmap()
-        self.load_condition_into_cc(op.getarg(0))
-        self.assembler.cond_call(op, gcmap, imm_func, arglocs)
+        if op.type == 'v':   # COND_CALL_N
+            self.load_condition_into_cc(op.getarg(0))
+            resloc = None
+        else:
+            condvalue_loc = self.loc(args[0])
+            assert not isinstance(condvalue_loc, ImmedLoc)
+            self.assembler.mc.CMP(condvalue_loc, self.loc(args[1]))
+            self.assembler.guard_success_cc = rx86.Conditions['E']
+            resloc = self.rm.force_result_in_reg(op, args[0])
+        self.assembler.cond_call(op, gcmap, imm_func, arglocs, resloc)
+
+    consider_cond_call_i = consider_cond_call_n
+    consider_cond_call_r = consider_cond_call_n
 
     def consider_call_malloc_nursery(self, op):
         size_box = op.getarg(0)

From pypy.commits at gmail.com  Sun Sep 11 06:25:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 03:25:27 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Add a (passing) test
Message-ID: <57d53117.c70a1c0a.2484a.fda9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86997:599b72a6e219
Date: 2016-09-11 12:24 +0200
http://bitbucket.org/pypy/pypy/changeset/599b72a6e219/

Log:	Add a (passing) test

diff --git a/rpython/jit/backend/x86/test/test_call.py b/rpython/jit/backend/x86/test/test_call.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_call.py
@@ -0,0 +1,7 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test import test_call
+
+class TestCall(Jit386Mixin, test_call.TestCall):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_call.py
+    pass

From pypy.commits at gmail.com  Sun Sep 11 09:39:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 06:39:40 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: more in-progress
Message-ID: <57d55e9c.11331c0a.74120.3fb7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86998:1e11cddce2d6
Date: 2016-09-11 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/1e11cddce2d6/

Log:	more in-progress

diff --git a/rpython/jit/backend/x86/test/test_call.py b/rpython/jit/backend/x86/test/test_call.py
--- a/rpython/jit/backend/x86/test/test_call.py
+++ b/rpython/jit/backend/x86/test/test_call.py
@@ -1,7 +1,7 @@
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
 from rpython.jit.metainterp.test import test_call
 
-class TestCall(Jit386Mixin, test_call.TestCall):
+class TestCall(Jit386Mixin, test_call.CallTest):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_call.py
     pass
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -96,21 +96,21 @@
 do_call_may_force_f = do_call_f
 do_call_may_force_n = do_call_n
 
-def do_cond_call_i(cpu, metainterp, argboxes, descr):
+def do_cond_call_pure_i(cpu, metainterp, argboxes, descr):
     cond = argboxes[0].getint()
     specialval = argboxes[1].getint()
     if cond == specialval:
         return do_call_i(cpu, metainterp, argboxes[2:], descr)
     return cond
 
-def do_cond_call_r(cpu, metainterp, argboxes, descr):
+def do_cond_call_pure_r(cpu, metainterp, argboxes, descr):
     cond = argboxes[0].getref_base()
     specialval = argboxes[1].getref_base()
     if cond == specialval:
         return do_call_r(cpu, metainterp, argboxes[2:], descr)
     return cond
 
-def do_cond_call_n(cpu, metainterp, argboxes, descr):
+def do_cond_call(cpu, metainterp, argboxes, descr):
     cond = argboxes[0].getint()
     specialval = argboxes[1].getint()
     assert specialval == 1       # cond_call_n is only used with that case
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -516,28 +516,41 @@
     optimize_CALL_LOOPINVARIANT_F = optimize_CALL_LOOPINVARIANT_I
     optimize_CALL_LOOPINVARIANT_N = optimize_CALL_LOOPINVARIANT_I
 
-    def optimize_COND_CALL_N(self, op):
+    def optimize_COND_CALL(self, op):
         arg0 = self.get_box_replacement(op.getarg(0))
         arg1 = self.get_box_replacement(op.getarg(1))
-        if arg0.type == 'i':
+        equal = -1    # unknown
+        if isinstance(arg0, Const):
+            equal = arg0.same_constant(arg1)
+        elif arg0.type == 'i':
             b1 = self.getintbound(arg0)
             b2 = self.getintbound(arg1)
-            drop = b1.known_gt(b2) or b1.known_lt(b2)
-        elif arg0.type == 'r' and arg1.same_constant(CONST_NULL):
-            drop = self.getnullness(arg0) == INFO_NONNULL
-        else:
-            drop = False
-        if drop:
+            if b1.known_gt(b2) or b1.known_lt(b2):
+                equal = 0    # different
+        elif arg0.type == 'r':
+            if arg1.same_constant(CONST_NULL):
+                if self.getnullness(arg0) == INFO_NONNULL:
+                    equal = 0    # different
+            else:
+                info0 = self.getptrinfo(arg0)
+                if info0 and info0.is_virtual():
+                    equal = 0    # a virtual can't be equal to a constant
+        #
+        if equal == 1:
+            if op.type == 'v':
+                opnum = rop.CALL_N
+            else:
+                opnum = OpHelpers.call_pure_for_descr(op.getdescr())
+            op = self.replace_op_with(op, opnum, args=op.getarglist()[2:])
+            self.send_extra_operation(op)
+        elif equal == 0:
             if op.type != 'v':
                 self.make_equal_to(op, arg0)
             self.last_emitted_operation = REMOVED
-            return
-        if arg0.same_box(arg1):
-            opnum = OpHelpers.call_for_type(op.type)
-            op = self.replace_op_with(op, opnum, args=op.getarglist()[2:])
-        self.emit_operation(op)
-    optimize_COND_CALL_I = optimize_COND_CALL_N
-    optimize_COND_CALL_R = optimize_COND_CALL_N
+        else:
+            self.emit_operation(op)
+    optimize_COND_CALL_PURE_I = optimize_COND_CALL
+    optimize_COND_CALL_PURE_R = optimize_COND_CALL
 
     def _optimize_nullness(self, op, box, expect_nonnull):
         info = self.getnullness(box)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7592,7 +7592,7 @@
         ops = """
         [i0]
         p1 = new_with_vtable(descr=nodesize)
-        cond_call_n(i0, 1, 123, p1, descr=clear_vable)
+        cond_call(i0, 1, 123, p1, descr=clear_vable)
         jump(i0)
         """
         expected = """
@@ -8652,7 +8652,7 @@
     def test_cond_call_with_a_constant(self):
         ops = """
         [p1]
-        cond_call_n(1, 1, 123, p1, descr=plaincalldescr)
+        cond_call(1, 1, 123, p1, descr=plaincalldescr)
         jump(p1)
         """
         expected = """
@@ -8665,7 +8665,7 @@
     def test_cond_call_with_a_constant_2(self):
         ops = """
         [p1]
-        cond_call_n(0, 1, 123, p1, descr=plaincalldescr)
+        cond_call(0, 1, 123, p1, descr=plaincalldescr)
         jump(p1)
         """
         expected = """
@@ -8677,7 +8677,7 @@
     def test_cond_call_with_a_constant_i(self):
         ops = """
         [p1]
-        i2 = cond_call_i(12, 12, 123, p1, descr=plaincalldescr)
+        i2 = cond_call_pure_i(12, 12, 123, p1, descr=plaincalldescr)
         escape_n(i2)
         jump(p1)
         """
@@ -8692,7 +8692,7 @@
     def test_cond_call_with_a_constant_i2(self):
         ops = """
         [p1]
-        i2 = cond_call_i(12, 45, 123, p1, descr=plaincalldescr)
+        i2 = cond_call_pure_i(12, 45, 123, p1, descr=plaincalldescr)
         escape_n(i2)
         jump(p1)
         """
@@ -8708,7 +8708,7 @@
         [p1, i1]
         i0 = int_gt(i1, 100)
         guard_true(i0) []
-        i2 = cond_call_i(i1, 45, 123, p1, descr=plaincalldescr)
+        i2 = cond_call_pure_i(i1, 45, 123, p1, descr=plaincalldescr)
         i3 = escape_i(i2)
         jump(p1, i3)
         """
@@ -8724,7 +8724,7 @@
     def test_cond_call_r1(self):
         ops = """
         [p1]
-        p2 = cond_call_r(p1, NULL, 123, 45, descr=plaincalldescr)
+        p2 = cond_call_pure_r(p1, NULL, 123, p1, descr=plain_r_calldescr)
         jump(p2)
         """
         self.optimize_loop(ops, ops)
@@ -8733,7 +8733,7 @@
         ops = """
         [p1]
         guard_nonnull(p1) []
-        p2 = cond_call_r(p1, NULL, 123, 45, descr=plaincalldescr)
+        p2 = cond_call_pure_r(p1, NULL, 123, p1, descr=plain_r_calldescr)
         p3 = escape_r(p2)
         jump(p3)
         """
@@ -8745,6 +8745,24 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_cond_call_r3(self):
+        ops = """
+        [p0]
+        p4 = escape_r(4)
+        p1 = same_as_r(ConstPtr(myptr))
+        p2 = cond_call_pure_r(p1, ConstPtr(myptr), 123, p4, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p0]
+        p4 = escape_r(4)
+        p2 = call_r(123, p4, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_hippyvm_unroll_bug(self):
         ops = """
         [p0, i1, i2]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -436,6 +436,10 @@
                     oopspecindex=EffectInfo.OS_INT_PY_MOD)
     int_py_mod_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
 
+    FUNC = lltype.FuncType([], llmemory.GCREF)
+    ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_ELIDABLE_CAN_RAISE)
+    plain_r_calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
+
     namespace = locals()
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -94,7 +94,7 @@
     optimize_CALL_MAY_FORCE_F = optimize_CALL_MAY_FORCE_I
     optimize_CALL_MAY_FORCE_N = optimize_CALL_MAY_FORCE_I
 
-    def optimize_COND_CALL_N(self, op):
+    def optimize_COND_CALL(self, op):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUALIZABLE:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1062,14 +1062,14 @@
                                      funcbox, argboxes, calldescr, pc):
         return self.do_conditional_call(condbox, specialvalbox,
                                         funcbox, argboxes, calldescr, pc,
-                                        rop.COND_CALL_I)
+                                        rop.COND_CALL_PURE_I)
 
     @arguments("box", "box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_r(self, condbox, specialvalbox,
                                      funcbox, argboxes, calldescr, pc):
         return self.do_conditional_call(condbox, specialvalbox,
                                         funcbox, argboxes, calldescr, pc,
-                                        rop.COND_CALL_R)
+                                        rop.COND_CALL_PURE_R)
 
     @arguments("box", "box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, specialvalbox,
@@ -1730,7 +1730,7 @@
                 assert False
 
     def do_conditional_call(self, condbox, specialvalbox,
-                            funcbox, argboxes, descr, pc, rop_num):
+                            funcbox, argboxes, descr, pc, opnum):
         if (isinstance(condbox, Const) and
                 not condbox.same_constant(specialvalbox)):
             return condbox  # so that the heapcache can keep argboxes virtual
@@ -1739,7 +1739,8 @@
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
         pure = effectinfo.check_is_elidable()
-        return self.execute_varargs(rop_num,
+        assert pure == (opnum != rop.COND_CALL_N)
+        return self.execute_varargs(opnum,
                                     [condbox, specialvalbox] + allboxes,
                                     descr, exc, pure)
 
@@ -3081,6 +3082,10 @@
         """ Patch a CALL into a CALL_PURE.
         """
         resbox_as_const = executor.constant_from_op(op)
+        is_cond = (op.opnum == rop.COND_CALL_PURE_I or
+                   op.opnum == rop.COND_CALL_PURE_R)
+        if is_cond:
+            argboxes = argboxes[2:]
         for argbox in argboxes:
             if not isinstance(argbox, Const):
                 break
@@ -3093,6 +3098,8 @@
         # be either removed later by optimizeopt or turned back into CALL.
         arg_consts = [executor.constant_from_op(a) for a in argboxes]
         self.call_pure_results[arg_consts] = resbox_as_const
+        if is_cond:
+            return op    # there is no COND_CALL_I/R
         opnum = OpHelpers.call_pure_for_descr(descr)
         self.history.cut(patch_pos)
         newop = self.history.record_nospec(opnum, argboxes, descr)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1149,7 +1149,7 @@
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
     'CALL/*d/rfin',
-    'COND_CALL/*d/rin',
+    'COND_CALL/*d/n',
     # a conditional call, with first argument as a condition
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',
@@ -1157,6 +1157,7 @@
     'CALL_RELEASE_GIL/*d/fin',
     # release the GIL and "close the stack" for asmgcc
     'CALL_PURE/*d/rfin',             # removed before it's passed to the backend
+    'COND_CALL_PURE/*d/ri',
     'CHECK_MEMORY_ERROR/1/n',   # after a CALL: NULL => propagate MemoryError
     'CALL_MALLOC_NURSERY/1/r',  # nursery malloc, const number of bytes, zeroed
     'CALL_MALLOC_NURSERY_VARSIZE/3d/r',
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -2,7 +2,7 @@
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.rlib import jit
 
-class TestCall(LLJitMixin):
+class CallTest(object):
     def test_indirect_call(self):
         @jit.dont_look_inside
         def f1(x):
@@ -54,19 +54,21 @@
         self.check_resops(guard_no_exception=0)
 
     def test_cond_call_i(self):
+        @jit.elidable   # not really, for tests
         def f(l, n):
             l.append(n)
             return 1000
 
         def main(n):
             l = []
-            x = jit.conditional_call_value(n, 10, f, l, n)
+            x = jit.conditional_call_elidable(n, 10, f, l, n)
             return x + len(l)
 
         assert self.interp_operations(main, [10]) == 1001
         assert self.interp_operations(main, [5]) == 5
 
     def test_cond_call_r(self):
+        @jit.elidable
         def f(n):
             return [n]
 
@@ -75,8 +77,24 @@
                 l = []
             else:
                 l = None
-            l = jit.conditional_call_value(l, None, f, n)
+            l = jit.conditional_call_elidable(l, None, f, n)
             return len(l)
 
         assert self.interp_operations(main, [10]) == 0
         assert self.interp_operations(main, [5]) == 1
+
+    def test_cond_call_constant_in_pyjitpl(self):
+        @jit.elidable
+        def f(a, b):
+            return a + b
+        def main(n):
+            # this is completely constant-folded because the arguments
+            # to f() are constants.
+            return jit.conditional_call_elidable(n, 23, f, 40, 2)
+
+        assert main(12) == 12    # because 12 != 23
+        assert self.interp_operations(main, [12]) == 42   # == f(40, 2)
+
+
+class TestCall(LLJitMixin, CallTest):
+    pass
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1181,11 +1181,20 @@
 
 def _jit_conditional_call(value, ignored, function, *args):
     """NOT_RPYTHON"""
-def _jit_conditional_call_value(value, special_constant, function, *args):
+def _jit_conditional_call_elidable(value, special_constant, function, *args):
     """NOT_RPYTHON"""
 
 @specialize.call_location()
 def conditional_call(condition, function, *args):
+    """Does the same as:
+        
+         if condition:
+             function(*args)
+     
+    but is better for the JIT, in case the condition is often false
+    but could be true occasionally.  It allows the JIT to always produce
+    bridge-free code.
+    """
     if we_are_jitted():
         _jit_conditional_call(condition, True, function, *args)
     else:
@@ -1194,11 +1203,25 @@
 conditional_call._always_inline_ = True
 
 @specialize.call_location()
-def conditional_call_value(value, special_constant, function, *args):
+def conditional_call_elidable(value, special_constant, function, *args):
+    """Does the same as:
+
+        return ONE OF function(*args) OR (value if value != special_constant)
+
+    whichever is better for the JIT.  Usually it first checks if 'value'
+    is equal to 'special_constant', and only if it is, it calls
+    'function(*args)'.  The 'function' must be marked as @elidable.  An
+    example of an "unusual" case is if, say, all arguments are constant.
+    In this case the JIT knows the result of the call in advance, and
+    so it always uses the 'function(*args)' path without comparing
+    'value' and 'special_constant' at all.
+    """
     if we_are_jitted():
-        return _jit_conditional_call_value(value, special_constant,
-                                           function, *args)
+        return _jit_conditional_call_elidable(value, special_constant,
+                                              function, *args)
     else:
+        if not we_are_translated():
+            assert function._elidable_function_ # must call an elidable function
         if not we_are_translated() or isinstance(value, int):
             if value == special_constant:
                 value = function(*args)
@@ -1206,16 +1229,20 @@
             if value is special_constant:
                 value = function(*args)
         return value
-conditional_call_value._always_inline_ = True
+conditional_call_elidable._always_inline_ = True
 
 class ConditionalCallEntry(ExtRegistryEntry):
-    _about_ = _jit_conditional_call_value, _jit_conditional_call
+    _about_ = _jit_conditional_call_elidable, _jit_conditional_call
 
     def compute_result_annotation(self, *args_s):
         from rpython.annotator import model as annmodel
         self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
                                          args_s[2], args_s[3:])
-        if self.instance is _jit_conditional_call_value:
+        if self.instance is _jit_conditional_call_elidable:
+            function = args_s[2].const
+            assert getattr(function, '_elidable_function_', False), (
+                "jit.conditional_call_elidable() must call an elidable "
+                "function, but got %r" % (function,))
             return args_s[0]
 
     def specialize_call(self, hop):
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -3,8 +3,8 @@
 from rpython.conftest import option
 from rpython.annotator.model import UnionError
 from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote,
-    JitHintError, oopspec, isconstant, conditional_call, conditional_call_value,
-    elidable, unroll_safe, dont_look_inside,
+    JitHintError, oopspec, isconstant, conditional_call,
+    elidable, unroll_safe, dont_look_inside, conditional_call_elidable,
     enter_portal_frame, leave_portal_frame)
 from rpython.rlib.rarithmetic import r_uint
 from rpython.rtyper.test.tool import BaseRtypingTest
@@ -310,23 +310,27 @@
         t = Translation(g, [])
         t.compile_c() # does not crash
 
-    def test_conditional_call_value(self):
+    def test_conditional_call_elidable(self):
+        @elidable
         def g(m):
             return m + 42
         def f(n, m):
-            return conditional_call_value(n, -1, g, m)
+            return conditional_call_elidable(n, -1, g, m)
 
+        assert f(10, 200) == 10
+        assert f(-1, 200) == 242
         res = self.interpret(f, [10, 200])
         assert res == 10
         res = self.interpret(f, [-1, 200])
         assert res == 242
 
-    def test_compiled_conditional_call_value(self):
+    def test_compiled_conditional_call_elidable(self):
         from rpython.translator.c.test.test_genc import compile
+        @elidable
         def g(m):
             return m + 42
         def f(n, m):
-            return conditional_call_value(n, -1, g, m)
+            return conditional_call_elidable(n, -1, g, m)
         fn = compile(f, [int, int], backendopt=False)
         assert fn(10, 200) == 10
         assert fn(-1, 200) == 242

From pypy.commits at gmail.com  Sun Sep 11 10:10:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 07:10:18 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: in-progress
Message-ID: <57d565ca.814ac20a.9e7a1.79d9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r86999:ae04add86453
Date: 2016-09-11 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/ae04add86453/

Log:	in-progress

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -1328,17 +1328,17 @@
         x = math.sqrt(y)
         return support.cast_to_floatstorage(x)
 
-    def execute_cond_call_i(self, calldescr, cond, specialval, func, *args):
+    def execute_cond_call_pure_i(self, calldescr, cond, specialval, func,*args):
         if cond == specialval:
             cond = self.execute_call_i(calldescr, func, *args)
         return cond
 
-    def execute_cond_call_r(self, calldescr, cond, specialval, func, *args):
+    def execute_cond_call_pure_r(self, calldescr, cond, specialval, func,*args):
         if cond == specialval:
             cond = self.execute_call_r(calldescr, func, *args)
         return cond
 
-    def execute_cond_call_n(self, calldescr, cond, specialval, func, *args):
+    def execute_cond_call(self, calldescr, cond, specialval, func, *args):
         assert specialval == 1
         assert cond in (0, 1)
         if cond == specialval:
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -271,8 +271,7 @@
             return
         if (OpHelpers.is_plain_call(opnum) or
             OpHelpers.is_call_loopinvariant(opnum) or
-            opnum == rop.COND_CALL_I or
-            opnum == rop.COND_CALL_R):
+            OpHelpers.is_cond_call_pure(opnum)):
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
             if (ef == effectinfo.EF_LOOPINVARIANT or
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1076,7 +1076,7 @@
                                      funcbox, argboxes, calldescr, pc):
         self.do_conditional_call(condbox, specialvalbox,
                                  funcbox, argboxes, calldescr, pc,
-                                 rop.COND_CALL_N)
+                                 rop.COND_CALL)
 
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
@@ -1554,8 +1554,8 @@
         op = self.metainterp.execute_and_record_varargs(opnum, argboxes,
                                                             descr=descr)
         if pure and not self.metainterp.last_exc_value and op:
-            op = self.metainterp.record_result_of_call_pure(op, argboxes, descr,
-                patch_pos)
+            op = self.metainterp.record_result_of_call_pure(op, opnum,
+                argboxes, descr, patch_pos)
             exc = exc and not isinstance(op, Const)
         if exc:
             if op is not None:
@@ -1739,7 +1739,7 @@
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
         pure = effectinfo.check_is_elidable()
-        assert pure == (opnum != rop.COND_CALL_N)
+        assert pure == (opnum != rop.COND_CALL)
         return self.execute_varargs(opnum,
                                     [condbox, specialvalbox] + allboxes,
                                     descr, exc, pure)
@@ -3078,12 +3078,12 @@
         debug_stop("jit-abort-longest-function")
         return max_jdsd, max_key
 
-    def record_result_of_call_pure(self, op, argboxes, descr, patch_pos):
+    def record_result_of_call_pure(self, op, opnum, argboxes, descr, patch_pos):
         """ Patch a CALL into a CALL_PURE.
         """
         resbox_as_const = executor.constant_from_op(op)
-        is_cond = (op.opnum == rop.COND_CALL_PURE_I or
-                   op.opnum == rop.COND_CALL_PURE_R)
+        is_cond = (opnum == rop.COND_CALL_PURE_I or
+                   opnum == rop.COND_CALL_PURE_R)
         if is_cond:
             argboxes = argboxes[2:]
         for argbox in argboxes:
@@ -3099,6 +3099,7 @@
         arg_consts = [executor.constant_from_op(a) for a in argboxes]
         self.call_pure_results[arg_consts] = resbox_as_const
         if is_cond:
+            import pdb;pdb.set_trace()
             return op    # there is no COND_CALL_I/R
         opnum = OpHelpers.call_pure_for_descr(descr)
         self.history.cut(patch_pos)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -884,6 +884,8 @@
     _args = None
 
     def initarglist(self, args):
+        if self.opnum == rop.CALL_PURE_I:
+            import pdb;pdb.set_trace()
         self._args = args
         if not we_are_translated() and \
                self.__class__.__name__.startswith('FINISH'):   # XXX remove me
@@ -1439,6 +1441,11 @@
                 opnum == rop.CALL_RELEASE_GIL_N)
 
     @staticmethod
+    def is_cond_call_pure(opnum):
+        return (opnum == rop.COND_CALL_PURE_I or
+                opnum == rop.COND_CALL_PURE_R)
+
+    @staticmethod
     def is_ovf(opnum):
         return rop._OVF_FIRST <= opnum <= rop._OVF_LAST
 
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -54,18 +54,15 @@
         self.check_resops(guard_no_exception=0)
 
     def test_cond_call_i(self):
-        @jit.elidable   # not really, for tests
-        def f(l, n):
-            l.append(n)
-            return 1000
+        @jit.elidable
+        def f(n):
+            return n * 200
 
         def main(n):
-            l = []
-            x = jit.conditional_call_elidable(n, 10, f, l, n)
-            return x + len(l)
+            return jit.conditional_call_elidable(n, 10, f, n)
 
-        assert self.interp_operations(main, [10]) == 1001
-        assert self.interp_operations(main, [5]) == 5
+        assert self.interp_operations(main, [10]) == 2000
+        assert self.interp_operations(main, [15]) == 15
 
     def test_cond_call_r(self):
         @jit.elidable
@@ -92,8 +89,11 @@
             # to f() are constants.
             return jit.conditional_call_elidable(n, 23, f, 40, 2)
 
-        assert main(12) == 12    # because 12 != 23
-        assert self.interp_operations(main, [12]) == 42   # == f(40, 2)
+        assert main(12) == 12                            # because 12 != 23
+        assert self.interp_operations(main, [12]) == 12  # because 12 != 23
+        self.check_operations_history(finish=1)   # empty history
+        assert self.interp_operations(main, [23]) == 42  # because 23 == 23
+        self.check_operations_history(finish=1)   # empty history
 
 
 class TestCall(LLJitMixin, CallTest):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1236,14 +1236,14 @@
 
     def compute_result_annotation(self, *args_s):
         from rpython.annotator import model as annmodel
-        self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
-                                         args_s[2], args_s[3:])
+        s_res = self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
+                                                 args_s[2], args_s[3:])
         if self.instance is _jit_conditional_call_elidable:
             function = args_s[2].const
             assert getattr(function, '_elidable_function_', False), (
                 "jit.conditional_call_elidable() must call an elidable "
                 "function, but got %r" % (function,))
-            return args_s[0]
+            return annmodel.unionof(s_res, args_s[0])
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype

From pypy.commits at gmail.com  Sun Sep 11 10:28:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 07:28:04 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Test. Improve comment
Message-ID: <57d569f4.948e1c0a.a05af.0e7b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r87000:72155a41e42e
Date: 2016-09-11 16:27 +0200
http://bitbucket.org/pypy/pypy/changeset/72155a41e42e/

Log:	Test. Improve comment

diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -95,6 +95,27 @@
         assert self.interp_operations(main, [23]) == 42  # because 23 == 23
         self.check_operations_history(finish=1)   # empty history
 
+    def test_cond_call_constant_in_optimizer(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        @jit.elidable
+        def externfn(x):
+            return x - 3
+        class V:
+            def __init__(self, value):
+                self.value = value
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                v = V(m)
+                n -= jit.conditional_call_elidable(p, -42, externfn, v.value)
+            return n
+        res = self.meta_interp(f, [21, 5, -42])
+        assert res == -1
+        # the COND_CALL_PURE is constant-folded away by optimizeopt.py
+        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=0,
+                          int_sub=2)
+
 
 class TestCall(LLJitMixin, CallTest):
     pass
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1209,12 +1209,18 @@
         return ONE OF function(*args) OR (value if value != special_constant)
 
     whichever is better for the JIT.  Usually it first checks if 'value'
-    is equal to 'special_constant', and only if it is, it calls
-    'function(*args)'.  The 'function' must be marked as @elidable.  An
-    example of an "unusual" case is if, say, all arguments are constant.
-    In this case the JIT knows the result of the call in advance, and
-    so it always uses the 'function(*args)' path without comparing
-    'value' and 'special_constant' at all.
+    is equal to 'special_constant', and conditionally calls
+    'function(*args)' if it is.  The 'function' must be marked as
+    @elidable.
+
+    If 'value != special_constant', then the JIT assumes that both
+    solutions would work and _are equal_.  This means that if, say, all
+    arguments to the call are constants, then the JIT removes everything
+    and replaces it with a constant (by constant-folding
+    'function(*args)').  That constant is whatever value was seen as
+    result during tracing (which is the original 'value' if different
+    from 'special_constant'--that's supposed to be equal to
+    'function(*args)').
     """
     if we_are_jitted():
         return _jit_conditional_call_elidable(value, special_constant,

From pypy.commits at gmail.com  Sun Sep 11 11:18:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 08:18:54 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Next test, passing
Message-ID: <57d575de.411d1c0a.99c34.5e69@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r87001:8e0af8639310
Date: 2016-09-11 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/8e0af8639310/

Log:	Next test, passing

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -3099,7 +3099,6 @@
         arg_consts = [executor.constant_from_op(a) for a in argboxes]
         self.call_pure_results[arg_consts] = resbox_as_const
         if is_cond:
-            import pdb;pdb.set_trace()
             return op    # there is no COND_CALL_I/R
         opnum = OpHelpers.call_pure_for_descr(descr)
         self.history.cut(patch_pos)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -884,8 +884,6 @@
     _args = None
 
     def initarglist(self, args):
-        if self.opnum == rop.CALL_PURE_I:
-            import pdb;pdb.set_trace()
         self._args = args
         if not we_are_translated() and \
                self.__class__.__name__.startswith('FINISH'):   # XXX remove me
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -116,6 +116,29 @@
         self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=0,
                           int_sub=2)
 
+    def test_cond_call_constant_in_optimizer_2(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        @jit.elidable
+        def externfn(x):
+            return 2
+        class V:
+            def __init__(self, value):
+                self.value = value
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                assert p > 14
+                assert p < 16
+                n -= jit.conditional_call_elidable(p, 15, externfn, n)
+            return n
+        res = self.meta_interp(f, [21, 5, 15])
+        assert res == -1
+        # optimizer: the COND_CALL_PURE is turned into a regular
+        # CALL_PURE, which itself becomes CALL
+        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
+                          int_sub=2)
+
 
 class TestCall(LLJitMixin, CallTest):
     pass

From pypy.commits at gmail.com  Sun Sep 11 11:18:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 08:18:58 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: in-progress. Issue:
 there is a lot of logic to handle call_pure already.
Message-ID: <57d575e2.442d1c0a.36d62.5a96@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r87003:c0e117b21066
Date: 2016-09-11 17:18 +0200
http://bitbucket.org/pypy/pypy/changeset/c0e117b21066/

Log:	in-progress. Issue: there is a lot of logic to handle call_pure
	already. Maybe we should kill the resop call_pure and convert
	(hopefully not too hard) that logic to be about cond_call_pure?

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1738,8 +1738,7 @@
         effectinfo = descr.get_extra_info()
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
-        pure = effectinfo.check_is_elidable()
-        assert pure == (opnum != rop.COND_CALL)
+        pure = effectinfo.check_is_elidable() or (opnum != rop.COND_CALL)
         return self.execute_varargs(opnum,
                                     [condbox, specialvalbox] + allboxes,
                                     descr, exc, pure)
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -54,7 +54,6 @@
         self.check_resops(guard_no_exception=0)
 
     def test_cond_call_i(self):
-        @jit.elidable
         def f(n):
             return n * 200
 
@@ -65,7 +64,6 @@
         assert self.interp_operations(main, [15]) == 15
 
     def test_cond_call_r(self):
-        @jit.elidable
         def f(n):
             return [n]
 
@@ -81,7 +79,6 @@
         assert self.interp_operations(main, [5]) == 1
 
     def test_cond_call_constant_in_pyjitpl(self):
-        @jit.elidable
         def f(a, b):
             return a + b
         def main(n):
@@ -97,7 +94,6 @@
 
     def test_cond_call_constant_in_optimizer(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
-        @jit.elidable
         def externfn(x):
             return x - 3
         class V:
@@ -118,12 +114,8 @@
 
     def test_cond_call_constant_in_optimizer_2(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
-        @jit.elidable
         def externfn(x):
             return 2
-        class V:
-            def __init__(self, value):
-                self.value = value
         def f(n, m, p):
             while n > 0:
                 myjitdriver.can_enter_jit(n=n, p=p, m=m)
@@ -141,12 +133,8 @@
 
     def test_cond_call_constant_in_optimizer_3(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
-        @jit.elidable
         def externfn(x):
             return 1
-        class V:
-            def __init__(self, value):
-                self.value = value
         def f(n, m, p):
             while n > 0:
                 myjitdriver.can_enter_jit(n=n, p=p, m=m)
@@ -165,6 +153,25 @@
         self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
                           int_sub=4)
 
+    def test_cond_call_constant_in_optimizer_4(self):
+        class X:
+            def __init__(self, value):
+                self.value = value
+                self.triple = -1
+            def _compute_triple(self):
+                self.triple = self.value * 3
+                return self.triple
+            def get_triple(self):
+                return jit.conditional_call_elidable(self.triple, -1,
+                                                     X._compute_triple, self)
+        def main(n, initvalue):
+            x = X(n)
+            x.triple = initvalue
+            return x.get_triple() + x.get_triple()
+
+        assert self.interp_operations(main, [100, -1]) == 600
+        self.check_operations_history(finish=1)   # empty history
+
 
 class TestCall(LLJitMixin, CallTest):
     pass
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1210,8 +1210,8 @@
 
     whichever is better for the JIT.  Usually it first checks if 'value'
     is equal to 'special_constant', and conditionally calls
-    'function(*args)' if it is.  The 'function' must be marked as
-    @elidable.
+    'function(*args)' if it is.  The 'function' must not be marked as
+    @elidable, but is treated almost like an @elidable.
 
     If 'value != special_constant', then the JIT assumes that both
     solutions would work and _are equal_.  This means that if, say, all
@@ -1226,8 +1226,6 @@
         return _jit_conditional_call_elidable(value, special_constant,
                                               function, *args)
     else:
-        if not we_are_translated():
-            assert function._elidable_function_ # must call an elidable function
         if not we_are_translated() or isinstance(value, int):
             if value == special_constant:
                 value = function(*args)
@@ -1245,10 +1243,6 @@
         s_res = self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
                                                  args_s[2], args_s[3:])
         if self.instance is _jit_conditional_call_elidable:
-            function = args_s[2].const
-            assert getattr(function, '_elidable_function_', False), (
-                "jit.conditional_call_elidable() must call an elidable "
-                "function, but got %r" % (function,))
             return annmodel.unionof(s_res, args_s[0])
 
     def specialize_call(self, hop):
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -311,7 +311,6 @@
         t.compile_c() # does not crash
 
     def test_conditional_call_elidable(self):
-        @elidable
         def g(m):
             return m + 42
         def f(n, m):
@@ -326,7 +325,6 @@
 
     def test_compiled_conditional_call_elidable(self):
         from rpython.translator.c.test.test_genc import compile
-        @elidable
         def g(m):
             return m + 42
         def f(n, m):

From pypy.commits at gmail.com  Sun Sep 11 11:18:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Sep 2016 08:18:56 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_2: Next test, passing
Message-ID: <57d575e0.c15e1c0a.bf721.5554@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_2
Changeset: r87002:cc788cdd76d7
Date: 2016-09-11 16:37 +0200
http://bitbucket.org/pypy/pypy/changeset/cc788cdd76d7/

Log:	Next test, passing

diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -139,6 +139,32 @@
         self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
                           int_sub=2)
 
+    def test_cond_call_constant_in_optimizer_3(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        @jit.elidable
+        def externfn(x):
+            return 1
+        class V:
+            def __init__(self, value):
+                self.value = value
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                assert p > 14
+                assert p < 16
+                n0 = n
+                n -= jit.conditional_call_elidable(p, 15, externfn, n0)
+                n -= jit.conditional_call_elidable(p, 15, externfn, n0)
+            return n
+        res = self.meta_interp(f, [21, 5, 15])
+        assert res == -1
+        # same as test_cond_call_constant_in_optimizer_2, but the two
+        # intermediate CALL_PUREs are replaced with only one, because
+        # they are called with the same arguments
+        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
+                          int_sub=4)
+
 
 class TestCall(LLJitMixin, CallTest):
     pass

From pypy.commits at gmail.com  Sun Sep 11 15:26:43 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 12:26:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: test, fix Py_buffer format,
 which can be a string (issue #2396 and IRC discussion)
Message-ID: <57d5aff3.94071c0a.829af.a3d9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87004:a1c19f421bb3
Date: 2016-09-09 12:37 +0300
http://bitbucket.org/pypy/pypy/changeset/a1c19f421bb3/

Log:	test, fix Py_buffer format, which can be a string (issue #2396 and
	IRC discussion) (grafted from
	fe0add22fd7e549d497ce95929a3a1f36e1c4d46)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -118,7 +118,7 @@
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
@@ -647,7 +647,7 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
-        ('_format', rffi.UCHAR),
+        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
         ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -133,6 +133,7 @@
 
 /* Py3k buffer interface, adapted for PyPy */
 #define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -147,7 +148,7 @@
     Py_ssize_t *shape;
     Py_ssize_t *strides;
     Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format;
+    unsigned char _format[Py_MAX_FMT];
     Py_ssize_t _strides[Py_MAX_NDIMS];
     Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,5 +1,5 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -45,10 +45,22 @@
     view.c_len = w_obj.getlength()
     view.c_itemsize = w_obj.buf.getitemsize()
     rffi.setintfield(view, 'c_ndim', ndim)
-    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
     view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
     view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
     view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = w_obj.buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        if ord(fmt[i]) > 255:
+            view.c_format[i] = '*'
+        else:
+            view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'        
     shape = w_obj.buf.getshape()
     strides = w_obj.buf.getstrides()
     for i in range(ndim):
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
 import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,4 +1,5 @@
 import pytest
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
@@ -17,8 +18,12 @@
         w_buf = space.newbuffer(StringBuffer("hello"))
         w_memoryview = api.PyMemoryView_FromObject(w_buf)
         w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
-        ndim = w_view.c_ndim
-        assert ndim == 1
+        assert w_view.c_ndim == 1
+        f = rffi.charp2str(w_view.c_format)
+        assert f == 'B'
+        assert w_view.c_shape[0] == 5
+        assert w_view.c_strides[0] == 1
+        assert w_view.c_len == 5
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     def test_fillWithObject(self):

From pypy.commits at gmail.com  Sun Sep 11 15:26:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 12:26:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: move all Py_buffer to memoryobject,
 create helper function
Message-ID: <57d5aff5.c398c20a.dfeea.ad32@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87005:391d8ffc245f
Date: 2016-09-09 13:59 +0300
http://bitbucket.org/pypy/pypy/changeset/391d8ffc245f/

Log:	move all Py_buffer to memoryobject, create helper function (grafted
	from f8f2080418c8ba11b6bc85e752ba9ad10740d59a)

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,66 +1,19 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import buffer
-from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer)
-from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
+from pypy.module.cpyext.pyobject import PyObject
 
-def _IsFortranContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
+ at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
+def PyObject_CheckBuffer(space, pyobj):
+    """Return 1 if obj supports the buffer interface otherwise 0."""
+    as_buffer = pyobj.c_ob_type.c_tp_as_buffer
+    flags = pyobj.c_ob_type.c_tp_flags
+    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
         return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(view.c_ndim):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-
-def _IsCContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(ndim - 1, -1, -1):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-        
-
- at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fort):
-    """Return 1 if the memory defined by the view is C-style (fortran is
-    'C') or Fortran-style (fortran is 'F') contiguous or either one
-    (fortran is 'A').  Return 0 otherwise."""
-    # traverse the strides, checking for consistent stride increases from
-    # right-to-left (c) or left-to-right (fortran). Copied from cpython
-    if not view.c_suboffsets:
-        return 0
-    if (fort == 'C'):
-        return _IsCContiguous(view)
-    elif (fort == 'F'):
-        return _IsFortranContiguous(view)
-    elif (fort == 'A'):
-        return (_IsCContiguous(view) or _IsFortranContiguous(view))
     return 0
 
-    
+
 
 class CBuffer(buffer.Buffer):
 
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -2,6 +2,7 @@
                          Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
 
 from pypy.interpreter.error import oefmt
@@ -10,6 +11,119 @@
 from pypy.objspace.std.memoryobject import W_MemoryView
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    return fill_Py_buffer(space, w_obj, view, flags)
+    view.c_obj = make_ref(space, w_obj)
+
+def fill_Py_buffer(space, buf, view):    
+    # c_buf, c_obj have been filled in
+    ndim = buf.getndim()
+    view.c_len = buf.getlength()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        if ord(fmt[i]) > 255:
+            view.c_format[i] = '*'
+        else:
+            view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'        
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+ at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+    """Return 1 if the memory defined by the view is C-style (fortran is
+    'C') or Fortran-style (fortran is 'F') contiguous or either one
+    (fortran is 'A').  Return 0 otherwise."""
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
     return space.call_method(space.builtin, "memoryview", w_obj)
@@ -42,32 +156,7 @@
         view.c_obj = make_ref(space, w_s)
         rffi.setintfield(view, 'c_readonly', 1)
         isstr = True
-    view.c_len = w_obj.getlength()
-    view.c_itemsize = w_obj.buf.getitemsize()
-    rffi.setintfield(view, 'c_ndim', ndim)
-    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
-    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-    fmt = w_obj.buf.getformat()
-    n = Py_MAX_FMT - 1 # NULL terminated buffer
-    if len(fmt) > n:
-        ### WARN?
-        pass
-    else:
-        n = len(fmt)
-    for i in range(n):
-        if ord(fmt[i]) > 255:
-            view.c_format[i] = '*'
-        else:
-            view.c_format[i] = fmt[i]
-    view.c_format[n] = '\x00'        
-    shape = w_obj.buf.getshape()
-    strides = w_obj.buf.getstrides()
-    for i in range(ndim):
-        view.c_shape[i] = shape[i]
-        view.c_strides[i] = strides[i]
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    fill_Py_buffer(space, w_obj.buf, view)     
     return view
 
 @cpython_api([lltype.Ptr(Py_buffer)], PyObject)

From pypy.commits at gmail.com  Sun Sep 11 15:26:47 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 12:26:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Delete RPython impls of
 PyObject_CheckBuffer and PyObject_GetBuffer once again
Message-ID: <57d5aff7.ccd11c0a.e2d2e.a962@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87006:47b8ec8f4aa3
Date: 2016-09-11 20:03 +0100
http://bitbucket.org/pypy/pypy/changeset/47b8ec8f4aa3/

Log:	Delete RPython impls of PyObject_CheckBuffer and PyObject_GetBuffer
	once again

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -4,17 +4,6 @@
     cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, pyobj):
-    """Return 1 if obj supports the buffer interface otherwise 0."""
-    as_buffer = pyobj.c_ob_type.c_tp_as_buffer
-    flags = pyobj.c_ob_type.c_tp_flags
-    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
-        return 1
-    return 0
-
-
-
 class CBuffer(buffer.Buffer):
 
     _immutable_ = True
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -11,36 +11,7 @@
 from pypy.objspace.std.memoryobject import W_MemoryView
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    flags = widen(flags)
-    buf = space.buffer_w(w_obj, flags)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-    except ValueError:
-        raise BufferError("could not create buffer from object")
-    return fill_Py_buffer(space, w_obj, view, flags)
-    view.c_obj = make_ref(space, w_obj)
-
-def fill_Py_buffer(space, buf, view):    
+def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
     ndim = buf.getndim()
     view.c_len = buf.getlength()
@@ -61,7 +32,7 @@
             view.c_format[i] = '*'
         else:
             view.c_format[i] = fmt[i]
-    view.c_format[n] = '\x00'        
+    view.c_format[n] = '\x00'
     shape = buf.getshape()
     strides = buf.getstrides()
     for i in range(ndim):
@@ -156,7 +127,7 @@
         view.c_obj = make_ref(space, w_s)
         rffi.setintfield(view, 'c_readonly', 1)
         isstr = True
-    fill_Py_buffer(space, w_obj.buf, view)     
+    fill_Py_buffer(space, w_obj.buf, view)
     return view
 
 @cpython_api([lltype.Ptr(Py_buffer)], PyObject)

From pypy.commits at gmail.com  Sun Sep 11 15:26:49 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 12:26:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57d5aff9.010c1c0a.8043.c1a5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87007:fbcb88bba801
Date: 2016-09-11 20:26 +0100
http://bitbucket.org/pypy/pypy/changeset/fbcb88bba801/

Log:	hg merge default

diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
--- a/pypy/doc/release-pypy2.7-v5.4.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -9,16 +9,16 @@
     this was unfortunately left out of 5.4.0. My apologies to the new
     contributors
 
-  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+  * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a
     dynamically loadable file
 
-  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+  * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython
 
-  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+  * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
 
-  * Fix for an issue where os.access() accepted a float for mode
+  * Fix for an issue for translating the sandbox
 
-  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+  * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
     the last byte of a unicode string sometimes
 
   * Update built-in cffi_ to version 1.8.1
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -12,4 +12,7 @@
 Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
 in numpypy
 
-
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
 
 from .support import c_compile
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
+
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0
@@ -53,7 +55,48 @@
         libraries=libraries)
     return soname
 
-def compile_extension_module(space, modname, include_dirs=[],
+class SystemCompilationInfo(object):
+    """Bundles all the generic information required to compile extensions.
+
+    Note: here, 'system' means OS + target interpreter + test config + ...
+    """
+    def __init__(self, include_extra=None, compile_extra=None, link_extra=None,
+            extra_libs=None, ext=None):
+        self.include_extra = include_extra or []
+        self.compile_extra = compile_extra
+        self.link_extra = link_extra
+        self.extra_libs = extra_libs
+        self.ext = ext
+
+def get_cpyext_info(space):
+    from pypy.module.imp.importing import get_so_extension
+    state = space.fromcache(State)
+    api_library = state.api_lib
+    if sys.platform == 'win32':
+        libraries = [api_library]
+        # '%s' undefined; assuming extern returning int
+        compile_extra = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+            (space.int_w(w_maj), space.int_w(w_min))]
+    else:
+        libraries = []
+        if sys.platform.startswith('linux'):
+            compile_extra = [
+                "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
+    return SystemCompilationInfo(
+        include_extra=api.include_dirs,
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        extra_libs=libraries,
+        ext=get_so_extension(space))
+
+
+def compile_extension_module(sys_info, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
@@ -65,35 +108,15 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    state = space.fromcache(State)
-    api_library = state.api_lib
-    if sys.platform == 'win32':
-        libraries = [api_library]
-        # '%s' undefined; assuming extern returning int
-        compile_extra = ["/we4013"]
-        # prevent linking with PythonXX.lib
-        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
-                              (space.int_w(w_maj), space.int_w(w_min))]
-    else:
-        libraries = []
-        if sys.platform.startswith('linux'):
-            compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
-            link_extra = ["-g"]
-        else:
-            compile_extra = link_extra = None
-
     modname = modname.split('.')[-1]
     soname = create_so(modname,
-            include_dirs=api.include_dirs + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra,
-            libraries=libraries)
-    from pypy.module.imp.importing import get_so_extension
-    ext = get_so_extension(space)
-    pydname = soname.new(purebasename=modname, ext=ext)
+        include_dirs=sys_info.include_extra + include_dirs,
+        source_files=source_files,
+        source_strings=source_strings,
+        compile_extra=sys_info.compile_extra,
+        link_extra=sys_info.link_extra,
+        libraries=sys_info.extra_libs)
+    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
     soname.rename(pydname)
     return str(pydname)
 
@@ -106,18 +129,8 @@
         raise RuntimeError("This interpreter does not define a filename "
             "suffix for C extensions!")
 
-def compile_extension_module_applevel(space, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
+def get_sys_info_app():
+    from distutils.sysconfig import get_python_inc
     if sys.platform == 'win32':
         compile_extra = ["/we4013"]
         link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
@@ -128,18 +141,13 @@
         compile_extra = [
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
         link_extra = None
+    ext = get_so_suffix()
+    return SystemCompilationInfo(
+        include_extra=[get_python_inc()],
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        ext=get_so_suffix())
 
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-            include_dirs=[space.include_dir] + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra)
-    ext = get_so_suffix()
-    pydname = soname.new(purebasename=modname, ext=ext)
-    soname.rename(pydname)
-    return str(pydname)
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -154,9 +162,7 @@
 class FakeSpace(object):
     """Like TinyObjSpace, but different"""
     def __init__(self, config):
-        from distutils.sysconfig import get_python_inc
         self.config = config
-        self.include_dir = get_python_inc()
 
     def passthrough(self, arg):
         return arg
@@ -273,11 +279,11 @@
             "the test actually passed in the first place; if it failed "
             "it is likely to reach this place.")
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_only_import(self):
         import cpyext
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_load_error(self):
         import cpyext
         raises(ImportError, cpyext.load_module, "missing.file", "foo")
@@ -305,26 +311,23 @@
     def setup_method(self, func):
         @gateway.unwrap_spec(name=str)
         def compile_module(space, name,
-                           w_separate_module_files=None,
-                           w_separate_module_sources=None):
+                           w_source_files=None,
+                           w_source_strings=None):
             """
             Build an extension module linked against the cpyext api library.
             """
-            if not space.is_none(w_separate_module_files):
-                separate_module_files = space.unwrap(w_separate_module_files)
-                assert separate_module_files is not None
+            if not space.is_none(w_source_files):
+                source_files = space.unwrap(w_source_files)
             else:
-                separate_module_files = []
-            if not space.is_none(w_separate_module_sources):
-                separate_module_sources = space.listview_bytes(
-                    w_separate_module_sources)
-                assert separate_module_sources is not None
+                source_files = None
+            if not space.is_none(w_source_strings):
+                source_strings = space.listview_bytes(w_source_strings)
             else:
-                separate_module_sources = []
-            pydname = self.compile_extension_module(
-                space, name,
-                source_files=separate_module_files,
-                source_strings=separate_module_sources)
+                source_strings = None
+            pydname = compile_extension_module(
+                self.sys_info, name,
+                source_files=source_files,
+                source_strings=source_strings)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -375,7 +378,7 @@
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
                 kwds = dict(source_files=[filename])
-            mod = self.compile_extension_module(space, name,
+            mod = compile_extension_module(self.sys_info, name,
                     include_dirs=include_dirs, **kwds)
 
             if load_it:
@@ -475,11 +478,11 @@
                 return run
             def wrap(func):
                 return func
-            self.compile_extension_module = compile_extension_module_applevel
+            self.sys_info = get_sys_info_app()
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
-            self.compile_extension_module = compile_extension_module
+            self.sys_info = get_cpyext_info(self.space)
         self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
         self.w_reimport_module = wrap(interp2app(reimport_module))
@@ -629,10 +632,10 @@
             skip('record_imported_module not supported in runappdirect mode')
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", separate_module_files=[self.here + 'banana.c'])
+            "apple.banana", source_files=[self.here + 'banana.c'])
         self.record_imported_module("apple.banana")
         date = self.compile_module(
-            "cherry.date", separate_module_files=[self.here + 'date.c'])
+            "cherry.date", source_files=[self.here + 'date.c'])
         self.record_imported_module("cherry.date")
 
         # Set up some package state so that the extensions can actually be
@@ -901,7 +904,7 @@
             ])
         raises(SystemError, mod.newexc, "name", Exception, {})
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy specific test')
+    @pytest.mark.skipif(only_pypy, reason='pypy specific test')
     def test_hash_pointer(self):
         mod = self.import_extension('foo', [
             ('get_hash', 'METH_NOARGS',
@@ -952,7 +955,7 @@
         print(p)
         assert 'py' in p
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_version(self):
         mod = self.import_extension('foo', [
             ('get_version', 'METH_NOARGS',
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -88,14 +88,10 @@
         from _numpypy import multiarray as np
         module = self.import_module(name='buffer_test')
         get_buffer_info = module.get_buffer_info
-        # test_export_flags from numpy test_multiarray
         raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
-        # test_relaxed_strides from numpy test_multiarray
-        arr = np.zeros((1, 10))
-        if arr.flags.f_contiguous:
-            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
-            assert strides[0] == 8
-            arr = np.ones((10, 1), order='F')
-            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
-            assert strides[-1] == 8
-
+        arr = np.zeros((1, 10), order='F')
+        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+        assert strides[0] == 8
+        arr = np.zeros((10, 1), order='C')
+        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+        assert strides[-1] == 8
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -1,12 +1,13 @@
 import sys
 
-import py, pytest
+import pytest
 
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 class AppTestThread(AppTestCpythonExtensionBase):
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_thread_ident(self):
         module = self.import_extension('foo', [
             ("get_thread_ident", "METH_NOARGS",
@@ -33,7 +34,7 @@
 
         assert results[0][0] != results[1][0]
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_acquire_lock(self):
         module = self.import_extension('foo', [
             ("test_acquire_lock", "METH_NOARGS",
@@ -57,7 +58,7 @@
             ])
         module.test_acquire_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_release_lock(self):
         module = self.import_extension('foo', [
             ("test_release_lock", "METH_NOARGS",
@@ -79,7 +80,7 @@
             ])
         module.test_release_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_tls(self):
         module = self.import_extension('foo', [
             ("create_key", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,6 +3,7 @@
 import py, pytest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 def test_pragma_version():
     from pypy.module.sys.version import CPYTHON_VERSION
@@ -41,11 +42,9 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -95,8 +95,9 @@
              )
     setuppkg("pkg.pkg2", a='', b='')
     setuppkg("pkg.withall",
-             __init__  = "__all__ = ['foobar']",
-             foobar    = "found = 123")
+             __init__  = "__all__ = ['foobar', 'barbaz']",
+             foobar    = "found = 123",
+             barbaz    = "other = 543")
     setuppkg("pkg.withoutall",
              __init__  = "",
              foobar    = "found = 123")
@@ -724,6 +725,7 @@
             d = {}
             exec("from pkg.withall import *", d)
             assert d["foobar"].found == 123
+            assert d["barbaz"].other == 543
 
     def test_import_star_does_not_find_submodules_without___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -392,31 +392,39 @@
         extobj_w = space.newlist([space.wrap(8192), space.wrap(0), space.w_None])
         return extobj_w
 
+
+_reflected_ops = {
+        'add': 'radd',
+        'subtract': 'rsub',
+        'multiply': 'rmul',
+        'divide': 'rdiv',
+        'true_divide': 'rtruediv',
+        'floor_divide': 'rfloordiv',
+        'remainder': 'rmod',
+        'power': 'rpow',
+        'left_shift': 'rlshift',
+        'right_shift': 'rrshift',
+        'bitwise_and': 'rand',
+        'bitwise_xor': 'rxor',
+        'bitwise_or': 'ror',
+        #/* Comparisons */
+        'equal': 'eq',
+        'not_equal': 'ne',
+        'greater': 'lt',
+        'less': 'gt',
+        'greater_equal': 'le',
+        'less_equal': 'ge',
+}
+
+for key, value in _reflected_ops.items():
+    _reflected_ops[key] = "__" + value + "__"
+del key
+del value
+
 def _has_reflected_op(space, w_obj, op):
-    refops ={ 'add': 'radd',
-            'subtract': 'rsub',
-            'multiply': 'rmul',
-            'divide': 'rdiv',
-            'true_divide': 'rtruediv',
-            'floor_divide': 'rfloordiv',
-            'remainder': 'rmod',
-            'power': 'rpow',
-            'left_shift': 'rlshift',
-            'right_shift': 'rrshift',
-            'bitwise_and': 'rand',
-            'bitwise_xor': 'rxor',
-            'bitwise_or': 'ror',
-            #/* Comparisons */
-            'equal': 'eq',
-            'not_equal': 'ne',
-            'greater': 'lt',
-            'less': 'gt',
-            'greater_equal': 'le',
-            'less_equal': 'ge',
-        }
-    if op not in refops:
+    if op not in _reflected_ops:
         return False
-    return space.getattr(w_obj, space.wrap('__' + refops[op] + '__')) is not None
+    return space.getattr(w_obj, space.wrap(_reflected_ops[op])) is not None
 
 def safe_casting_mode(casting):
     assert casting is not None
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -365,6 +365,13 @@
     def visitor_dispatch_virtual_type(self, visitor):
         raise NotImplementedError("abstract")
 
+    def make_guards(self, op, short, optimizer):
+        from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
+        op = ResOperation(rop.INT_EQ, [op, CONST_0])
+        short.append(op)
+        op = ResOperation(rop.GUARD_FALSE, [op])
+        short.append(op)
+
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7592,7 +7592,7 @@
         ops = """
         [i0]
         p1 = new_with_vtable(descr=nodesize)
-        cond_call(1, 123, p1, descr=clear_vable)
+        cond_call(i0, 123, p1, descr=clear_vable)
         jump(i0)
         """
         expected = """
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,7 +16,7 @@
 from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
      have_debug_prints
 
-class UnrollableOptimizer(Optimizer):    
+class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             if self.optunroll.short_preamble_producer is None:
@@ -120,7 +120,8 @@
                 assert op.get_forwarded() is None
         if check_newops:
             assert not self.optimizer._newoperations
-    
+
+
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
@@ -156,7 +157,7 @@
         current_vs = self.get_virtual_state(end_jump.getarglist())
         # pick the vs we want to jump to
         assert isinstance(celltoken, JitCellToken)
-        
+
         target_virtual_state = self.pick_virtual_state(current_vs,
                                                        state.virtual_state,
                                                 celltoken.target_tokens)
@@ -180,17 +181,27 @@
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)            
+                    self.optimizer._newoperations)
 
         try:
-            new_virtual_state = self.jump_to_existing_trace(end_jump, label_op,
-                                                            state.runtime_boxes)
+            new_virtual_state = self.jump_to_existing_trace(
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
                     self.optimizer._newoperations)
+
+        if new_virtual_state is not None:
+            # Attempt to force virtual boxes in order to avoid jumping
+            # to the preamble.
+            try:
+                new_virtual_state = self.jump_to_existing_trace(
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+            except InvalidLoop:
+                pass
+
         if new_virtual_state is not None:
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
@@ -199,7 +210,7 @@
 
         self.disable_retracing_if_max_retrace_guards(
             self.optimizer._newoperations, target_token)
-        
+
         return (UnrollInfo(target_token, label_op, extra_same_as,
                            self.optimizer.quasi_immutable_deps),
                 self.optimizer._newoperations)
@@ -241,7 +252,8 @@
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
         try:
-            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes)
+            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                             force_boxes=False)
         except InvalidLoop:
             return self.jump_to_preamble(cell_token, jump_op, info)
         if vs is None:
@@ -252,6 +264,14 @@
             cell_token.retraced_count += 1
             debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
         else:
+            # Try forcing boxes to avoid jumping to the preamble
+            try:
+                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                                 force_boxes=True)
+            except InvalidLoop:
+                pass
+            if vs is None:
+                return info, self.optimizer._newoperations[:]
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
@@ -288,7 +308,7 @@
         return info, self.optimizer._newoperations[:]
 
 
-    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes):
+    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False):
         jitcelltoken = jump_op.getdescr()
         assert isinstance(jitcelltoken, JitCellToken)
         virtual_state = self.get_virtual_state(jump_op.getarglist())
@@ -299,7 +319,8 @@
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, args, runtime_boxes, self.optimizer)
+                    virtual_state, args, runtime_boxes, self.optimizer,
+                    force_boxes=force_boxes)
                 patchguardop = self.optimizer.patchguardop
                 for guard in extra_guards.extra_guards:
                     if isinstance(guard, GuardResOp):
@@ -308,8 +329,18 @@
                     self.send_extra_operation(guard)
             except VirtualStatesCantMatch:
                 continue
-            args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
-                args, self.optimizer)
+
+            # When force_boxes == True, creating the virtual args can fail when
+            # components of the virtual state alias. If this occurs, we must
+            # recompute the virtual state as boxes will have been forced.
+            try:
+                args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
+                    args, self.optimizer, force_boxes=force_boxes)
+            except VirtualStatesCantMatch:
+                assert force_boxes
+                virtual_state = self.get_virtual_state(args)
+                continue
+
             short_preamble = target_token.short_preamble
             try:
                 extra = self.inline_short_preamble(args + virtuals, args,
@@ -452,7 +483,7 @@
         # by short preamble
         label_args = exported_state.virtual_state.make_inputargs(
             targetargs, self.optimizer)
-        
+
         self.short_preamble_producer = ShortPreambleBuilder(
             label_args, exported_state.short_boxes,
             exported_state.short_inputargs, exported_state.exported_infos,
@@ -497,7 +528,7 @@
     * runtime_boxes - runtime values for boxes, necessary when generating
                       guards to jump to
     """
-    
+
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
                  short_inputargs, runtime_boxes, memo):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
      ArrayStructInfo, AbstractStructPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound, IntLowerBound
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation, \
      InputArgInt, InputArgRef, InputArgFloat
 from rpython.rlib.debug import debug_print
 
@@ -20,7 +20,7 @@
 
 
 class GenerateGuardState(object):
-    def __init__(self, optimizer=None, guards=None, renum=None, bad=None):
+    def __init__(self, optimizer=None, guards=None, renum=None, bad=None, force_boxes=False):
         self.optimizer = optimizer
         self.cpu = optimizer.cpu
         if guards is None:
@@ -32,6 +32,7 @@
         if bad is None:
             bad = {}
         self.bad = bad
+        self.force_boxes = force_boxes
 
     def get_runtime_item(self, box, descr, i):
         array = box.getref_base()
@@ -303,7 +304,7 @@
             opinfo = state.optimizer.getptrinfo(box)
             assert isinstance(opinfo, ArrayPtrInfo)
         else:
-            opinfo = None            
+            opinfo = None
         for i in range(self.length):
             for descr in self.fielddescrs:
                 index = i * len(self.fielddescrs) + descr.get_index()
@@ -514,6 +515,8 @@
         NotVirtualStateInfo.__init__(self, cpu, type, info)
 
     def _generate_guards(self, other, box, runtime_box, state):
+        if state.force_boxes and isinstance(other, VirtualStateInfo):
+            return self._generate_virtual_guards(other, box, runtime_box, state)
         if not isinstance(other, NotVirtualStateInfoPtr):
             raise VirtualStatesCantMatch(
                     'The VirtualStates does not match as a ' +
@@ -545,6 +548,23 @@
     # to an existing compiled loop or retracing the loop. Both alternatives
     # will always generate correct behaviour, but performance will differ.
 
+    def _generate_virtual_guards(self, other, box, runtime_box, state):
+        """
+        Generate the guards and add state information for unifying a virtual
+        object with a non-virtual. This involves forcing the object in the
+        event that unification can succeed. Since virtual objects cannot be null,
+        this method need only check that the virtual object has the expected type.
+        """
+        assert state.force_boxes and isinstance(other, VirtualStateInfo)
+
+        if self.level == LEVEL_CONSTANT:
+            raise VirtualStatesCantMatch(
+                    "cannot unify a constant value with a virtual object")
+
+        if self.level == LEVEL_KNOWNCLASS:
+            if not self.known_class.same_constant(other.known_class):
+                raise VirtualStatesCantMatch("classes don't match")
+
     def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
                                  state):
         if not isinstance(other, NotVirtualStateInfoPtr):
@@ -617,10 +637,10 @@
             return False
         return True
 
-    def generate_guards(self, other, boxes, runtime_boxes, optimizer):
+    def generate_guards(self, other, boxes, runtime_boxes, optimizer, force_boxes=False):
         assert (len(self.state) == len(other.state) == len(boxes) ==
                 len(runtime_boxes))
-        state = GenerateGuardState(optimizer)
+        state = GenerateGuardState(optimizer, force_boxes=force_boxes)
         for i in range(len(self.state)):
             self.state[i].generate_guards(other.state[i], boxes[i],
                                           runtime_boxes[i], state)
@@ -644,8 +664,8 @@
 
         return boxes
 
-    def make_inputargs_and_virtuals(self, inputargs, optimizer):
-        inpargs = self.make_inputargs(inputargs, optimizer)
+    def make_inputargs_and_virtuals(self, inputargs, optimizer, force_boxes=False):
+        inpargs = self.make_inputargs(inputargs, optimizer, force_boxes)
         # we append the virtuals here in case some stuff is proven
         # to be not a virtual and there are getfields in the short preamble
         # that will read items out of there
@@ -653,7 +673,7 @@
         for i in range(len(inputargs)):
             if not isinstance(self.state[i], NotVirtualStateInfo):
                 virtuals.append(inputargs[i])
-            
+
         return inpargs, virtuals
 
     def debug_print(self, hdr='', bad=None, metainterp_sd=None):
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4507,3 +4507,54 @@
                 i += 1
             return i
         self.meta_interp(f, [])
+
+    def test_round_trip_raw_pointer(self):
+        # The goal of this test to to get a raw pointer op into the short preamble
+        # so we can check that the proper guards are generated
+        # In this case, the resulting short preamble contains
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # i2 = int_eq(i1, 0)
+        # guard_false(i2)
+        #
+        # as opposed to what the JIT used to produce
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # guard_nonnull(i1)
+        #
+        # Which will probably generate correct assembly, but the optimization
+        # pipline expects guard_nonnull arguments to be pointer ops and may crash
+        # and may crash on other input types.
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+
+        class Box(object):
+            _ptr = lltype.nullptr(rffi.CCHARP.TO)
+
+        def new_int_buffer(value):
+            data = lltype.malloc(rffi.CCHARP.TO, rffi.sizeof(rffi.INT), flavor='raw')
+            rffi.cast(rffi.INTP, data)[0] = rffi.cast(rffi.INT, value)
+            return data
+
+        def read_int_buffer(buf):
+            return rffi.cast(rffi.INTP, buf)[0]
+
+        def f():
+            i = 0
+            val = Box()
+            val._ptr = new_int_buffer(1)
+
+            set_param(None, 'retrace_limit', -1)
+            while i < 100:
+                driver.jit_merge_point(i=i, val=val)
+                driver.can_enter_jit(i=i, val=val)
+                # Just to produce a side exit
+                if i & 0b100:
+                    i += 1
+                i += int(read_int_buffer(val._ptr))
+                lltype.free(val._ptr, flavor='raw')
+                val._ptr = new_int_buffer(1)
+            lltype.free(val._ptr, flavor='raw')
+
+        self.meta_interp(f, [])
+        self.check_resops(guard_nonnull=0)
+
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1,8 +1,8 @@
 import py
-from rpython.rlib.jit import JitDriver, promote, dont_look_inside
+from rpython.rlib.jit import JitDriver, promote, dont_look_inside, set_param
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.jit.codewriter.policy import StopAtXPolicy
-from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.test.support import LLJitMixin, get_stats
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper import rclass
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -965,6 +965,82 @@
         self.check_aborted_count(0)
         self.check_target_token_count(4)
 
+    def test_avoid_preamble(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        class Box(object):
+            def __init__(self, v):
+                self.unbox = v
+
+        mask = -2
+        const = Box(X(5))
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            val   = X(0)
+            i     = 0
+            const.unbox = X(5)
+            while i < 17:
+                driver.can_enter_jit(i=i, val=val)
+                driver.jit_merge_point(i=i, val=val)
+                # Logical & rather than comparison to confuse range analysis.
+                # Test only succeeds on the first 2 iterations
+                if i & -2 == 0:
+                    val = const.unbox
+                else:
+                    val = X(i)
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
+
+        # With retracing disable, there will be one optimized loop expecting a
+        # non-virtual X object. The side exit creates a virtual object which must
+        # be allocated to jump to the optimized trace.
+        self.check_resops(jump=3, label=2, new_with_vtable=2)
+        self.check_target_token_count(2)
+        self.check_trace_count(3)
+
+    def test_conflated_virtual_states(self):
+        # All cases are covered when forcing one component of the virtual state
+        # also forces an as yet unseen component.
+        # i.e. expect [NotVirtual, Virtual] and given a pair of aliasing virtual
+        # objects
+        driver = JitDriver(greens=[], reds=['i', 'v1', 'v2'])
+        class Box(object):
+            def __init__(self, v):
+                self.v = v
+
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        const = Box(X(0))
+        def f():
+            set_param(None, 'retrace_limit', -1)
+            set_param(None, 'threshold', 1)
+            i = 0
+            v1 = X(0)
+            v2 = X(0)
+            const.v = X(0)
+            while i < 17:
+                driver.jit_merge_point(i=i, v1=v1, v2=v2)
+                driver.can_enter_jit(i=i, v1=v1, v2=v2)
+                if i & 1 == 0:
+                    v1 = const.v
+                    v2 = X(i)
+                else:
+                    v1 = v2 = X(i)
+                i += 1
+            return None
+        self.meta_interp(f, [])
+        # assert did not crash
+
 
 class VirtualMiscTests:
     def test_multiple_equal_virtuals(self):
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,7 +97,8 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
-if OPENSSL_VERSION_NUMBER >= 0x10100000:
+if (OPENSSL_VERSION_NUMBER >= 0x10100000 and
+     OPENSSL_VERSION_NUMBER < 0x20000000):    # <= libressl :-(
     eci.pre_include_bits = ()
     eci.post_include_bits = ()
     raise Exception("""OpenSSL version >= 1.1 not supported yet.

From pypy.commits at gmail.com  Sun Sep 11 15:47:25 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 11 Sep 2016 12:47:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: test,
 fix for raising non-TypeError exception when conversion fails
Message-ID: <57d5b4cd.e856c20a.8ee44.b5f4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87008:2e99c5b56c4f
Date: 2016-09-11 22:46 +0300
http://bitbucket.org/pypy/pypy/changeset/2e99c5b56c4f/

Log:	test, fix for raising non-TypeError exception when conversion fails

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
         assert module.test_fast_sequence(s[0:-1])
         assert module.test_fast_sequence(s[::-1])
 
+    def test_fast_keyerror(self):
+        module = self.import_extension('foo', [
+            ("test_fast_sequence", "METH_VARARGS",
+             """
+                PyObject *foo;
+                PyObject * seq = PyTuple_GetItem(args, 0);
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "Could not convert object to sequence");
+                if (foo != NULL)
+                {
+                    return foo;
+                }
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                    return PyBool_FromLong(1);
+                }
+                return NULL;
+             """)])
+        class Map(object):
+            def __len__(self):
+                return 1
+
+            def __getitem__(self, index):
+                raise KeyError()
+
+        assert module.test_fast_sequence(Map()) is True
+

From pypy.commits at gmail.com  Sun Sep 11 15:56:51 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 12:56:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix skipping of AppTestApi
Message-ID: <57d5b703.d4301c0a.c5f2b.b418@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87009:038cf89fd5e2
Date: 2016-09-11 20:56 +0100
http://bitbucket.org/pypy/pypy/changeset/038cf89fd5e2/

Log:	Fix skipping of AppTestApi

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -255,9 +255,9 @@
         # enabled automatically by pypy.conftest.
         return leaking
 
+ at pytest.mark.xfail(reason="Skipped until other tests in this file are unskipped")
 class AppTestApi(LeakCheckingTest):
     def setup_class(cls):
-        skip("Skipped until other tests in this file are unskipped")
         from rpython.rlib.clibffi import get_libc_name
         if cls.runappdirect:
             cls.libc = get_libc_name()

From pypy.commits at gmail.com  Sun Sep 11 16:03:29 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 13:03:29 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: rm debugging leftover
Message-ID: <57d5b891.8628c20a.38292.b5e6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87010:13c43c49adbe
Date: 2016-09-11 21:03 +0100
http://bitbucket.org/pypy/pypy/changeset/13c43c49adbe/

Log:	rm debugging leftover

diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -61,7 +61,6 @@
         py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.array._charbuf_start())
         py_buf.c_b_size = buf.getlength()
     else:
-        raise RuntimeError
         raise oefmt(space.w_NotImplementedError, "buffer flavor not supported")
 
 

From pypy.commits at gmail.com  Sun Sep 11 16:04:30 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 13:04:30 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: hg merge default
Message-ID: <57d5b8ce.68adc20a.aeaa2.b1f3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87011:df6683b8ed30
Date: 2016-09-11 21:03 +0100
http://bitbucket.org/pypy/pypy/changeset/df6683b8ed30/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -12,4 +12,7 @@
 Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
 in numpypy
 
-
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,7 +119,7 @@
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
@@ -645,7 +645,7 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
-        ('_format', rffi.UCHAR),
+        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
         ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,9 +1,7 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib.rarithmetic import widen
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
-from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
+from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -14,102 +12,4 @@
         return 1
     return 0  
 
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    flags = widen(flags)
-    buf = space.buffer_w(w_obj, flags)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-    except ValueError:
-        raise BufferError("could not create buffer from object")
-    view.c_len = buf.getlength()
-    view.c_obj = make_ref(space, w_obj)
-    ndim = buf.getndim()
-    view.c_itemsize = buf.getitemsize()
-    rffi.setintfield(view, 'c_readonly', int(buf.readonly))
-    rffi.setintfield(view, 'c_ndim', ndim)
-    view.c_format = rffi.str2charp(buf.getformat())
-    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
-    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
-    shape = buf.getshape()
-    strides = buf.getstrides()
-    for i in range(ndim):
-        view.c_shape[i] = shape[i]
-        view.c_strides[i] = strides[i]
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    return 0
-
-def _IsFortranContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(view.c_ndim):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-
-def _IsCContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(ndim - 1, -1, -1):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-        
-
- at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fort):
-    """Return 1 if the memory defined by the view is C-style (fortran is
-    'C') or Fortran-style (fortran is 'F') contiguous or either one
-    (fortran is 'A').  Return 0 otherwise."""
-    # traverse the strides, checking for consistent stride increases from
-    # right-to-left (c) or left-to-right (fortran). Copied from cpython
-    if not view.c_suboffsets:
-        return 0
-    if (fort == 'C'):
-        return _IsCContiguous(view)
-    elif (fort == 'F'):
-        return _IsFortranContiguous(view)
-    elif (fort == 'A'):
-        return (_IsCContiguous(view) or _IsFortranContiguous(view))
-    return 0
-
     
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -144,6 +144,7 @@
 
 /* Py3k buffer interface, adapted for PyPy */
 #define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -158,7 +159,7 @@
     Py_ssize_t *shape;
     Py_ssize_t *strides;
     Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format;
+    unsigned char _format[Py_MAX_FMT];
     Py_ssize_t _strides[Py_MAX_NDIMS];
     Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,11 +1,125 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    view.c_obj = make_ref(space, w_obj)
+    return fill_Py_buffer(space, buf, view)
+
+def fill_Py_buffer(space, buf, view):    
+    # c_buf, c_obj have been filled in
+    ndim = buf.getndim()
+    view.c_len = buf.getlength()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        if ord(fmt[i]) > 255:
+            view.c_format[i] = '*'
+        else:
+            view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'        
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+ at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+    """Return 1 if the memory defined by the view is C-style (fortran is
+    'C') or Fortran-style (fortran is 'F') contiguous or either one
+    (fortran is 'A').  Return 0 otherwise."""
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
     return space.call_method(space.builtin, "memoryview", w_obj)
@@ -38,19 +152,6 @@
         view.c_obj = make_ref(space, w_s)
         rffi.setintfield(view, 'c_readonly', 1)
         isstr = True
-    view.c_len = w_obj.getlength()
-    view.c_itemsize = w_obj.buf.getitemsize()
-    rffi.setintfield(view, 'c_ndim', ndim)
-    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
-    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
-    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-    shape = w_obj.buf.getshape()
-    strides = w_obj.buf.getstrides()
-    for i in range(ndim):
-        view.c_shape[i] = shape[i]
-        view.c_strides[i] = strides[i]
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    fill_Py_buffer(space, w_obj.buf, view)     
     return view
 
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
 import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.api import PyObject
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
 
 from .support import c_compile
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0
@@ -288,11 +290,11 @@
             "the test actually passed in the first place; if it failed "
             "it is likely to reach this place.")
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_only_import(self):
         import cpyext
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_load_error(self):
         import cpyext
         raises(ImportError, cpyext.load_module, "missing.file", "foo")
@@ -873,7 +875,7 @@
             ])
         raises(SystemError, mod.newexc, "name", Exception, {})
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy specific test')
+    @pytest.mark.skipif(only_pypy, reason='pypy specific test')
     def test_hash_pointer(self):
         mod = self.import_extension('foo', [
             ('get_hash', 'METH_NOARGS',
@@ -924,7 +926,7 @@
         print p
         assert 'py' in p
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_version(self):
         mod = self.import_extension('foo', [
             ('get_version', 'METH_NOARGS',
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,3 +1,4 @@
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
@@ -16,8 +17,12 @@
         w_buf = space.newbuffer(StringBuffer("hello"))
         w_memoryview = api.PyMemoryView_FromObject(w_buf)
         w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
-        ndim = w_view.c_ndim
-        assert ndim == 1
+        assert w_view.c_ndim == 1
+        f = rffi.charp2str(w_view.c_format)
+        assert f == 'B'
+        assert w_view.c_shape[0] == 5
+        assert w_view.c_strides[0] == 1
+        assert w_view.c_len == 5
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):
@@ -38,14 +43,10 @@
         from _numpypy import multiarray as np
         module = self.import_module(name='buffer_test')
         get_buffer_info = module.get_buffer_info
-        # test_export_flags from numpy test_multiarray
         raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
-        # test_relaxed_strides from numpy test_multiarray
-        arr = np.zeros((1, 10))
-        if arr.flags.f_contiguous:
-            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
-            assert strides[0] == 8
-            arr = np.ones((10, 1), order='F')
-            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
-            assert strides[-1] == 8
-
+        arr = np.zeros((1, 10), order='F')
+        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+        assert strides[0] == 8
+        arr = np.zeros((10, 1), order='C')
+        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+        assert strides[-1] == 8
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
         assert module.test_fast_sequence(s[0:-1])
         assert module.test_fast_sequence(s[::-1])
 
+    def test_fast_keyerror(self):
+        module = self.import_extension('foo', [
+            ("test_fast_sequence", "METH_VARARGS",
+             """
+                PyObject *foo;
+                PyObject * seq = PyTuple_GetItem(args, 0);
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "Could not convert object to sequence");
+                if (foo != NULL)
+                {
+                    return foo;
+                }
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                    return PyBool_FromLong(1);
+                }
+                return NULL;
+             """)])
+        class Map(object):
+            def __len__(self):
+                return 1
+
+            def __getitem__(self, index):
+                raise KeyError()
+
+        assert module.test_fast_sequence(Map()) is True
+
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -1,12 +1,13 @@
 import sys
 
-import py, pytest
+import pytest
 
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 class AppTestThread(AppTestCpythonExtensionBase):
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_thread_ident(self):
         module = self.import_extension('foo', [
             ("get_thread_ident", "METH_NOARGS",
@@ -33,7 +34,7 @@
 
         assert results[0][0] != results[1][0]
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_acquire_lock(self):
         module = self.import_extension('foo', [
             ("test_acquire_lock", "METH_NOARGS",
@@ -57,7 +58,7 @@
             ])
         module.test_acquire_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_release_lock(self):
         module = self.import_extension('foo', [
             ("test_release_lock", "METH_NOARGS",
@@ -79,7 +80,7 @@
             ])
         module.test_release_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_tls(self):
         module = self.import_extension('foo', [
             ("create_key", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,6 +3,7 @@
 import py, pytest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 def test_pragma_version():
     from pypy.module.sys.version import CPYTHON_VERSION
@@ -32,11 +33,9 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -365,6 +365,13 @@
     def visitor_dispatch_virtual_type(self, visitor):
         raise NotImplementedError("abstract")
 
+    def make_guards(self, op, short, optimizer):
+        from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
+        op = ResOperation(rop.INT_EQ, [op, CONST_0])
+        short.append(op)
+        op = ResOperation(rop.GUARD_FALSE, [op])
+        short.append(op)
+
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7592,7 +7592,7 @@
         ops = """
         [i0]
         p1 = new_with_vtable(descr=nodesize)
-        cond_call(1, 123, p1, descr=clear_vable)
+        cond_call(i0, 123, p1, descr=clear_vable)
         jump(i0)
         """
         expected = """
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,7 +16,7 @@
 from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
      have_debug_prints
 
-class UnrollableOptimizer(Optimizer):    
+class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             if self.optunroll.short_preamble_producer is None:
@@ -120,7 +120,8 @@
                 assert op.get_forwarded() is None
         if check_newops:
             assert not self.optimizer._newoperations
-    
+
+
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
@@ -156,7 +157,7 @@
         current_vs = self.get_virtual_state(end_jump.getarglist())
         # pick the vs we want to jump to
         assert isinstance(celltoken, JitCellToken)
-        
+
         target_virtual_state = self.pick_virtual_state(current_vs,
                                                        state.virtual_state,
                                                 celltoken.target_tokens)
@@ -180,17 +181,27 @@
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)            
+                    self.optimizer._newoperations)
 
         try:
-            new_virtual_state = self.jump_to_existing_trace(end_jump, label_op,
-                                                            state.runtime_boxes)
+            new_virtual_state = self.jump_to_existing_trace(
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
                     self.optimizer._newoperations)
+
+        if new_virtual_state is not None:
+            # Attempt to force virtual boxes in order to avoid jumping
+            # to the preamble.
+            try:
+                new_virtual_state = self.jump_to_existing_trace(
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+            except InvalidLoop:
+                pass
+
         if new_virtual_state is not None:
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
@@ -199,7 +210,7 @@
 
         self.disable_retracing_if_max_retrace_guards(
             self.optimizer._newoperations, target_token)
-        
+
         return (UnrollInfo(target_token, label_op, extra_same_as,
                            self.optimizer.quasi_immutable_deps),
                 self.optimizer._newoperations)
@@ -241,7 +252,8 @@
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
         try:
-            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes)
+            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                             force_boxes=False)
         except InvalidLoop:
             return self.jump_to_preamble(cell_token, jump_op, info)
         if vs is None:
@@ -252,6 +264,14 @@
             cell_token.retraced_count += 1
             debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
         else:
+            # Try forcing boxes to avoid jumping to the preamble
+            try:
+                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                                 force_boxes=True)
+            except InvalidLoop:
+                pass
+            if vs is None:
+                return info, self.optimizer._newoperations[:]
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
@@ -288,7 +308,7 @@
         return info, self.optimizer._newoperations[:]
 
 
-    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes):
+    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False):
         jitcelltoken = jump_op.getdescr()
         assert isinstance(jitcelltoken, JitCellToken)
         virtual_state = self.get_virtual_state(jump_op.getarglist())
@@ -299,7 +319,8 @@
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, args, runtime_boxes, self.optimizer)
+                    virtual_state, args, runtime_boxes, self.optimizer,
+                    force_boxes=force_boxes)
                 patchguardop = self.optimizer.patchguardop
                 for guard in extra_guards.extra_guards:
                     if isinstance(guard, GuardResOp):
@@ -308,8 +329,18 @@
                     self.send_extra_operation(guard)
             except VirtualStatesCantMatch:
                 continue
-            args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
-                args, self.optimizer)
+
+            # When force_boxes == True, creating the virtual args can fail when
+            # components of the virtual state alias. If this occurs, we must
+            # recompute the virtual state as boxes will have been forced.
+            try:
+                args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
+                    args, self.optimizer, force_boxes=force_boxes)
+            except VirtualStatesCantMatch:
+                assert force_boxes
+                virtual_state = self.get_virtual_state(args)
+                continue
+
             short_preamble = target_token.short_preamble
             try:
                 extra = self.inline_short_preamble(args + virtuals, args,
@@ -452,7 +483,7 @@
         # by short preamble
         label_args = exported_state.virtual_state.make_inputargs(
             targetargs, self.optimizer)
-        
+
         self.short_preamble_producer = ShortPreambleBuilder(
             label_args, exported_state.short_boxes,
             exported_state.short_inputargs, exported_state.exported_infos,
@@ -497,7 +528,7 @@
     * runtime_boxes - runtime values for boxes, necessary when generating
                       guards to jump to
     """
-    
+
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
                  short_inputargs, runtime_boxes, memo):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
      ArrayStructInfo, AbstractStructPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound, IntLowerBound
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation, \
      InputArgInt, InputArgRef, InputArgFloat
 from rpython.rlib.debug import debug_print
 
@@ -20,7 +20,7 @@
 
 
 class GenerateGuardState(object):
-    def __init__(self, optimizer=None, guards=None, renum=None, bad=None):
+    def __init__(self, optimizer=None, guards=None, renum=None, bad=None, force_boxes=False):
         self.optimizer = optimizer
         self.cpu = optimizer.cpu
         if guards is None:
@@ -32,6 +32,7 @@
         if bad is None:
             bad = {}
         self.bad = bad
+        self.force_boxes = force_boxes
 
     def get_runtime_item(self, box, descr, i):
         array = box.getref_base()
@@ -303,7 +304,7 @@
             opinfo = state.optimizer.getptrinfo(box)
             assert isinstance(opinfo, ArrayPtrInfo)
         else:
-            opinfo = None            
+            opinfo = None
         for i in range(self.length):
             for descr in self.fielddescrs:
                 index = i * len(self.fielddescrs) + descr.get_index()
@@ -514,6 +515,8 @@
         NotVirtualStateInfo.__init__(self, cpu, type, info)
 
     def _generate_guards(self, other, box, runtime_box, state):
+        if state.force_boxes and isinstance(other, VirtualStateInfo):
+            return self._generate_virtual_guards(other, box, runtime_box, state)
         if not isinstance(other, NotVirtualStateInfoPtr):
             raise VirtualStatesCantMatch(
                     'The VirtualStates does not match as a ' +
@@ -545,6 +548,23 @@
     # to an existing compiled loop or retracing the loop. Both alternatives
     # will always generate correct behaviour, but performance will differ.
 
+    def _generate_virtual_guards(self, other, box, runtime_box, state):
+        """
+        Generate the guards and add state information for unifying a virtual
+        object with a non-virtual. This involves forcing the object in the
+        event that unification can succeed. Since virtual objects cannot be null,
+        this method need only check that the virtual object has the expected type.
+        """
+        assert state.force_boxes and isinstance(other, VirtualStateInfo)
+
+        if self.level == LEVEL_CONSTANT:
+            raise VirtualStatesCantMatch(
+                    "cannot unify a constant value with a virtual object")
+
+        if self.level == LEVEL_KNOWNCLASS:
+            if not self.known_class.same_constant(other.known_class):
+                raise VirtualStatesCantMatch("classes don't match")
+
     def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
                                  state):
         if not isinstance(other, NotVirtualStateInfoPtr):
@@ -617,10 +637,10 @@
             return False
         return True
 
-    def generate_guards(self, other, boxes, runtime_boxes, optimizer):
+    def generate_guards(self, other, boxes, runtime_boxes, optimizer, force_boxes=False):
         assert (len(self.state) == len(other.state) == len(boxes) ==
                 len(runtime_boxes))
-        state = GenerateGuardState(optimizer)
+        state = GenerateGuardState(optimizer, force_boxes=force_boxes)
         for i in range(len(self.state)):
             self.state[i].generate_guards(other.state[i], boxes[i],
                                           runtime_boxes[i], state)
@@ -644,8 +664,8 @@
 
         return boxes
 
-    def make_inputargs_and_virtuals(self, inputargs, optimizer):
-        inpargs = self.make_inputargs(inputargs, optimizer)
+    def make_inputargs_and_virtuals(self, inputargs, optimizer, force_boxes=False):
+        inpargs = self.make_inputargs(inputargs, optimizer, force_boxes)
         # we append the virtuals here in case some stuff is proven
         # to be not a virtual and there are getfields in the short preamble
         # that will read items out of there
@@ -653,7 +673,7 @@
         for i in range(len(inputargs)):
             if not isinstance(self.state[i], NotVirtualStateInfo):
                 virtuals.append(inputargs[i])
-            
+
         return inpargs, virtuals
 
     def debug_print(self, hdr='', bad=None, metainterp_sd=None):
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4507,3 +4507,54 @@
                 i += 1
             return i
         self.meta_interp(f, [])
+
+    def test_round_trip_raw_pointer(self):
+        # The goal of this test to to get a raw pointer op into the short preamble
+        # so we can check that the proper guards are generated
+        # In this case, the resulting short preamble contains
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # i2 = int_eq(i1, 0)
+        # guard_false(i2)
+        #
+        # as opposed to what the JIT used to produce
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # guard_nonnull(i1)
+        #
+        # Which will probably generate correct assembly, but the optimization
+        # pipline expects guard_nonnull arguments to be pointer ops and may crash
+        # and may crash on other input types.
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+
+        class Box(object):
+            _ptr = lltype.nullptr(rffi.CCHARP.TO)
+
+        def new_int_buffer(value):
+            data = lltype.malloc(rffi.CCHARP.TO, rffi.sizeof(rffi.INT), flavor='raw')
+            rffi.cast(rffi.INTP, data)[0] = rffi.cast(rffi.INT, value)
+            return data
+
+        def read_int_buffer(buf):
+            return rffi.cast(rffi.INTP, buf)[0]
+
+        def f():
+            i = 0
+            val = Box()
+            val._ptr = new_int_buffer(1)
+
+            set_param(None, 'retrace_limit', -1)
+            while i < 100:
+                driver.jit_merge_point(i=i, val=val)
+                driver.can_enter_jit(i=i, val=val)
+                # Just to produce a side exit
+                if i & 0b100:
+                    i += 1
+                i += int(read_int_buffer(val._ptr))
+                lltype.free(val._ptr, flavor='raw')
+                val._ptr = new_int_buffer(1)
+            lltype.free(val._ptr, flavor='raw')
+
+        self.meta_interp(f, [])
+        self.check_resops(guard_nonnull=0)
+
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1,8 +1,8 @@
 import py
-from rpython.rlib.jit import JitDriver, promote, dont_look_inside
+from rpython.rlib.jit import JitDriver, promote, dont_look_inside, set_param
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.jit.codewriter.policy import StopAtXPolicy
-from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.test.support import LLJitMixin, get_stats
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper import rclass
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -965,6 +965,82 @@
         self.check_aborted_count(0)
         self.check_target_token_count(4)
 
+    def test_avoid_preamble(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        class Box(object):
+            def __init__(self, v):
+                self.unbox = v
+
+        mask = -2
+        const = Box(X(5))
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            val   = X(0)
+            i     = 0
+            const.unbox = X(5)
+            while i < 17:
+                driver.can_enter_jit(i=i, val=val)
+                driver.jit_merge_point(i=i, val=val)
+                # Logical & rather than comparison to confuse range analysis.
+                # Test only succeeds on the first 2 iterations
+                if i & -2 == 0:
+                    val = const.unbox
+                else:
+                    val = X(i)
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
+
+        # With retracing disable, there will be one optimized loop expecting a
+        # non-virtual X object. The side exit creates a virtual object which must
+        # be allocated to jump to the optimized trace.
+        self.check_resops(jump=3, label=2, new_with_vtable=2)
+        self.check_target_token_count(2)
+        self.check_trace_count(3)
+
+    def test_conflated_virtual_states(self):
+        # All cases are covered when forcing one component of the virtual state
+        # also forces an as yet unseen component.
+        # i.e. expect [NotVirtual, Virtual] and given a pair of aliasing virtual
+        # objects
+        driver = JitDriver(greens=[], reds=['i', 'v1', 'v2'])
+        class Box(object):
+            def __init__(self, v):
+                self.v = v
+
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        const = Box(X(0))
+        def f():
+            set_param(None, 'retrace_limit', -1)
+            set_param(None, 'threshold', 1)
+            i = 0
+            v1 = X(0)
+            v2 = X(0)
+            const.v = X(0)
+            while i < 17:
+                driver.jit_merge_point(i=i, v1=v1, v2=v2)
+                driver.can_enter_jit(i=i, v1=v1, v2=v2)
+                if i & 1 == 0:
+                    v1 = const.v
+                    v2 = X(i)
+                else:
+                    v1 = v2 = X(i)
+                i += 1
+            return None
+        self.meta_interp(f, [])
+        # assert did not crash
+
 
 class VirtualMiscTests:
     def test_multiple_equal_virtuals(self):
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,7 +97,8 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
-if OPENSSL_VERSION_NUMBER >= 0x10100000:
+if (OPENSSL_VERSION_NUMBER >= 0x10100000 and
+     OPENSSL_VERSION_NUMBER < 0x20000000):    # <= libressl :-(
     eci.pre_include_bits = ()
     eci.post_include_bits = ()
     raise Exception("""OpenSSL version >= 1.1 not supported yet.

From pypy.commits at gmail.com  Sun Sep 11 20:30:48 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 17:30:48 -0700 (PDT)
Subject: [pypy-commit] pypy _py3k-cpyext-A: Merge test-cpyext
Message-ID: <57d5f738.948e1c0a.a05af.bb90@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: _py3k-cpyext-A
Changeset: r87012:b8bcaa6c2cfb
Date: 2016-09-11 21:17 +0100
http://bitbucket.org/pypy/pypy/changeset/b8bcaa6c2cfb/

Log:	Merge test-cpyext

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -42,19 +42,6 @@
         files.append(filename)
     return files
 
-def create_so(modname, include_dirs, source_strings=None, source_files=None,
-        compile_extra=None, link_extra=None, libraries=None):
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    if source_strings:
-        assert not source_files
-        files = convert_sources_to_files(source_strings, dirname)
-        source_files = files
-    soname = c_compile(source_files, outputfilename=str(dirname/modname),
-        compile_extra=compile_extra, link_extra=link_extra,
-        include_dirs=include_dirs,
-        libraries=libraries)
-    return soname
-
 class SystemCompilationInfo(object):
     """Bundles all the generic information required to compile extensions.
 
@@ -68,6 +55,33 @@
         self.extra_libs = extra_libs
         self.ext = ext
 
+    def compile_extension_module(self, name, include_dirs=[],
+            source_files=None, source_strings=None):
+        """
+        Build an extension module and return the filename of the resulting
+        native code file.
+
+        name is the name of the module, possibly including dots if it is a
+        module inside a package.
+
+        Any extra keyword arguments are passed on to ExternalCompilationInfo to
+        build the module (so specify your source with one of those).
+        """
+        modname = name.split('.')[-1]
+        dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+        if source_strings:
+            assert not source_files
+            files = convert_sources_to_files(source_strings, dirname)
+            source_files = files
+        soname = c_compile(source_files, outputfilename=str(dirname/modname),
+            compile_extra=self.compile_extra,
+            link_extra=self.link_extra,
+            include_dirs=self.include_extra + include_dirs,
+            libraries=self.extra_libs)
+        pydname = soname.new(purebasename=modname, ext=self.ext)
+        soname.rename(pydname)
+        return str(pydname)
+
 def get_cpyext_info(space):
     from pypy.module.imp.importing import get_so_extension
     state = space.fromcache(State)
@@ -96,30 +110,6 @@
         ext=get_so_extension(space))
 
 
-def compile_extension_module(sys_info, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-        include_dirs=sys_info.include_extra + include_dirs,
-        source_files=source_files,
-        source_strings=source_strings,
-        compile_extra=sys_info.compile_extra,
-        link_extra=sys_info.link_extra,
-        libraries=sys_info.extra_libs)
-    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
-    soname.rename(pydname)
-    return str(pydname)
-
 def get_so_suffix():
     from imp import get_suffixes, C_EXTENSION
     for suffix, mode, typ in get_suffixes():
@@ -148,6 +138,35 @@
         link_extra=link_extra,
         ext=get_so_suffix())
 
+def make_methods(functions, modname):
+    methods_table = []
+    codes = []
+    for funcname, flags, code in functions:
+        cfuncname = "%s_%s" % (modname, funcname)
+        methods_table.append("{\"%s\", %s, %s}," %
+                                (funcname, cfuncname, flags))
+        func_code = """
+        static PyObject* %s(PyObject* self, PyObject* args)
+        {
+        %s
+        }
+        """ % (cfuncname, code)
+        codes.append(func_code)
+
+    body = "\n".join(codes) + """
+    static PyMethodDef methods[] = {
+    %(methods)s
+    { NULL }
+    };
+    static struct PyModuleDef moduledef = {
+        PyModuleDef_HEAD_INIT,
+        "%(modname)s",  /* m_name */
+        NULL,           /* m_doc */
+        -1,             /* m_size */
+        methods,        /* m_methods */
+    };
+    """ % dict(methods='\n'.join(methods_table), modname=modname)
+    return body
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -168,6 +187,7 @@
         return arg
     listview = passthrough
     str_w = passthrough
+    wrap = passthrough
 
     def unwrap(self, args):
         try:
@@ -308,6 +328,15 @@
         if not cls.runappdirect:
             cls.w_runappdirect = space.wrap(cls.runappdirect)
 
+    def record_imported_module(self, name):
+        """
+        Record a module imported in a test so that it can be cleaned up in
+        teardown before the check for leaks is done.
+
+        name gives the name of the module in the space's sys.modules.
+        """
+        self.imported_module_names.append(name)
+
     def setup_method(self, func):
         @gateway.unwrap_spec(name=str)
         def compile_module(space, name,
@@ -324,16 +353,20 @@
                 source_strings = space.listview_bytes(w_source_strings)
             else:
                 source_strings = None
-            pydname = compile_extension_module(
-                self.sys_info, name,
+            pydname = self.sys_info.compile_extension_module(
+                name,
                 source_files=source_files,
                 source_strings=source_strings)
+
+            # hackish, but tests calling compile_module() always end up
+            # importing the result
+            self.record_imported_module(name)
+
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
-                     load_it=bool, filename='str_or_None',
-                     PY_SSIZE_T_CLEAN=bool)
-        def import_module(space, name, init=None, body='', load_it=True,
+                     filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
+        def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
                           PY_SSIZE_T_CLEAN=False):
             """
@@ -378,69 +411,30 @@
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
                 kwds = dict(source_files=[filename])
-            mod = compile_extension_module(self.sys_info, name,
-                    include_dirs=include_dirs, **kwds)
+            mod = self.sys_info.compile_extension_module(
+                name, include_dirs=include_dirs, **kwds)
+            w_result = load_module(space, mod, name)
+            if not self.runappdirect:
+                self.record_imported_module(name)
+            return w_result
 
-            if load_it:
-                if self.runappdirect:
-                    import imp
-                    return imp.load_dynamic(name, mod)
-                else:
-                    api.load_extension_module(space, mod, name)
-                    self.imported_module_names.append(name)
-                    return space.getitem(
-                        space.sys.get('modules'),
-                        space.wrap(name))
-            else:
-                path = os.path.dirname(mod)
-                if self.runappdirect:
-                    return path
-                else:
-                    return space.wrap(path)
 
         @gateway.unwrap_spec(mod=str, name=str)
-        def reimport_module(space, mod, name):
+        def load_module(space, mod, name):
             if self.runappdirect:
                 import imp
                 return imp.load_dynamic(name, mod)
             else:
                 api.load_extension_module(space, mod, name)
-            return space.getitem(
-                space.sys.get('modules'),
-                space.wrap(name))
+                return space.getitem(
+                    space.sys.get('modules'), space.wrap(name))
 
         @gateway.unwrap_spec(modname=str, prologue=str,
                              more_init=str, PY_SSIZE_T_CLEAN=bool)
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
-            methods_table = []
-            codes = []
-            for funcname, flags, code in functions:
-                cfuncname = "%s_%s" % (modname, funcname)
-                methods_table.append("{\"%s\", %s, %s}," %
-                                     (funcname, cfuncname, flags))
-                func_code = """
-                static PyObject* %s(PyObject* self, PyObject* args)
-                {
-                %s
-                }
-                """ % (cfuncname, code)
-                codes.append(func_code)
-
-            body = prologue + "\n".join(codes) + """
-            static PyMethodDef methods[] = {
-            %(methods)s
-            { NULL }
-            };
-            static struct PyModuleDef moduledef = {
-                PyModuleDef_HEAD_INIT,
-                "%(modname)s",  /* m_name */
-                NULL,           /* m_doc */
-                -1,             /* m_size */
-                methods,        /* m_methods */
-            };
-            """ % dict(methods='\n'.join(methods_table), modname=modname)
+            body = prologue + make_methods(functions, modname)
             init = """PyObject *mod = PyModule_Create(&moduledef);"""
             if more_init:
                 init += more_init
@@ -449,16 +443,6 @@
                                  w_include_dirs=w_include_dirs,
                                  PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
 
-        @gateway.unwrap_spec(name=str)
-        def record_imported_module(name):
-            """
-            Record a module imported in a test so that it can be cleaned up in
-            teardown before the check for leaks is done.
-
-            name gives the name of the module in the space's sys.modules.
-            """
-            self.imported_module_names.append(name)
-
         def debug_collect(space):
             rawrefcount._collect()
 
@@ -479,15 +463,15 @@
             def wrap(func):
                 return func
             self.sys_info = get_sys_info_app()
+            self.compile_module = self.sys_info.compile_extension_module
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
             self.sys_info = get_cpyext_info(self.space)
-        self.w_compile_module = wrap(interp2app(compile_module))
+            self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
-        self.w_reimport_module = wrap(interp2app(reimport_module))
+        self.w_load_module = wrap(interp2app(load_module))
         self.w_import_extension = wrap(interp2app(import_extension))
-        self.w_record_imported_module = wrap(interp2app(record_imported_module))
         self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/')
         self.w_debug_collect = wrap(interp2app(debug_collect))
 
@@ -628,15 +612,11 @@
         If `cherry.date` is an extension module which imports `apple.banana`,
         the latter is added to `sys.modules` for the `"apple.banana"` key.
         """
-        if self.runappdirect:
-            skip('record_imported_module not supported in runappdirect mode')
         # Build the extensions.
         banana = self.compile_module(
             "apple.banana", source_files=[self.here + 'banana.c'])
-        self.record_imported_module("apple.banana")
         date = self.compile_module(
             "cherry.date", source_files=[self.here + 'date.c'])
-        self.record_imported_module("cherry.date")
 
         # Set up some package state so that the extensions can actually be
         # imported.
@@ -648,7 +628,6 @@
         apple.__path__ = [os.path.dirname(banana)]
 
         import cherry.date
-        import apple.banana
 
         assert sys.modules['apple.banana'].__name__ == 'apple.banana'
         assert sys.modules['cherry.date'].__name__ == 'cherry.date'
@@ -1004,7 +983,7 @@
             f.write('not again!\n')
             f.close()
             m1 = sys.modules['foo']
-            m2 = self.reimport_module(m1.__file__, name='foo')
+            m2 = self.load_module(m1.__file__, name='foo')
             assert m1 is m2
             assert m1 is sys.modules['foo']
 
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 
 class TestImport(BaseApiTest):
     def test_import(self, space, api):
@@ -39,9 +39,9 @@
 
 class AppTestImportLogic(AppTestCpythonExtensionBase):
     def test_import_logic(self):
-        path = self.import_module(name='test_import_module', load_it=False)
-        import sys
-        sys.path.append(path)
+        import sys, os
+        path = self.compile_module('test_import_module',
+            source_files=[os.path.join(self.here, 'test_import_module.c')])
+        sys.path.append(os.path.dirname(path))
         import test_import_module
         assert test_import_module.TEST is None
-
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
         assert module.test_fast_sequence(s[0:-1])
         assert module.test_fast_sequence(s[::-1])
 
+    def test_fast_keyerror(self):
+        module = self.import_extension('foo', [
+            ("test_fast_sequence", "METH_VARARGS",
+             """
+                PyObject *foo;
+                PyObject * seq = PyTuple_GetItem(args, 0);
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "Could not convert object to sequence");
+                if (foo != NULL)
+                {
+                    return foo;
+                }
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                    return PyBool_FromLong(1);
+                }
+                return NULL;
+             """)])
+        class Map(object):
+            def __len__(self):
+                return 1
+
+            def __getitem__(self, index):
+                raise KeyError()
+
+        assert module.test_fast_sequence(Map()) is True
+

From pypy.commits at gmail.com  Sun Sep 11 20:30:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 17:30:50 -0700 (PDT)
Subject: [pypy-commit] pypy _py3k-cpyext-A: hack around to get -A tests to
 run
Message-ID: <57d5f73a.0cce1c0a.3080.ed88@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: _py3k-cpyext-A
Changeset: r87013:1d896b40d4f1
Date: 2016-09-12 00:20 +0100
http://bitbucket.org/pypy/pypy/changeset/1d896b40d4f1/

Log:	hack around to get -A tests to run

diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -11,8 +11,6 @@
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
-    if config.option.runappdirect:
-        return True # "cannot be run by py.test -A"
     # ensure additional functions are registered
     import pypy.module.cpyext.test.test_cpyext
     return False
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -327,6 +327,10 @@
         #state.non_heaptypes_w[:] = []
         if not cls.runappdirect:
             cls.w_runappdirect = space.wrap(cls.runappdirect)
+        else:
+            def w_compile_module(self, name, source_files=None, source_strings=None):
+                return self.sys_info(name, source_files, source_strings)
+            cls.w_compile_module = w_compile_module
 
     def record_imported_module(self, name):
         """
@@ -463,7 +467,7 @@
             def wrap(func):
                 return func
             self.sys_info = get_sys_info_app()
-            self.compile_module = self.sys_info.compile_extension_module
+            #self.compile_module = self.sys_info.compile_extension_module
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
@@ -614,9 +618,9 @@
         """
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", source_files=[self.here + 'banana.c'])
+            "apple.banana", source_files=[self.here + b'banana.c'])
         date = self.compile_module(
-            "cherry.date", source_files=[self.here + 'date.c'])
+            "cherry.date", source_files=[self.here + b'date.c'])
 
         # Set up some package state so that the extensions can actually be
         # imported.
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -121,6 +121,7 @@
         elif isinstance(value, types.MethodType):
             # "def w_method(self)"
             code = py.code.Code(value)
+            #import pdb; pdb.set_trace()
             defs.append(str(code.source()))
             defs.append("type(self).%s = %s\n" % (symbol, value.__name__))
         elif isinstance(value, types.ModuleType):
@@ -149,7 +150,7 @@
                     missing.remove(name)
             del name
         if missing:
-            sys.exit(81)
+            pass #sys.exit(81)
     del missing""" % usemodules
 
     source = list(py.code.Source(target_))

From pypy.commits at gmail.com  Sun Sep 11 20:30:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 11 Sep 2016 17:30:52 -0700 (PDT)
Subject: [pypy-commit] pypy _py3k-cpyext-A: hack some more: one test passes!
Message-ID: <57d5f73c.411d1c0a.99c34.fa97@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: _py3k-cpyext-A
Changeset: r87014:a0993b1fb7ba
Date: 2016-09-12 01:29 +0100
http://bitbucket.org/pypy/pypy/changeset/a0993b1fb7ba/

Log:	hack some more: one test passes!

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -7,7 +7,6 @@
 from pypy import pypydir
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
-from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
 from pypy.module.cpyext import api
 from pypy.module.cpyext.state import State
@@ -16,7 +15,7 @@
 from rpython.tool import leakfinder
 from rpython.rlib import rawrefcount
 
-from .support import c_compile
+from pypy.tool.cpyext.buildext import SystemCompilationInfo, get_sys_info_app
 
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
@@ -33,55 +32,6 @@
         assert 'PyModule_Check' in api.FUNCTIONS
         assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
 
-def convert_sources_to_files(sources, dirname):
-    files = []
-    for i, source in enumerate(sources):
-        filename = dirname / ('source_%d.c' % i)
-        with filename.open('w') as f:
-            f.write(str(source))
-        files.append(filename)
-    return files
-
-class SystemCompilationInfo(object):
-    """Bundles all the generic information required to compile extensions.
-
-    Note: here, 'system' means OS + target interpreter + test config + ...
-    """
-    def __init__(self, include_extra=None, compile_extra=None, link_extra=None,
-            extra_libs=None, ext=None):
-        self.include_extra = include_extra or []
-        self.compile_extra = compile_extra
-        self.link_extra = link_extra
-        self.extra_libs = extra_libs
-        self.ext = ext
-
-    def compile_extension_module(self, name, include_dirs=[],
-            source_files=None, source_strings=None):
-        """
-        Build an extension module and return the filename of the resulting
-        native code file.
-
-        name is the name of the module, possibly including dots if it is a
-        module inside a package.
-
-        Any extra keyword arguments are passed on to ExternalCompilationInfo to
-        build the module (so specify your source with one of those).
-        """
-        modname = name.split('.')[-1]
-        dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-        if source_strings:
-            assert not source_files
-            files = convert_sources_to_files(source_strings, dirname)
-            source_files = files
-        soname = c_compile(source_files, outputfilename=str(dirname/modname),
-            compile_extra=self.compile_extra,
-            link_extra=self.link_extra,
-            include_dirs=self.include_extra + include_dirs,
-            libraries=self.extra_libs)
-        pydname = soname.new(purebasename=modname, ext=self.ext)
-        soname.rename(pydname)
-        return str(pydname)
-
 def get_cpyext_info(space):
     from pypy.module.imp.importing import get_so_extension
     state = space.fromcache(State)
@@ -110,34 +60,6 @@
         ext=get_so_extension(space))
 
 
-def get_so_suffix():
-    from imp import get_suffixes, C_EXTENSION
-    for suffix, mode, typ in get_suffixes():
-        if typ == C_EXTENSION:
-            return suffix
-    else:
-        raise RuntimeError("This interpreter does not define a filename "
-            "suffix for C extensions!")
-
-def get_sys_info_app():
-    from distutils.sysconfig import get_python_inc
-    if sys.platform == 'win32':
-        compile_extra = ["/we4013"]
-        link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
-    elif sys.platform == 'darwin':
-        compile_extra = link_extra = None
-        pass
-    elif sys.platform.startswith('linux'):
-        compile_extra = [
-            "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
-        link_extra = None
-    ext = get_so_suffix()
-    return SystemCompilationInfo(
-        include_extra=[get_python_inc()],
-        compile_extra=compile_extra,
-        link_extra=link_extra,
-        ext=get_so_suffix())
-
 def make_methods(functions, modname):
     methods_table = []
     codes = []
@@ -329,7 +251,11 @@
             cls.w_runappdirect = space.wrap(cls.runappdirect)
         else:
             def w_compile_module(self, name, source_files=None, source_strings=None):
-                return self.sys_info(name, source_files, source_strings)
+                from buildext import get_sys_info_app
+                sys_info = get_sys_info_app()
+                assert source_files or source_strings
+                return sys_info.compile_extension_module(name,
+                    source_files=source_files, source_strings=source_strings)
             cls.w_compile_module = w_compile_module
 
     def record_imported_module(self, name):
@@ -618,9 +544,9 @@
         """
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", source_files=[self.here + b'banana.c'])
+            "apple.banana", source_files=[self.here.decode() + 'banana.c'])
         date = self.compile_module(
-            "cherry.date", source_files=[self.here + b'date.c'])
+            "cherry.date", source_files=[self.here.decode() + 'date.c'])
 
         # Set up some package state so that the extensions can actually be
         # imported.
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -76,8 +76,8 @@
     def skip(message):
         print(message)
         raise SystemExit(0)
-    __builtins__.skip = skip
-    __builtins__.py3k_skip = skip
+    #__builtins__.skip = skip
+    #__builtins__.py3k_skip = skip
     class ExceptionWrapper:
         pass
     def raises(exc, func, *args, **kwargs):
@@ -98,7 +98,7 @@
             return res
         else:
             raise AssertionError("DID NOT RAISE")
-    __builtins__.raises = raises
+    #__builtins__.raises = raises
     class Test:
         pass
     self = Test()
@@ -170,8 +170,11 @@
         f.write('def %s():\n' % target_name)
         f.write('\n'.join(source))
         f.write("\n%s()\n" % target_name)
+    helper_dir = os.path.join(pypydir, 'tool', 'cpyext')
+    env = os.environ.copy()
+    env['PYTHONPATH'] = helper_dir
     res, stdout, stderr = runsubprocess.run_subprocess(
-        python_, [str(pyfile)])
+        python_, [str(pyfile)], env=env)
     print pyfile.read()
     print >> sys.stdout, stdout
     print >> sys.stderr, stderr
@@ -325,4 +328,3 @@
                                           space.newtuple([]),
                                           space.newdict())
         self.w_class = w_class
-
diff --git a/rpython/tool/udir.py b/rpython/tool/udir.py
--- a/rpython/tool/udir.py
+++ b/rpython/tool/udir.py
@@ -21,7 +21,11 @@
 import sys
 
 from rpython.tool.version import get_repo_version_info
-from py.path import local 
+from py.path import local
+try:
+    unicode
+except NameError:
+    class unicode: pass
 
 PYPY_KEEP = int(os.environ.get('PYPY_USESSION_KEEP', '3'))
 
diff --git a/rpython/tool/version.py b/rpython/tool/version.py
--- a/rpython/tool/version.py
+++ b/rpython/tool/version.py
@@ -16,6 +16,7 @@
 
 def get_repo_version_info(hgexe=None, root=rpythonroot):
     '''Obtain version information by invoking the 'hg' or 'git' commands.'''
+    return default_retval
 
     # Try to see if we can get info from Git if hgexe is not specified.
     if not hgexe:
@@ -137,4 +138,4 @@
 
 
 if __name__ == '__main__':
-    print get_repo_version_info()
+    print(get_repo_version_info())

From pypy.commits at gmail.com  Mon Sep 12 03:41:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 00:41:08 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: kill import
Message-ID: <57d65c14.4398c20a.783a0.e452@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87017:864a4750a84f
Date: 2016-09-12 09:40 +0200
http://bitbucket.org/pypy/pypy/changeset/864a4750a84f/

Log:	kill import

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -384,8 +384,6 @@
                if y.field:                 r = y.field
                    return y.field          if r: return r
         """
-        from rpython.jit.metainterp.history import getkind
-
         block = graph.startblock
         if len(block.operations) == 0:
             return

From pypy.commits at gmail.com  Mon Sep 12 04:08:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 01:08:26 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Fixes, more cases
Message-ID: <57d6627a.0575c20a.8f0a7.6034@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87018:cdb2025d2457
Date: 2016-09-12 10:07 +0200
http://bitbucket.org/pypy/pypy/changeset/cdb2025d2457/

Log:	Fixes, more cases

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -10,6 +10,7 @@
     EffectInfo, CallInfoCollection, CallShortcut)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
+from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -391,6 +392,8 @@
         if op.opname != 'getfield':
             return
         [v_inst, c_fieldname] = op.args
+        if not isinstance(v_inst, Variable):
+            return
         v_result = op.result
         if v_result.concretetype != graph.getreturnvar().concretetype:
             return
@@ -405,12 +408,19 @@
         v_check = v_result
         fastcase = True
         for op in block.operations[1:]:
-            if ((op.opname == 'int_is_true' or op.opname == 'ptr_nonzero')
+            if (op.opname in ('int_is_true', 'ptr_nonzero')
                     and v_check is op.args[0]):
                 v_check = op.result
             elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
                 v_check = op.result
                 fastcase = not fastcase
+            elif (op.opname in ('int_eq', 'int_ne')
+                    and v_check is op.args[0]
+                    and isinstance(op.args[1], Constant)
+                    and op.args[1].value == 0):
+                v_check = op.result
+                if op.opname == 'int_eq':
+                    fastcase = not fastcase
             else:
                 return
         if v_check.concretetype is not lltype.Bool:
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -372,12 +372,14 @@
 def test_find_call_shortcut():
     class FakeCPU:
         def fielddescrof(self, TYPE, fieldname):
-            assert isinstance(TYPE, lltype.GcStruct)
-            if fieldname == 'inst_foobar':
-                return 'foobardescr'
-            if fieldname == 'inst_fooref':
-                return 'foorefdescr'
-            assert False, fieldname
+            if isinstance(TYPE, lltype.GcStruct):
+                if fieldname == 'inst_foobar':
+                    return 'foobardescr'
+                if fieldname == 'inst_fooref':
+                    return 'foorefdescr'
+            if TYPE == RAW and fieldname == 'x':
+                return 'xdescr'
+            assert False, (TYPE, fieldname)
     cc = CallControl(FakeCPU())
 
     class B(object):
@@ -408,11 +410,21 @@
             r = b.foobar = 123
         return r
 
+    def f4(raw):
+        r = raw.x
+        if r != 0:
+            return r
+        raw.x = 123
+        return 123
+    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
+
     def f(a, c):
         b = B()
         f1(a, b, c)
         f2(a, c, a, b)
         f3(space, b)
+        r = lltype.malloc(RAW, flavor='raw')
+        f4(r)
 
     rtyper = support.annotate(f, [10, 20])
     f1_graph = rtyper.annotator.translator._graphof(f1)
@@ -421,6 +433,8 @@
     assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
     f3_graph = rtyper.annotator.translator._graphof(f3)
     assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
+    f4_graph = rtyper.annotator.translator._graphof(f4)
+    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
 
 def test_cant_find_call_shortcut():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU

From pypy.commits at gmail.com  Mon Sep 12 04:41:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 01:41:57 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Rewrite: expand to
 cond_call_value
Message-ID: <57d66a55.0205c20a.8950.74ec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87019:377865d888be
Date: 2016-09-12 10:41 +0200
http://bitbucket.org/pypy/pypy/changeset/377865d888be/

Log:	Rewrite: expand to cond_call_value

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -11,7 +11,7 @@
 from rpython.jit.backend.llsupport.symbolic import (WORD,
         get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER
+     FLAG_POINTER, CallDescr
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -370,7 +370,9 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- call assembler -----------
+            # ---------- calls -----------
+            if OpHelpers.is_plain_call(op.getopnum()):
+                self.expand_call_shortcut(op)
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -616,6 +618,28 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
+    def expand_call_shortcut(self, op):
+        descr = op.getdescr()
+        if descr is None:
+            return
+        assert isinstance(descr, CallDescr)
+        effectinfo = descr.get_extra_info()
+        if effectinfo is None or effectinfo.call_shortcut is None:
+            return
+        if op.type == 'r':
+            cond_call_opnum = rop.COND_CALL_VALUE_R
+        elif op.type == 'i':
+            cond_call_opnum = rop.COND_CALL_VALUE_I
+        else:
+            return
+        cs = effectinfo.call_shortcut
+        ptr_box = op.getarg(1 + cs.argnum)
+        value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
+                                       raw=(ptr_box.type == 'i'))
+        self.replace_op_with(op, ResOperation(cond_call_opnum,
+                                              [value_box] + op.getarglist(),
+                                              descr=descr))
+
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,7 +1,8 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr
+     SizeDescr, get_interiorfield_descr, get_call_descr
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -80,6 +81,14 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
+        call_shortcut = CallShortcut(0, tzdescr)
+        effectinfo = EffectInfo(None, None, None, None, None, None,
+                                EffectInfo.EF_RANDOM_EFFECTS,
+                                call_shortcut=call_shortcut)
+        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
+            [lltype.Ptr(T)], lltype.Signed,
+            effectinfo)
+        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -1429,3 +1438,15 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
+
+    def test_handle_call_shortcut(self):
+        self.check_rewrite("""
+            [p0]
+            i1 = call_i(123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """, """
+            [p0]
+            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
+            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -366,6 +366,8 @@
                          rop.CALL_ASSEMBLER_I,
                          rop.CALL_ASSEMBLER_N,
                          rop.INCREMENT_DEBUG_COUNTER,
+                         rop.COND_CALL_VALUE_R,
+                         rop.COND_CALL_VALUE_I,
                          rop.COND_CALL_GC_WB,
                          rop.COND_CALL_GC_WB_ARRAY,
                          rop.ZERO_ARRAY,
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1150,6 +1150,7 @@
     '_CALL_FIRST',
     'CALL/*d/rfin',
     'COND_CALL/*d/n',
+    'COND_CALL_VALUE/*d/ri',
     # a conditional call, with first argument as a condition
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',

From pypy.commits at gmail.com  Mon Sep 12 05:06:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 02:06:36 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: a runner_test for
 cond_call_value
Message-ID: <57d6701c.d7bb1c0a.9ab4d.7d57@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87020:7c6e114509d2
Date: 2016-09-12 11:05 +0200
http://bitbucket.org/pypy/pypy/changeset/7c6e114509d2/

Log:	a runner_test for cond_call_value

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -1334,6 +1334,16 @@
         # cond_call can't have a return value
         self.execute_call_n(calldescr, func, *args)
 
+    def execute_cond_call_value_i(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_i(calldescr, func, *args)
+        return value
+
+    def execute_cond_call_value_r(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_r(calldescr, func, *args)
+        return value
+
     def _execute_call(self, calldescr, func, *args):
         effectinfo = calldescr.get_extra_info()
         if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'):
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2389,7 +2389,7 @@
             f2 = longlong.getfloatstorage(3.4)
             frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5, f1, f2)
             assert not called
-            for j in range(5):
+            for j in range(6):
                 assert self.cpu.get_int_value(frame, j) == j
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 6)) == 1.2
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 3.4
@@ -2447,6 +2447,51 @@
                                            67, 89)
             assert called == [(67, 89)]
 
+    def test_cond_call_value(self):
+        def func_int(*args):
+            called.append(args)
+            return len(args) * 100 + 1000
+
+        for i in range(5):
+            called = []
+
+            FUNC = self.FuncType([lltype.Signed] * i, lltype.Signed)
+            func_ptr = llhelper(lltype.Ptr(FUNC), func_int)
+            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                             EffectInfo.MOST_GENERAL)
+
+            ops = '''
+            [i0, i1, i2, i3, i4, i5, i6, f0, f1]
+            i15 = cond_call_value_i(i1, ConstClass(func_ptr), %s)
+            guard_false(i0, descr=faildescr) [i1,i2,i3,i4,i5,i6,i15, f0,f1]
+            finish(i15)
+            ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] +
+                            ["descr=calldescr"])
+            loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
+                                         'func_ptr': func_ptr,
+                                         'calldescr': calldescr})
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+            f1 = longlong.getfloatstorage(1.2)
+            f2 = longlong.getfloatstorage(3.4)
+            frame = self.cpu.execute_token(looptoken, 1, 50, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert not called
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        50, 1, 2, 3, 4, 5, 50]
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 8)) == 3.4
+            #
+            frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert called == [(1, 2, 3, 4)[:i]]
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        0, 1, 2, 3, 4, 5, i * 100 + 1000]
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 8)) == 3.4
+
     def test_force_operations_returning_void(self):
         values = []
         def maybe_force(token, flag):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1149,9 +1149,8 @@
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
     'CALL/*d/rfin',
-    'COND_CALL/*d/n',
-    'COND_CALL_VALUE/*d/ri',
-    # a conditional call, with first argument as a condition
+    'COND_CALL/*d/n',   # a conditional call, with first argument as a condition
+    'COND_CALL_VALUE/*d/ri',  # same but returns a result; emitted by rewrite
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',
     'CALL_LOOPINVARIANT/*d/rfin',

From pypy.commits at gmail.com  Mon Sep 12 05:09:32 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 12 Sep 2016 02:09:32 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase
Message-ID: <57d670cc.081dc20a.434ed.7dc4@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87021:636d1082681c
Date: 2016-09-12 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/636d1082681c/

Log:	Update qcgc codebase

diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
--- a/rpython/translator/c/src/qcgc/allocator.c
+++ b/rpython/translator/c/src/qcgc/allocator.c
@@ -215,7 +215,7 @@
 
 object_t *qcgc_fit_allocate(size_t bytes) {
 #if CHECKED
-	free_list_consistency_check();
+	//free_list_consistency_check();
 #endif
 	size_t cells = bytes_to_cells(bytes);
 	cell_t *mem;
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -13,8 +13,7 @@
 #define LOG_ALLOCATION 0					// Enable allocation log (warning:
 											// significant performance impact)
 
-#define QCGC_SHADOWSTACK_SIZE 128			// Number of initial entries for
-											// shadow stack
+#define QCGC_SHADOWSTACK_SIZE 163840		// Total shadowstack size
 #define QCGC_ARENA_BAG_INIT_SIZE 16			// Initial size of the arena bag
 #define QCGC_ARENA_SIZE_EXP 20				// Between 16 (64kB) and 20 (1MB)
 #define QCGC_LARGE_ALLOC_THRESHOLD_EXP 14	// Less than QCGC_ARENA_SIZE_EXP
@@ -52,3 +51,4 @@
 
 #define MAX(a,b) (((a)>(b))?(a):(b))
 #define MIN(a,b) (((a)<(b))?(a):(b))
+#define UNUSED(x) (void)(x)
diff --git a/rpython/translator/c/src/qcgc/gc_state.h b/rpython/translator/c/src/qcgc/gc_state.h
--- a/rpython/translator/c/src/qcgc/gc_state.h
+++ b/rpython/translator/c/src/qcgc/gc_state.h
@@ -25,7 +25,8 @@
  * Global state of the garbage collector
  */
 struct qcgc_state {
-	shadow_stack_t *shadow_stack;
+	object_t **shadow_stack;
+	object_t **shadow_stack_base;
 	shadow_stack_t *prebuilt_objects;
 	weakref_bag_t *weakrefs;
 	gray_stack_t *gp_gray_stack;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -5,10 +5,12 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <sys/mman.h>
 
 #include "allocator.h"
+#include "event_logger.h"
 #include "hugeblocktable.h"
-#include "event_logger.h"
+#include "signal_handler.h"
 
 #define env_or_fallback(var, env_name, fallback) while(0) {		\
 	char *env_val = getenv(env_name);							\
@@ -28,9 +30,11 @@
 static size_t incmark_threshold = QCGC_INCMARK_THRESHOLD;
 
 QCGC_STATIC void update_weakrefs(void);
+QCGC_STATIC void initialize_shadowstack(void);
+QCGC_STATIC void destroy_shadowstack(void);
 
 void qcgc_initialize(void) {
-	qcgc_state.shadow_stack = qcgc_shadow_stack_create(QCGC_SHADOWSTACK_SIZE);
+	initialize_shadowstack();
 	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); // XXX
 	qcgc_state.weakrefs = qcgc_weakref_bag_create(16); // XXX
 	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
@@ -47,13 +51,15 @@
 	env_or_fallback(major_collection_threshold, "QCGC_MAJOR_COLLECTION",
 			QCGC_MAJOR_COLLECTION_THRESHOLD);
 	env_or_fallback(incmark_threshold, "QCGC_INCMARK", QCGC_INCMARK_THRESHOLD);
+
+	setup_signal_handler();
 }
 
 void qcgc_destroy(void) {
 	qcgc_event_logger_destroy();
 	qcgc_hbtable_destroy();
 	qcgc_allocator_destroy();
-	free(qcgc_state.shadow_stack);
+	destroy_shadowstack();
 	free(qcgc_state.prebuilt_objects);
 	free(qcgc_state.weakrefs);
 	free(qcgc_state.gp_gray_stack);
@@ -67,14 +73,14 @@
 		qcgc_state.phase = GC_MARK;
 		qcgc_push_object(object);
 	}
-	qcgc_state.shadow_stack =
-		qcgc_shadow_stack_push(qcgc_state.shadow_stack, object);
+
+	*qcgc_state.shadow_stack = object;
+	qcgc_state.shadow_stack++;
 }
 
 object_t *qcgc_shadowstack_pop(void) {
-	object_t *result = qcgc_shadow_stack_top(qcgc_state.shadow_stack);
-	qcgc_state.shadow_stack = qcgc_shadow_stack_pop(qcgc_state.shadow_stack);
-	return result;
+	qcgc_state.shadow_stack--;
+	return *qcgc_state.shadow_stack;
 }
 
 /*******************************************************************************
@@ -225,8 +231,10 @@
 		// If we do this for the first time, push all roots.
 		// All further changes to the roots (new additions) will be added
 		// by qcgc_shadowstack_push
-		for (size_t i = 0; i < qcgc_state.shadow_stack->count; i++) {
-			qcgc_push_object(qcgc_state.shadow_stack->items[i]);
+		for (object_t **it = qcgc_state.shadow_stack_base;
+			it < qcgc_state.shadow_stack;
+			it++) {
+			qcgc_push_object(*it);
 		}
 
 		// If we do this for the first time, push all prebuilt objects.
@@ -483,3 +491,28 @@
 		}
 	}
 }
+
+QCGC_STATIC void *trap_page_addr(object_t **shadow_stack) {
+	object_t **shadow_stack_end = shadow_stack + QCGC_SHADOWSTACK_SIZE;
+	char *in_trap_page = (((char *)shadow_stack_end) + 4095);
+	void *rounded_trap_page = (void *)(((uintptr_t)in_trap_page) & (~4095));
+	return rounded_trap_page;
+}
+
+QCGC_STATIC void initialize_shadowstack(void) {
+	size_t stack_size = QCGC_SHADOWSTACK_SIZE * sizeof(object_t *);
+	// allocate stack + size for alignement + trap page
+	object_t **stack = (object_t **) malloc(stack_size + 8192);
+	assert(stack != NULL);
+	mprotect(trap_page_addr(stack), 4096, PROT_NONE);
+
+	qcgc_state.shadow_stack = stack;
+	qcgc_state.shadow_stack_base = stack;
+}
+
+QCGC_STATIC void destroy_shadowstack(void) {
+	mprotect(trap_page_addr(qcgc_state.shadow_stack_base), 4096, PROT_READ |
+				PROT_WRITE);
+
+	free(qcgc_state.shadow_stack_base);
+}
diff --git a/rpython/translator/c/src/qcgc/signal_handler.c b/rpython/translator/c/src/qcgc/signal_handler.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/signal_handler.c
@@ -0,0 +1,61 @@
+#include "signal_handler.h"
+
+#include <signal.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "arena.h"
+#include "allocator.h"
+#include "gc_state.h"
+
+QCGC_STATIC void handle_error(int signo, siginfo_t *siginfo, void *context);
+QCGC_STATIC bool is_stack_overflow(void *addr);
+QCGC_STATIC bool is_in_arena(void *addr);
+
+void setup_signal_handler(void) {
+	struct sigaction sa;
+
+	sa.sa_handler = NULL;
+	sigemptyset(&sa.sa_mask);
+	sa.sa_flags = SA_SIGINFO | SA_NODEFER;
+	sa.sa_sigaction = &handle_error;
+
+	if (sigaction(SIGSEGV, &sa, NULL) < 0) {
+		perror("sigaction");
+		abort();
+	}
+}
+
+QCGC_STATIC void handle_error(int signo, siginfo_t *siginfo, void *context) {
+	UNUSED(signo);
+	UNUSED(context);
+
+	if (is_stack_overflow(siginfo->si_addr)) {
+		fprintf(stderr, "Stack overflow: Too many root objects\n");
+	} else if (is_in_arena(siginfo->si_addr)) {
+		fprintf(stderr, "Internal segmentation fault: accessing %p\n",
+				siginfo->si_addr);
+	} else {
+		fprintf(stderr, "External segmentation fault: accessing %p\n",
+				siginfo->si_addr);
+	}
+	exit(EXIT_FAILURE);
+}
+
+QCGC_STATIC bool is_stack_overflow(void *addr) {
+	void *shadow_stack_end = (void *)(qcgc_state.shadow_stack_base +
+		QCGC_SHADOWSTACK_SIZE);
+	return (addr >= shadow_stack_end && addr < shadow_stack_end + 8192);
+}
+
+QCGC_STATIC bool is_in_arena(void *addr) {
+	arena_t *arena = qcgc_arena_addr((cell_t *) addr);
+	size_t count = qcgc_allocator_state.arenas->count;
+	for (size_t i = 0; i < count; i++) {
+		if (arena == qcgc_allocator_state.arenas->items[i]) {
+			return true;
+		}
+	}
+	return false;
+}
diff --git a/rpython/translator/c/src/qcgc/signal_handler.h b/rpython/translator/c/src/qcgc/signal_handler.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/signal_handler.h
@@ -0,0 +1,5 @@
+#pragma once
+
+#include "config.h"
+
+void setup_signal_handler(void);

From pypy.commits at gmail.com  Mon Sep 12 05:14:45 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 12 Sep 2016 02:14:45 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Fix compilation rule
Message-ID: <57d67205.0e101c0a.7be55.c83d@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87022:b2518a7b0fa2
Date: 2016-09-12 11:14 +0200
http://bitbucket.org/pypy/pypy/changeset/b2518a7b0fa2/

Log:	Fix compilation rule

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -899,7 +899,8 @@
             separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
                 ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
-                    "gray_stack.c", "shadow_stack.c", "hugeblocktable.c"]],
+                    "gray_stack.c", "shadow_stack.c", "hugeblocktable.c",
+                    "signal_handler.c"]],
             )
     return eci
 

From pypy.commits at gmail.com  Mon Sep 12 05:15:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 02:15:58 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Implement
 COND_CALL_VALUE (code copied from conditional_call_value_2)
Message-ID: <57d6724e.a3adc20a.3122b.90fb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87023:503727324fab
Date: 2016-09-12 11:15 +0200
http://bitbucket.org/pypy/pypy/changeset/503727324fab/

Log:	Implement COND_CALL_VALUE (code copied from
	conditional_call_value_2)

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -174,8 +174,8 @@
         # copy registers to the frame, with the exception of the
         # 'cond_call_register_arguments' and eax, because these have already
         # been saved by the caller.  Note that this is not symmetrical:
-        # these 5 registers are saved by the caller but restored here at
-        # the end of this function.
+        # these 5 registers are saved by the caller but 4 of them are
+        # restored here at the end of this function.
         self._push_all_regs_to_frame(mc, cond_call_register_arguments + [eax],
                                      supports_floats, callee_only)
         # the caller already did push_gcmap(store=True)
@@ -198,7 +198,7 @@
             mc.ADD(esp, imm(WORD * 7))
         self.set_extra_stack_depth(mc, 0)
         self.pop_gcmap(mc)   # cancel the push_gcmap(store=True) in the caller
-        self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only)
+        self._pop_all_regs_from_frame(mc, [eax], supports_floats, callee_only)
         mc.RET()
         return mc.materialize(self.cpu, [])
 
@@ -1703,7 +1703,8 @@
         self.implement_guard(guard_token)
         # If the previous operation was a COND_CALL, overwrite its conditional
         # jump to jump over this GUARD_NO_EXCEPTION as well, if we can
-        if self._find_nearby_operation(-1).getopnum() == rop.COND_CALL:
+        if self._find_nearby_operation(-1).getopnum() in (
+                rop.COND_CALL, rop.COND_CALL_VALUE_I, rop.COND_CALL_VALUE_R):
             jmp_adr = self.previous_cond_call_jcond
             offset = self.mc.get_relative_pos() - jmp_adr
             if offset <= 127:
@@ -2381,7 +2382,7 @@
     def label(self):
         self._check_frame_depth_debug(self.mc)
 
-    def cond_call(self, op, gcmap, imm_func, arglocs):
+    def cond_call(self, op, gcmap, imm_func, arglocs, resloc=None):
         assert self.guard_success_cc >= 0
         self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0)
                                                             # patched later
@@ -2394,11 +2395,14 @@
         # plus the register 'eax'
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         should_be_saved = self._regalloc.rm.reg_bindings.values()
+        restore_eax = False
         for gpr in cond_call_register_arguments + [eax]:
-            if gpr not in should_be_saved:
+            if gpr not in should_be_saved or gpr is resloc:
                 continue
             v = gpr_reg_mgr_cls.all_reg_indexes[gpr.value]
             self.mc.MOV_br(v * WORD + base_ofs, gpr.value)
+            if gpr is eax:
+                restore_eax = True
         #
         # load the 0-to-4 arguments into these registers
         from rpython.jit.backend.x86.jump import remap_frame_layout
@@ -2422,8 +2426,16 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.CALL(imm(follow_jump(cond_call_adr)))
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        if resloc is not None and resloc is not eax:
+            self.mc.MOV(resloc, eax)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  We must only restore eax, if
+        # needed.
+        if restore_eax:
+            v = gpr_reg_mgr_cls.all_reg_indexes[eax.value]
+            self.mc.MOV_rb(eax.value, v * WORD + base_ofs)
+        #
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -938,16 +938,28 @@
                     self.rm.force_spill_var(box)
                     assert box not in self.rm.reg_bindings
         #
-        assert op.type == 'v'
         args = op.getarglist()
         assert 2 <= len(args) <= 4 + 2     # maximum 4 arguments
-        v = args[1]
-        assert isinstance(v, Const)
-        imm_func = self.rm.convert_to_imm(v)
+        v_func = args[1]
+        assert isinstance(v_func, Const)
+        imm_func = self.rm.convert_to_imm(v_func)
         arglocs = [self.loc(args[i]) for i in range(2, len(args))]
         gcmap = self.get_gcmap()
-        self.load_condition_into_cc(op.getarg(0))
-        self.assembler.cond_call(op, gcmap, imm_func, arglocs)
+        if op.type == 'v':
+            # a plain COND_CALL
+            self.load_condition_into_cc(op.getarg(0))
+            resloc = None
+        else:
+            # COND_CALL_VALUE_I/R
+            condvalue_loc = self.loc(args[0])
+            assert not isinstance(condvalue_loc, ImmedLoc)
+            self.assembler.test_location(condvalue_loc)
+            self.assembler.guard_success_cc = rx86.Conditions['Z']
+            resloc = self.rm.force_result_in_reg(op, args[0])
+        self.assembler.cond_call(op, gcmap, imm_func, arglocs, resloc)
+
+    consider_cond_call_value_i = consider_cond_call
+    consider_cond_call_value_r = consider_cond_call
 
     def consider_call_malloc_nursery(self, op):
         size_box = op.getarg(0)

From pypy.commits at gmail.com  Mon Sep 12 05:33:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 02:33:00 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Random tests for
 cond_call_value.
Message-ID: <57d6764c.87adc20a.db50d.8d06@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87024:4eff2cbf173d
Date: 2016-09-12 11:31 +0200
http://bitbucket.org/pypy/pypy/changeset/4eff2cbf173d/

Log:	Random tests for cond_call_value.

diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -594,7 +594,7 @@
         return subset, d['f'], vtableptr
 
     def getresulttype(self):
-        if self.opnum == rop.CALL_I:
+        if self.opnum == rop.CALL_I or self.opnum == rop.COND_CALL_VALUE_I:
             return lltype.Signed
         elif self.opnum == rop.CALL_F:
             return lltype.Float
@@ -709,10 +709,13 @@
         builder.loop.operations.append(op)
 
 # 6. a conditional call (for now always with no exception raised)
-class CondCallOperation(BaseCallOperation):
+class BaseCondCallOperation(BaseCallOperation):
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
-        v_cond = builder.get_bool_var(r)
+        if self.opnum == rop.COND_CALL:
+            v_cond = builder.get_bool_var(r)
+        else:
+            v_cond = r.choice(builder.intvars)
         subset = builder.subset_of_intvars(r)[:4]
         for i in range(len(subset)):
             if r.random() < 0.35:
@@ -724,8 +727,10 @@
                 seen.append(args)
             else:
                 assert seen[0] == args
+            if self.RESULT_TYPE is lltype.Signed:
+                return len(args) - 42000
         #
-        TP = lltype.FuncType([lltype.Signed] * len(subset), lltype.Void)
+        TP = lltype.FuncType([lltype.Signed] * len(subset), self.RESULT_TYPE)
         ptr = llhelper(lltype.Ptr(TP), call_me)
         c_addr = ConstAddr(llmemory.cast_ptr_to_adr(ptr), builder.cpu)
         args = [v_cond, c_addr] + subset
@@ -736,6 +741,14 @@
         op.setfailargs(fail_subset)
         builder.loop.operations.append(op)
 
+class CondCallOperation(BaseCondCallOperation):
+    RESULT_TYPE = lltype.Void
+    opnum = rop.COND_CALL
+
+class CondCallValueOperation(BaseCondCallOperation):
+    RESULT_TYPE = lltype.Signed
+    opnum = rop.COND_CALL_VALUE_I
+
 # ____________________________________________________________
 
 OPERATIONS = test_random.OPERATIONS[:]
@@ -769,6 +782,7 @@
 for i in range(2):
     OPERATIONS.append(GuardClassOperation(rop.GUARD_CLASS))
     OPERATIONS.append(CondCallOperation(rop.COND_CALL))
+    OPERATIONS.append(CondCallValueOperation(rop.COND_CALL_VALUE_I))
     OPERATIONS.append(RaisingCallOperation(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationGuardNoException(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationWrongGuardException(rop.CALL_N))
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -946,11 +946,17 @@
         arglocs = [self.loc(args[i]) for i in range(2, len(args))]
         gcmap = self.get_gcmap()
         if op.type == 'v':
-            # a plain COND_CALL
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
             self.load_condition_into_cc(op.getarg(0))
             resloc = None
         else:
-            # COND_CALL_VALUE_I/R
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
             condvalue_loc = self.loc(args[0])
             assert not isinstance(condvalue_loc, ImmedLoc)
             self.assembler.test_location(condvalue_loc)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -101,6 +101,18 @@
     if condbox.getint():
         do_call_n(cpu, metainterp, argboxes[1:], descr)
 
+def do_cond_call_value_i(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getint()
+    if value == 0:
+        value = do_call_i(cpu, metainterp, argboxes[1:], descr)
+    return value
+
+def do_cond_call_value_r(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getref_base()
+    if not value:
+        value = do_call_r(cpu, metainterp, argboxes[1:], descr)
+    return value
+
 def do_getarrayitem_gc_i(cpu, _, arraybox, indexbox, arraydescr):
     array = arraybox.getref_base()
     index = indexbox.getint()

From pypy.commits at gmail.com  Mon Sep 12 05:36:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 02:36:36 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Use @jit.call_shortcut
 here.
Message-ID: <57d67724.811f1c0a.6a776.8356@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87025:3f78315bc18d
Date: 2016-09-12 11:35 +0200
http://bitbucket.org/pypy/pypy/changeset/3f78315bc18d/

Log:	Use @jit.call_shortcut here.

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -369,13 +369,19 @@
         return b
 
     @staticmethod
+    def ll_strhash(s):
+        if s:
+            return LLHelpers._ll_strhash(s)
+        else:
+            return 0
+
+    @staticmethod
     @jit.elidable
-    def ll_strhash(s):
+    @jit.call_shortcut
+    def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the
         # special non-computed-yet value.
-        if not s:
-            return 0
         x = s.hash
         if x == 0:
             x = _hash_string(s.chars)

From pypy.commits at gmail.com  Mon Sep 12 05:37:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 02:37:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: Clean up some variants of
 'conditional_call_*' that are rarely or never
Message-ID: <57d6773f.c50e1c0a.65a6b.8c8d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87026:6de307feaa03
Date: 2016-09-12 11:16 +0200
http://bitbucket.org/pypy/pypy/changeset/6de307feaa03/

Log:	Clean up some variants of 'conditional_call_*' that are rarely or
	never used. Keep only the 'ir' variant.

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -364,7 +364,7 @@
         return getattr(self, 'handle_%s_indirect_call' % kind)(op)
 
     def rewrite_call(self, op, namebase, initialargs, args=None,
-                     calldescr=None):
+                     calldescr=None, force_ir=False):
         """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)'
            into 'i0 = xxx_call_ir_i(fn, descr, [i1,i2], [ref1,ref2])'.
            The name is one of '{residual,direct}_call_{r,ir,irf}_{i,r,f,v}'."""
@@ -374,8 +374,9 @@
         lst_i, lst_r, lst_f = self.make_three_lists(args)
         reskind = getkind(op.result.concretetype)[0]
         if lst_f or reskind == 'f': kinds = 'irf'
-        elif lst_i: kinds = 'ir'
+        elif lst_i or force_ir: kinds = 'ir'
         else: kinds = 'r'
+        if force_ir: assert kinds == 'ir'    # no 'f'
         sublists = []
         if 'i' in kinds: sublists.append(lst_i)
         if 'r' in kinds: sublists.append(lst_r)
@@ -1577,7 +1578,7 @@
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
         op1 = self.rewrite_call(op, 'conditional_call',
                                 op.args[:2], args=op.args[2:],
-                                calldescr=calldescr)
+                                calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1200,28 +1200,12 @@
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
     # conditional calls - note that they cannot return stuff
-    @arguments("cpu", "i", "i", "I", "d")
-    def bhimpl_conditional_call_i_v(cpu, condition, func, args_i, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, None, None, calldescr)
-
-    @arguments("cpu", "i", "i", "R", "d")
-    def bhimpl_conditional_call_r_v(cpu, condition, func, args_r, calldescr):
-        if condition:
-            cpu.bh_call_v(func, None, args_r, None, calldescr)
-
     @arguments("cpu", "i", "i", "I", "R", "d")
     def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
                                      calldescr):
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
-    @arguments("cpu", "i", "i", "I", "R", "F", "d")
-    def bhimpl_conditional_call_irf_v(cpu, condition, func, args_i, args_r,
-                                      args_f, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
-
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1056,23 +1056,11 @@
     opimpl_residual_call_irf_f = _opimpl_residual_call3
     opimpl_residual_call_irf_v = _opimpl_residual_call3
 
-    @arguments("box", "box", "boxes", "descr", "orgpc")
-    def opimpl_conditional_call_i_v(self, condbox, funcbox, argboxes, calldescr,
-                                    pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
-
-    opimpl_conditional_call_r_v = opimpl_conditional_call_i_v
-
     @arguments("box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, funcbox, argboxes,
                                      calldescr, pc):
         self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
 
-    @arguments("box", "box", "boxes3", "descr", "orgpc")
-    def opimpl_conditional_call_irf_v(self, condbox, funcbox, argboxes,
-                                      calldescr, pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
-
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
         targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]

From pypy.commits at gmail.com  Mon Sep 12 05:37:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 02:37:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <57d67740.68adc20a.aeaa2.814e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87027:1a0c32b2613b
Date: 2016-09-12 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/1a0c32b2613b/

Log:	merge heads

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
         assert module.test_fast_sequence(s[0:-1])
         assert module.test_fast_sequence(s[::-1])
 
+    def test_fast_keyerror(self):
+        module = self.import_extension('foo', [
+            ("test_fast_sequence", "METH_VARARGS",
+             """
+                PyObject *foo;
+                PyObject * seq = PyTuple_GetItem(args, 0);
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "Could not convert object to sequence");
+                if (foo != NULL)
+                {
+                    return foo;
+                }
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                    return PyBool_FromLong(1);
+                }
+                return NULL;
+             """)])
+        class Map(object):
+            def __len__(self):
+                return 1
+
+            def __getitem__(self, index):
+                raise KeyError()
+
+        assert module.test_fast_sequence(Map()) is True
+

From pypy.commits at gmail.com  Mon Sep 12 06:05:19 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 12 Sep 2016 03:05:19 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: fixed vecopt tests with the new
 delayed execution
Message-ID: <57d67ddf.0205c20a.8950.960a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87028:88be11973c7a
Date: 2016-09-12 12:04 +0200
http://bitbucket.org/pypy/pypy/changeset/88be11973c7a/

Log:	fixed vecopt tests with the new delayed execution

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -1057,7 +1057,7 @@
             var = ResOperation(rop.INT_ADD, args)
             tolist.append(var)
         if self.constant < 0:
-            args = [var, ConstInt(self.constant)]
+            args = [var, ConstInt(-self.constant)]
             var = ResOperation(rop.INT_SUB, args)
             tolist.append(var)
         return tolist
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -55,18 +55,29 @@
             op = node.getoperation()
             if op in needs_resolving:
                 # either it is a normal operation, or we know that there is a linear combination
+                del needs_resolving[op]
                 if op in indexvars:
                     indexvar = indexvars[op]
                     for operation in indexvar.get_operations():
-                        self.oplist.append(operation)
+                        self.append_to_oplist(operation)
                         last = operation
+                    indexvars[last] = indexvar
                     self.renamer.start_renaming(op, last)
-                    del needs_resolving[op]
                 else: 
-                    del needs_resolving[op]
                     self.resolve_delayed(needs_resolving, delayed, op)
-                    self.oplist.append(op)
+                    self.append_to_oplist(op)
+                self.seen[op] = None
+                if len(delayed) > i:
+                    del delayed[i]
             i -= 1
+            # some times the recursive call can remove several items from delayed,
+            # thus we correct the index here
+            if len(delayed) <= i:
+                i = len(delayed)-1
+
+    def append_to_oplist(self, op):
+        self.renamer.rename(op)
+        self.oplist.append(op)
 
 
     def post_schedule(self):
@@ -106,7 +117,79 @@
 
     def try_emit_or_delay(self, node):
         # implement me in subclass. e.g. as in VecScheduleState
-        raise NotImplementedError
+
+        if not node.is_imaginary() and node.is_pure():
+            # this operation might never be emitted. only if it is really needed
+            self.delay_emit(node)
+            return
+        # emit a now!
+        self.pre_emit(node)
+        self.mark_emitted(node)
+        if not node.is_imaginary():
+            op = node.getoperation()
+            self.seen[op] = None
+            self.append_to_oplist(op)
+
+    def delay_emit(self, node):
+        """ it has been decided that the operation might be scheduled later """
+        delayed = node.delayed or []
+        delayed.append(node)
+        node.delayed = None
+        provides = node.provides()
+        if len(provides) == 0:
+            self.delayed.append(node)
+        else:
+            for to in node.provides():
+                tnode = to.target_node()
+                self.delegate_delay(tnode, delayed[:])
+        self.mark_emitted(node)
+
+    def delegate_delay(self, node, delayed):
+        """ Chain up delays, this can reduce many more of the operations """
+        if node.delayed is None:
+            node.delayed = delayed
+        else:
+            delayedlist = node.delayed
+            for d in delayed:
+                delayedlist.append(d)
+
+
+    def mark_emitted(state, node, unpack=True):
+        """ An operation has been emitted, adds new operations to the worklist
+            whenever their dependency count drops to zero.
+            Keeps worklist sorted (see priority) """
+        worklist = state.worklist
+        provides = node.provides()[:]
+        for dep in provides: # COPY
+            target = dep.to
+            node.remove_edge_to(target)
+            if not target.emitted and target.depends_count() == 0:
+                # sorts them by priority
+                i = len(worklist)-1
+                while i >= 0:
+                    cur = worklist[i]
+                    c = (cur.priority - target.priority)
+                    if c < 0: # meaning itnode.priority < target.priority:
+                        worklist.insert(i+1, target)
+                        break
+                    elif c == 0:
+                        # if they have the same priority, sort them
+                        # using the original position in the trace
+                        if target.getindex() < cur.getindex():
+                            worklist.insert(i+1, target)
+                            break
+                    i -= 1
+                else:
+                    worklist.insert(0, target)
+        node.clear_dependencies()
+        node.emitted = True
+        if not node.is_imaginary():
+            op = node.getoperation()
+            state.renamer.rename(op)
+            if unpack:
+                state.ensure_args_unpacked(op)
+            state.post_emit(node)
+
 
     def delay(self, node):
         return False
@@ -120,8 +203,27 @@
     def post_emit(self, node):
         pass
 
-    def pre_emit(self, node):
-        pass
+    def pre_emit(self, node, pack_first=True):
+        delayed = node.delayed
+        if delayed:
+            # there are some nodes that have been delayed just for this operation
+            if pack_first:
+                op = node.getoperation()
+                self.resolve_delayed({}, delayed, op)
+            for node in delayed:
+                if node in self.seen:
+                    continue
+                if node is not None:
+                    provides = node.provides()
+                    if len(provides) == 0:
+                        # add this node to the final delay list
+                        # might be emitted before jump!
+                        self.delayed.append(node)
+                    else:
+                        for to in node.provides():
+                            tnode = to.target_node()
+                            self.delegate_delay(tnode, [node])
+                        node.delayed = None
 
 class Scheduler(object):
     """ Create an instance of this class to (re)schedule a vector trace. """
@@ -412,7 +514,7 @@
     assert vecinfo.count > argvecinfo.count
 
 def expand(state, pack, args, arg, index):
-    """ Expand a value into a vector box. useful for arith metic
+    """ Expand a value into a vector box. useful for arithmetic
         of one vector with a scalar (either constant/varialbe)
     """
     left = pack.leftmost()
@@ -547,23 +649,8 @@
                     failargs[i] = self.renamer.rename_map.get(seed, seed)
             op.setfailargs(failargs)
 
-        delayed = node.delayed
-        if delayed:
-            # there are some nodes that have been delayed just for this operation
-            if pack_first:
-                self.resolve_delayed({}, delayed, op)
-            for node in delayed:
-                if node is not None:
-                    provides = node.provides()
-                    if len(provides) == 0:
-                        # add this node to the final delay list
-                        # might be emitted before jumping!
-                        self.delayed.append(node)
-                    else:
-                        for to in node.provides():
-                            tnode = to.target_node()
-                            self.delegate_delay(tnode, [node])
-                        node.delayed = None
+        SchedulerState.pre_emit(self, node, pack_first)
+
 
     def profitable(self):
         return self.costmodel.profitable()
@@ -584,76 +671,7 @@
             turn_into_vector(self, node.pack)
             return
         elif not node.emitted:
-            if not node.is_imaginary() and node.is_pure():
-                # this operation might never be emitted. only if it is really needed
-                self.delay_emit(node)
-                return
-            # emit a now!
-            self.pre_emit(node)
-            self.mark_emitted(node)
-            if not node.is_imaginary():
-                op = node.getoperation()
-                self.seen[op] = None
-                self.oplist.append(op)
-
-    def delay_emit(self, node):
-        """ it has been decided that the operation might be scheduled later """
-        delayed = node.delayed or []
-        delayed.append(node)
-        node.delayed = None
-        provides = node.provides()
-        if len(provides) == 0:
-            self.delayed.append(node)
-        else:
-            for to in node.provides():
-                tnode = to.target_node()
-                self.delegate_delay(tnode, delayed[:])
-        self.mark_emitted(node)
-
-    def delegate_delay(self, node, delayed):
-        """ Chain up delays, this can reduce many more of the operations """
-        if node.delayed is None:
-            node.delayed = delayed
-        else:
-            delayedlist = node.delayed
-            for d in delayed:
-                delayedlist.append(d)
-
-    def mark_emitted(state, node, unpack=True):
-        """ An operation has been emitted, adds new operations to the worklist
-            whenever their dependency count drops to zero.
-            Keeps worklist sorted (see priority) """
-        worklist = state.worklist
-        provides = node.provides()[:]
-        for dep in provides: # COPY
-            target = dep.to
-            node.remove_edge_to(target)
-            if not target.emitted and target.depends_count() == 0:
-                # sorts them by priority
-                i = len(worklist)-1
-                while i >= 0:
-                    cur = worklist[i]
-                    c = (cur.priority - target.priority)
-                    if c < 0: # meaning itnode.priority < target.priority:
-                        worklist.insert(i+1, target)
-                        break
-                    elif c == 0:
-                        # if they have the same priority, sort them
-                        # using the original position in the trace
-                        if target.getindex() < cur.getindex():
-                            worklist.insert(i+1, target)
-                            break
-                    i -= 1
-                else:
-                    worklist.insert(0, target)
-        node.clear_dependencies()
-        node.emitted = True
-        if not node.is_imaginary():
-            op = node.getoperation()
-            state.renamer.rename(op)
-            if unpack:
-                state.ensure_args_unpacked(op)
-            state.post_emit(node)
+            SchedulerState.try_emit_or_delay(self, node)
 
     def delay(self, node):
         if node.pack:
@@ -706,7 +724,7 @@
             self.renamer.start_renaming(arg, vecop)
             self.seen[vecop] = None
             self.costmodel.record_vector_unpack(var, pos, 1)
-            self.oplist.append(vecop)
+            self.append_to_oplist(vecop)
             return vecop
         return arg
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -271,7 +271,7 @@
 
     def test_opcount_filling_guard(self):
         descr = ArrayDescr(0,4, None, 'S')
-        vec = ResOperation(rop.VEC_LOAD_I, ['a','i'], descr=descr)
+        vec = ResOperation(rop.VEC_LOAD_I, ['a','i', 8, 0], descr=descr)
         vec.count = 4
         pack = Pack([Node(ResOperation(rop.GUARD_TRUE, [vec]), 0),
                      Node(ResOperation(rop.GUARD_TRUE, [vec]), 1),
@@ -319,7 +319,7 @@
         self.vectorize(trace)
         self.debug_print_operations(trace)
         self.ensure_operations([
-            'v10[4xi32] = vec_load_i(p0,i0,descr=int32arraydescr)',
+            'v10[4xi32] = vec_load_i(p0,i0,4,0,descr=int32arraydescr)',
             'v11[4xi32] = vec_int_is_true(v10[4xi32])',
             'i100 = vec_unpack_i(v11[4xi32], 0, 1)',
             'vec_guard_true(v11[4xi32]) [i100]',
@@ -384,9 +384,8 @@
         """
         opt = """
         [p0,i0]
-        i1 = int_add(i0,1)
+        v3[2xi64] = vec_load_i(p0,i0,8,0,descr=arraydescr)
         i2 = int_add(i0,2)
-        v3[2xi64] = vec_load_i(p0,i0,descr=arraydescr)
         jump(p0,i2)
         """
         loop = self.parse_loop(ops)
@@ -799,13 +798,11 @@
         for i in range(15):
             for j in range(15):
                 try:
+                    mref1 = graph.getmemref(i)
+                    mref2 = graph.getmemref(j)
                     if i-4 == j or i+4 == j:
-                        mref1 = graph.getmemref(i)
-                        mref2 = graph.getmemref(j)
                         assert mref1.is_adjacent_to(mref2)
                     else:
-                        mref1 = graph.getmemref(i)
-                        mref2 = graph.getmemref(j)
                         assert not mref1.is_adjacent_to(mref2)
                 except KeyError:
                     pass
@@ -846,11 +843,13 @@
         loop = self.parse_loop(ops)
         vopt, graph = self.extend_packset(loop,1)
         assert len(graph.memory_refs) == 2
-        self.assert_independent(graph, 5,10)
-        assert len(vopt.packset.packs) == 2
+        self.assert_independent(graph, 3,7)
+        # the delayed scheduling strips away the vectorized addition,
+        # because it is never used
+        assert len(vopt.packset.packs) == 1
         self.assert_packset_empty(vopt.packset,
                                   len(loop.operations),
-                                  [(5,10), (4,9)])
+                                  [(4,8)])
 
     def test_packset_extend_load_modify_store(self):
         ops = """
@@ -1009,11 +1008,11 @@
         i1 = int_add(i0, {stride})
         i11 = int_le(i1, 128)
         guard_true(i11) [p0,p1,p2,i1]
-        i12 = int_add(i1, {stride})
-        v1 = vec_load{suffix}(p0, i0, descr={descr}arraydescr)
-        v2 = vec_load{suffix}(p1, i0, descr={descr}arraydescr)
+        v1 = vec_load{suffix}(p0, i0,8,0, descr={descr}arraydescr)
+        v2 = vec_load{suffix}(p1, i0,8,0, descr={descr}arraydescr)
         v3 = {op}(v1,v2)
-        vec_store(p2, i0, v3, descr={descr}arraydescr)
+        vec_store(p2, i0, v3,8,0, descr={descr}arraydescr)
+        i12 = int_add(i0, 2)
         jump(p0,p1,p2,i12)
         """.format(op='vec_'+op,descr=descr,stride=1,suffix=suffix)
         loop = self.parse_loop(ops)
@@ -1038,15 +1037,14 @@
         i11 = int_add(i0, 1) 
         i12 = int_lt(i11, i1) 
         guard_true(i12) [i0,i1,i2,i3,i4]
-        i6 = int_mul(i0, 8) 
-        i13 = int_add(i11, 1) 
+        i13 = int_add(i0, 2) 
         i18 = int_lt(i13, i1) 
         guard_true(i18) [i11,i1,i2,i3,i4]
-        i14 = int_mul(i11, 8) 
-        v19[2xi64] = vec_load_i(i2, i6, descr=arraydescr) 
-        v20[2xi64] = vec_load_i(i3, i6, descr=arraydescr) 
+        i6 = int_mul(i0, 8) 
+        v19[2xi64] = vec_load_i(i2, i6, 1, 0, descr=arraydescr) 
+        v20[2xi64] = vec_load_i(i3, i6, 1, 0, descr=arraydescr) 
         v21[2xi64] = vec_int_add(v19, v20) 
-        vec_store(i4, i6, v21, descr=arraydescr) 
+        vec_store(i4, i6, v21, 1, 0, descr=arraydescr) 
         jump(i13, i1, i2, i3, i4)
         """
         loop = self.parse_loop(ops)
@@ -1076,7 +1074,7 @@
         {dead_code}
         i500 = int_add(i0, 16)
         i501 = int_lt(i500, 102)
-        v10[16xi8] = vec_load_i(p0, i0, descr=chararraydescr)
+        v10[16xi8] = vec_load_i(p0, i0, 1, 0, descr=chararraydescr)
         jump(p0,i2)
         """.format(dead_code=dead_code)
         loop = self.parse_loop(ops)
@@ -1105,6 +1103,7 @@
         [p0,i0]
         i1 = getarrayitem_raw_i(p0, i0, descr=arraydescr)
         i4 = int_sub(i1, 42)
+        setarrayitem_raw(p0, i0, i4, descr=arraydescr)
         i3 = int_add(i0,1)
         i5 = int_lt(i3, 10)
         guard_true(i5) [p0, i0]
@@ -1121,8 +1120,9 @@
         guard_true(i3) [p0,i0]
         i4 = int_add(i0, 2)
         i5 = int_lt(i4, 10)
-        v1[2xf64] = vec_load_i(p0, i0, descr=arraydescr)
+        v1[2xf64] = vec_load_i(p0, i0, 8, 0, descr=arraydescr)
         v2[2xf64] = vec_int_sub(v1[2xf64], v3[2xf64])
+        vec_store(p0, i0, v2[2xf64], 8, 0, descr=arraydescr)
         jump(p0,i2,v3[2xf64])
         """
         loop = self.parse_loop(ops)
@@ -1134,6 +1134,7 @@
         [p0,i0,f3]
         f1 = getarrayitem_raw_f(p0, i0, descr=floatarraydescr)
         f4 = float_add(f1, f3)
+        setarrayitem_raw(p0, i0, f4, descr=floatarraydescr)
         i3 = int_add(i0,1)
         i5 = int_lt(i3, 10)
         guard_true(i5) [p0, i0]
@@ -1150,8 +1151,9 @@
         guard_true(i3) [p0,i0,f3]
         i4 = int_add(i0, 2)
         i5 = int_lt(i4, 10)
-        v1[2xf64] = vec_load_f(p0, i0, descr=floatarraydescr)
+        v1[2xf64] = vec_load_f(p0, i0, 8, 0, descr=floatarraydescr)
         v2[2xf64] = vec_float_add(v1[2xf64], v3[2xf64])
+        vec_store(p0, i0, v2[2xf64], 8, 0, descr=floatarraydescr)
         jump(p0,i2,f3,v3[2xf64])
         """
         loop = self.parse_loop(ops)
@@ -1181,7 +1183,7 @@
         guard_true(i2) [p0, i0, v2[2xf64]]
         i10 = int_add(i0, 16)
         i20 = int_lt(i10, 100)
-        v1[2xf64] = vec_load_f(p0, i0, descr=floatarraydescr)
+        v1[2xf64] = vec_load_f(p0, i0, 1, 0, descr=floatarraydescr)
         v3[2xf64] = vec_float_add(v2[2xf64], v1[2xf64])
         jump(p0, i1, v3[2xf64])
         """
@@ -1213,21 +1215,18 @@
         i54 = int_add(i28, 2) 
         i638 = int_ge(i54, i18)
         guard_false(i638) [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18]
-        i12 = int_add(i44, 8)
-        i56 = int_add(i41, 8) 
-        i46 = int_add(i37, 8) 
         i47 = int_add(i28, 2) 
         i52 = int_ge(i47, i18) 
+        v61[2xf64] = vec_load_f(i21, i44, 1, 0, descr=floatarraydescr) 
+        v62[2xf64] = vec_load_f(i4, i41, 1, 0, descr=floatarraydescr) 
+        v63[2xf64] = vec_float_add(v61, v62) 
+        vec_store(i0, i37, v63, 1, 0, descr=floatarraydescr) 
         i55 = int_add(i44, 16) 
         i629 = int_add(i41, 16)
         i637 = int_add(i37, 16)
-        v61[2xf64] = vec_load_f(i21, i44, descr=floatarraydescr) 
-        v62[2xf64] = vec_load_f(i4, i41, descr=floatarraydescr) 
-        v63[2xf64] = vec_float_add(v61, v62) 
-        vec_store(i0, i37, v63, descr=floatarraydescr) 
         f100 = vec_unpack_f(v61, 1, 1)
         f101 = vec_unpack_f(v62, 1, 1)
-        jump(p36, i637, p9, i56, p14, f100, p12, p38, f101, p39, i40, i54, p42, i43, i55, i21, i4, i0, i18)
+        jump(p36, i637, p9, i629, p14, f100, p12, p38, f101, p39, i40, i54, p42, i43, i55, i21, i4, i0, i18)
         """)
         vopt = self.vectorize(trace)
         self.assert_equal(trace, trace_opt)
@@ -1256,12 +1255,12 @@
         i11 = int_ge(i6, 36)
         i7 = int_add(i1, 4)
         i14 = int_ge(i7, 36)
-        v17 = vec_load_f(p0, i1, descr=floatarraydescr)
+        v17 = vec_load_f(p0, i1, 8, 0, descr=floatarraydescr)
         v19 = vec_cast_float_to_singlefloat(v17)
-        v18 = vec_load_f(p0, i5, descr=floatarraydescr)
+        v18 = vec_load_f(p0, i5, 8, 0, descr=floatarraydescr)
         v20 = vec_cast_float_to_singlefloat(v18)
         v21 = vec_pack_i(v19, v20, 2, 2)
-        vec_store(p1, i1, v21, descr=float32arraydescr)
+        vec_store(p1, i1, v21, 4, 0, descr=float32arraydescr)
         jump(p0, p1, i50)
         """
         loop = self.parse_loop(ops)
@@ -1291,21 +1290,18 @@
         i500 = int_add(i4, 16)
         i501 = int_lt(i500, 100)
         guard_true(i501) [p0, p1, p2, i0, i4]
-        i189 = int_add(i0, 4)
         i187 = int_add(i4, 8)
         i188 = int_lt(i187, 100)
-        i207 = int_add(i0, 8)
         i196 = int_add(i4, 12)
         i197 = int_lt(i196, 100)
-        i205 = int_add(i0, 12)
         i400 = int_add(i4, 16)
         i401= int_lt(i400, 100)
-        i402 = int_add(i0, 16)
-        v228[4xi32] = vec_load_i(p0, i0, descr=float32arraydescr)
+        v228[4xi32] = vec_load_i(p0, i0, 1, 0, descr=float32arraydescr)
         v229[2xf64] = vec_cast_singlefloat_to_float(v228)
         v230 = vec_unpack_i(v228, 2, 2)
         v231 = vec_cast_singlefloat_to_float(v230)
-        v232 = vec_load_i(p1, i189, descr=float32arraydescr)
+        i189 = int_add(i0, 4)
+        v232 = vec_load_i(p1, i189, 1, 0, descr=float32arraydescr)
         v233 = vec_cast_singlefloat_to_float(v232)
         v236 = vec_float_add(v229, v233)
         v238 = vec_cast_float_to_singlefloat(v236)
@@ -1314,7 +1310,8 @@
         v237 = vec_float_add(v231, v235)
         v239 = vec_cast_float_to_singlefloat(v237)
         v240 = vec_pack_i(v238, v239, 2, 2)
-        vec_store(p2, i4, v240, descr=float32arraydescr)
+        vec_store(p2, i4, v240, 1, 0, descr=float32arraydescr)
+        i207 = int_add(i0, 16)
         jump(p0, p1, p2, i207, i500)
         """)
         vopt = self.vectorize(trace)
@@ -1354,7 +1351,7 @@
         """)
         vopt = self.schedule(trace)
         self.ensure_operations([
-            'v10[2xf64] = vec_load_f(p0,i0,descr=floatarraydescr)',
+            'v10[2xf64] = vec_load_f(p0,i0,8,0,descr=floatarraydescr)',
             'v11[2xf64] = vec_float_mul(v10[2xf64], v9[2xf64])',
             'v12[2xf64] = vec_float_eq(v11[2xf64], v11[2xf64])',
             'i100 = vec_unpack_f(v12[4xi32], 0, 1)',

From pypy.commits at gmail.com  Mon Sep 12 06:08:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 03:08:58 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: I *think* this is not
 needed right now, and test_ll_random seems to
Message-ID: <57d67eba.2472c20a.5693d.9b0a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87029:02962b25dcee
Date: 2016-09-12 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/02962b25dcee/

Log:	I *think* this is not needed right now, and test_ll_random seems to
	agree with that, but better be explicit.

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -961,7 +961,8 @@
             assert not isinstance(condvalue_loc, ImmedLoc)
             self.assembler.test_location(condvalue_loc)
             self.assembler.guard_success_cc = rx86.Conditions['Z']
-            resloc = self.rm.force_result_in_reg(op, args[0])
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=arglocs)
         self.assembler.cond_call(op, gcmap, imm_func, arglocs, resloc)
 
     consider_cond_call_value_i = consider_cond_call

From pypy.commits at gmail.com  Mon Sep 12 06:18:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 03:18:26 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Extra test
Message-ID: <57d680f2.411d1c0a.99c34.9f01@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87030:99dfe8a97e71
Date: 2016-09-12 10:57 +0100
http://bitbucket.org/pypy/pypy/changeset/99dfe8a97e71/

Log:	Extra test

diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -418,6 +418,12 @@
         return 123
     RAW = lltype.Struct('RAW', ('x', lltype.Signed))
 
+    def f5(b):
+        r = b.foobar
+        if r == 0:
+            r = b.foobar = 123
+        return r
+
     def f(a, c):
         b = B()
         f1(a, b, c)
@@ -425,6 +431,7 @@
         f3(space, b)
         r = lltype.malloc(RAW, flavor='raw')
         f4(r)
+        f5(b)
 
     rtyper = support.annotate(f, [10, 20])
     f1_graph = rtyper.annotator.translator._graphof(f1)
@@ -435,6 +442,8 @@
     assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
     f4_graph = rtyper.annotator.translator._graphof(f4)
     assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
+    f5_graph = rtyper.annotator.translator._graphof(f5)
+    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
 
 def test_cant_find_call_shortcut():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU

From pypy.commits at gmail.com  Mon Sep 12 06:18:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 03:18:28 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Another case
Message-ID: <57d680f4.43921c0a.7f8a9.9848@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87031:f7694a9bde4e
Date: 2016-09-12 11:26 +0100
http://bitbucket.org/pypy/pypy/changeset/f7694a9bde4e/

Log:	Another case

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -408,7 +408,7 @@
         v_check = v_result
         fastcase = True
         for op in block.operations[1:]:
-            if (op.opname in ('int_is_true', 'ptr_nonzero')
+            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
                     and v_check is op.args[0]):
                 v_check = op.result
             elif op.opname == 'ptr_iszero' and v_check is op.args[0]:

From pypy.commits at gmail.com  Mon Sep 12 06:18:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 03:18:30 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: merge heads
Message-ID: <57d680f6.6937c20a.1be9a.9841@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87032:1d8aa0d6ff47
Date: 2016-09-12 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/1d8aa0d6ff47/

Log:	merge heads

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -961,7 +961,8 @@
             assert not isinstance(condvalue_loc, ImmedLoc)
             self.assembler.test_location(condvalue_loc)
             self.assembler.guard_success_cc = rx86.Conditions['Z']
-            resloc = self.rm.force_result_in_reg(op, args[0])
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=arglocs)
         self.assembler.cond_call(op, gcmap, imm_func, arglocs, resloc)
 
     consider_cond_call_value_i = consider_cond_call

From pypy.commits at gmail.com  Mon Sep 12 06:26:23 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Mon, 12 Sep 2016 03:26:23 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase (performance
 optimizations)
Message-ID: <57d682cf.898b1c0a.d90db.994f@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87033:332c546faefd
Date: 2016-09-12 12:25 +0200
http://bitbucket.org/pypy/pypy/changeset/332c546faefd/

Log:	Update qcgc codebase (performance optimizations)

diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
--- a/rpython/translator/c/src/qcgc/allocator.c
+++ b/rpython/translator/c/src/qcgc/allocator.c
@@ -276,24 +276,16 @@
 	for ( ; index < QCGC_SMALL_FREE_LISTS; index++) {
 		size_t list_cell_size = small_index_to_cells(index);
 
-		while (qcgc_allocator_state.fit_state.small_free_list[index]->count
-				> 0) {
+		if (qcgc_allocator_state.fit_state.small_free_list[index]->count > 0) {
 			result = qcgc_allocator_state.fit_state.small_free_list[index]->
-				items[qcgc_allocator_state.fit_state.small_free_list[index]
-				->count - 1];
+				items[0];
 			qcgc_allocator_state.fit_state.small_free_list[index] =
 				qcgc_linear_free_list_remove_index(
 						qcgc_allocator_state.fit_state.small_free_list[index],
-						qcgc_allocator_state.fit_state.small_free_list[index]->
-						count - 1);
-
-			// Check whether block is still valid
-			if (valid_block(result, list_cell_size)) {
-				// The next call might invalidate free_list, reload!
-				qcgc_arena_mark_allocated(result, cells);
-				qcgc_fit_allocator_add(result + cells, list_cell_size - cells);
-				return result;
-			}
+						0);
+			qcgc_arena_mark_allocated(result, cells);
+			qcgc_fit_allocator_add(result + cells, list_cell_size - cells);
+			return result;
 		}
 	}
 	return fit_allocator_large_first_fit(0, cells);
@@ -309,30 +301,18 @@
 
 	cell_t *result = NULL;
 	size_t best_fit_cells = SIZE_MAX;
-	size_t i = 0;
-	while (i < qcgc_allocator_state.fit_state.large_free_list[index]->count) {
-		if (valid_block(qcgc_allocator_state.fit_state.large_free_list[index] ->
-					items[i].ptr,
-					qcgc_allocator_state.fit_state.large_free_list[index]->
-					items[i].size)) {
-			if (qcgc_allocator_state.fit_state.large_free_list[index]->
-					items[i].size >= cells &&
-					qcgc_allocator_state.fit_state.large_free_list[index]->
-					items[i].size < best_fit_cells) {
-				result = qcgc_allocator_state.fit_state.large_free_list[index]->
-					items[i].ptr;
-				best_fit_cells = qcgc_allocator_state.fit_state.
-					large_free_list[index]->items[i].size;
-				best_fit_index = i;
-			}
-			i++;
-		} else {
-			qcgc_allocator_state.fit_state.large_free_list[index] =
-				qcgc_exp_free_list_remove_index(qcgc_allocator_state.fit_state.
-						large_free_list[index], i);
-			// NO i++ !
+	size_t count = qcgc_allocator_state.fit_state.large_free_list[index]->count;
+	for (size_t i = 0; i < count; i++) {
+		if (qcgc_allocator_state.fit_state.large_free_list[index]->
+				items[i].size >= cells &&
+				qcgc_allocator_state.fit_state.large_free_list[index]->
+				items[i].size < best_fit_cells) {
+			result = qcgc_allocator_state.fit_state.large_free_list[index]->
+				items[i].ptr;
+			best_fit_cells = qcgc_allocator_state.fit_state.
+				large_free_list[index]->items[i].size;
+			best_fit_index = i;
 		}
-
 		if (best_fit_cells == cells) {
 			break;
 		}
@@ -359,24 +339,17 @@
 	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP) >= cells);
 #endif
 	for ( ; index < QCGC_LARGE_FREE_LISTS; index++) {
-		while(qcgc_allocator_state.fit_state.large_free_list[index]->count
-				> 0) {
+		if (qcgc_allocator_state.fit_state.large_free_list[index]->count > 0) {
 			struct exp_free_list_item_s item =
-				qcgc_allocator_state.fit_state.large_free_list[index]->items[
-				qcgc_allocator_state.fit_state.large_free_list[index]->count - 1
-				];
+				qcgc_allocator_state.fit_state.large_free_list[index]->items[0];
 			qcgc_allocator_state.fit_state.large_free_list[index] =
 				qcgc_exp_free_list_remove_index(
 						qcgc_allocator_state.fit_state.large_free_list[index],
-						qcgc_allocator_state.fit_state.large_free_list[index]->
-						count - 1);
+						0);
 
-			// Check whether block is still valid
-			if (valid_block(item.ptr, item.size)) {
-				qcgc_arena_mark_allocated(item.ptr, cells);
-				qcgc_fit_allocator_add(item.ptr + cells, item.size - cells);
-				return item.ptr;
-			}
+			qcgc_arena_mark_allocated(item.ptr, cells);
+			qcgc_fit_allocator_add(item.ptr + cells, item.size - cells);
+			return item.ptr;
 		}
 	}
 	return NULL;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -69,11 +69,6 @@
  * Shadow stack
  */
 void qcgc_shadowstack_push(object_t *object) {
-	if (qcgc_state.phase != GC_PAUSE) {
-		qcgc_state.phase = GC_MARK;
-		qcgc_push_object(object);
-	}
-
 	*qcgc_state.shadow_stack = object;
 	qcgc_state.shadow_stack++;
 }
@@ -153,8 +148,7 @@
 
 	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
 		// Use bump / fit allocator
-		//if (qcgc_allocator_state.use_bump_allocator) {
-		if (false) {
+		if (qcgc_allocator_state.use_bump_allocator) {
 			result = qcgc_bump_allocate(size);
 		} else {
 			result = qcgc_fit_allocate(size);
@@ -209,10 +203,6 @@
 }
 
 void qcgc_mark(bool incremental) {
-	if (qcgc_state.phase == GC_COLLECT) {
-		return;	// Fast exit when there is nothing to mark
-	}
-
 	{
 		struct log_info_s {
 			bool incremental;
@@ -226,16 +216,6 @@
 	qcgc_state.bytes_since_incmark = 0;
 
 	if (qcgc_state.phase == GC_PAUSE) {
-		qcgc_state.phase = GC_MARK;
-
-		// If we do this for the first time, push all roots.
-		// All further changes to the roots (new additions) will be added
-		// by qcgc_shadowstack_push
-		for (object_t **it = qcgc_state.shadow_stack_base;
-			it < qcgc_state.shadow_stack;
-			it++) {
-			qcgc_push_object(*it);
-		}
 
 		// If we do this for the first time, push all prebuilt objects.
 		// All further changes to prebuilt objects will go to the gp_gray_stack
@@ -248,6 +228,15 @@
 		}
 	}
 
+	qcgc_state.phase = GC_MARK;
+
+	// Always push all roots to make shadowstack pushes faster
+	for (object_t **it = qcgc_state.shadow_stack_base;
+		it < qcgc_state.shadow_stack;
+		it++) {
+		qcgc_push_object(*it);
+	}
+
 	while (qcgc_state.gray_stack_size > 0) {
 		// General purpose gray stack (prebuilt objects and huge blocks)
 		size_t to_process = (incremental ?

From pypy.commits at gmail.com  Mon Sep 12 06:33:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 03:33:11 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Add a flag
 'supports_cond_call_value'. E.g. the ARM backend should first
Message-ID: <57d68467.255ac20a.7ccce.99fc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87034:88d339078b81
Date: 2016-09-12 12:32 +0200
http://bitbucket.org/pypy/pypy/changeset/88d339078b81/

Log:	Add a flag 'supports_cond_call_value'. E.g. the ARM backend should
	first improve the code generation of 'cond_call' before we consider
	adding 'cond_call_value' (see XXXs)

diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1002,6 +1002,9 @@
     prepare_op_cond_call_gc_wb_array = prepare_op_cond_call_gc_wb
 
     def prepare_op_cond_call(self, op, fcond):
+        # XXX don't force the arguments to be loaded in specific
+        # locations before knowing if we can take the fast path
+        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -325,6 +325,7 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
+    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_extension = True
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -619,6 +619,8 @@
                                       size, 1, ofs)
 
     def expand_call_shortcut(self, op):
+        if not self.cpu.supports_cond_call_value:
+            return
         descr = op.getdescr()
         if descr is None:
             return
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -209,6 +209,7 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
+    supports_cond_call_value = True
 
     translate_support_code = None
 
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,6 +16,7 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
+    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2448,6 +2448,9 @@
             assert called == [(67, 89)]
 
     def test_cond_call_value(self):
+        if not self.cpu.supports_cond_call_value:
+            py.test.skip("missing supports_cond_call_value")
+
         def func_int(*args):
             called.append(args)
             return len(args) * 100 + 1000
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -15,6 +15,7 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
+    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False

From pypy.commits at gmail.com  Mon Sep 12 07:55:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 04:55:46 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: Delicate ordering here
 (shown by test_ll_random)
Message-ID: <57d697c2.8a13c20a.222c9.c61f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87035:31565879b1f7
Date: 2016-09-12 13:43 +0200
http://bitbucket.org/pypy/pypy/changeset/31565879b1f7/

Log:	Delicate ordering here (shown by test_ll_random)

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -759,6 +759,8 @@
         if (opnum != rop.GUARD_TRUE and opnum != rop.GUARD_FALSE
                                     and opnum != rop.COND_CALL):
             return False
+        # NB: don't list COND_CALL_VALUE_I/R here, these two variants
+        # of COND_CALL don't accept a cc as input
         if next_op.getarg(0) is not op:
             return False
         if self.longevity[op][1] > i + 1:
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2382,7 +2382,7 @@
     def label(self):
         self._check_frame_depth_debug(self.mc)
 
-    def cond_call(self, op, gcmap, imm_func, arglocs, resloc=None):
+    def cond_call(self, gcmap, imm_func, arglocs, resloc=None):
         assert self.guard_success_cc >= 0
         self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0)
                                                             # patched later
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -943,8 +943,13 @@
         v_func = args[1]
         assert isinstance(v_func, Const)
         imm_func = self.rm.convert_to_imm(v_func)
+
+        # Delicate ordering here.  First get the argument's locations.
+        # If this also contains args[0], this returns the current
+        # location too.
         arglocs = [self.loc(args[i]) for i in range(2, len(args))]
         gcmap = self.get_gcmap()
+
         if op.type == 'v':
             # a plain COND_CALL.  Calls the function when args[0] is
             # true.  Often used just after a comparison operation.
@@ -957,13 +962,18 @@
             # Implemented by forcing the result to live in the same
             # register as args[0], and overwriting it if we really do
             # the call.
-            condvalue_loc = self.loc(args[0])
-            assert not isinstance(condvalue_loc, ImmedLoc)
-            self.assembler.test_location(condvalue_loc)
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+
+            # Test the register for the result.
+            self.assembler.test_location(resloc)
             self.assembler.guard_success_cc = rx86.Conditions['Z']
-            resloc = self.rm.force_result_in_reg(op, args[0],
-                                                 forbidden_vars=arglocs)
-        self.assembler.cond_call(op, gcmap, imm_func, arglocs, resloc)
+
+        self.assembler.cond_call(gcmap, imm_func, arglocs, resloc)
 
     consider_cond_call_value_i = consider_cond_call
     consider_cond_call_value_r = consider_cond_call

From pypy.commits at gmail.com  Mon Sep 12 08:07:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 05:07:28 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: tweak the test
Message-ID: <57d69a80.c62f1c0a.55ee2.cc55@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87036:6ba9f0326d65
Date: 2016-09-12 13:57 +0200
http://bitbucket.org/pypy/pypy/changeset/6ba9f0326d65/

Log:	tweak the test

diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -709,12 +709,14 @@
         builder.loop.operations.append(op)
 
 # 6. a conditional call (for now always with no exception raised)
-class BaseCondCallOperation(BaseCallOperation):
+class CondCallOperation(BaseCallOperation):
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
+            RESULT_TYPE = lltype.Void
             v_cond = builder.get_bool_var(r)
         else:
+            RESULT_TYPE = lltype.Signed
             v_cond = r.choice(builder.intvars)
         subset = builder.subset_of_intvars(r)[:4]
         for i in range(len(subset)):
@@ -727,10 +729,10 @@
                 seen.append(args)
             else:
                 assert seen[0] == args
-            if self.RESULT_TYPE is lltype.Signed:
+            if RESULT_TYPE is lltype.Signed:
                 return len(args) - 42000
         #
-        TP = lltype.FuncType([lltype.Signed] * len(subset), self.RESULT_TYPE)
+        TP = lltype.FuncType([lltype.Signed] * len(subset), RESULT_TYPE)
         ptr = llhelper(lltype.Ptr(TP), call_me)
         c_addr = ConstAddr(llmemory.cast_ptr_to_adr(ptr), builder.cpu)
         args = [v_cond, c_addr] + subset
@@ -741,14 +743,6 @@
         op.setfailargs(fail_subset)
         builder.loop.operations.append(op)
 
-class CondCallOperation(BaseCondCallOperation):
-    RESULT_TYPE = lltype.Void
-    opnum = rop.COND_CALL
-
-class CondCallValueOperation(BaseCondCallOperation):
-    RESULT_TYPE = lltype.Signed
-    opnum = rop.COND_CALL_VALUE_I
-
 # ____________________________________________________________
 
 OPERATIONS = test_random.OPERATIONS[:]

From pypy.commits at gmail.com  Mon Sep 12 08:28:25 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 12 Sep 2016 05:28:25 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: update the llgraph runner to
 exec vec_load/store
Message-ID: <57d69f69.452ec20a.bd23e.c951@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87037:7ee75be77c56
Date: 2016-09-12 14:17 +0200
http://bitbucket.org/pypy/pypy/changeset/7ee75be77c56/

Log:	update the llgraph runner to exec vec_load/store

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -330,7 +330,9 @@
     is_llgraph = True
     vector_ext = VectorExt()
     vector_ext.enable(16, accum=True)
+    vector_ext.setup_once = lambda asm: asm
     load_supported_factors = (1,2,4,8)
+    assembler = None
 
     def __init__(self, rtyper, stats=None, *ignored_args, **kwds):
         model.AbstractCPU.__init__(self)
@@ -927,50 +929,32 @@
     def bh_vec_int_signext(self, vx, ext, count):
         return [heaptracker.int_signext(_vx, ext) for _vx in vx]
 
-    def build_getarrayitem(func):
-        def method(self, struct, offset, descr, _count):
+    def build_load(func):
+        def method(self, struct, offset, scale, disp, descr, _count):
             values = []
             count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes()
             assert _count == count
             assert count > 0
+            adr = struct + (offset * scale + disp)
+            a = support.cast_arg(lltype.Ptr(descr.A), adr)
+            array = a._obj
             for i in range(count):
-                val = func(self, struct, offset + i, descr)
+                val = support.cast_result(descr.A.OF, array.getitem(i))
                 values.append(val)
             return values
         return method
 
-    bh_vec_getarrayitem_gc_i = build_getarrayitem(bh_getarrayitem_gc)
-    bh_vec_getarrayitem_gc_f = build_getarrayitem(bh_getarrayitem_gc)
-    bh_vec_getarrayitem_raw_i = build_getarrayitem(bh_getarrayitem_raw)
-    bh_vec_getarrayitem_raw_f = build_getarrayitem(bh_getarrayitem_raw)
-    del build_getarrayitem
+    bh_vec_load_i = build_load(bh_getarrayitem_raw)
+    bh_vec_load_f = build_load(bh_getarrayitem_raw)
+    del build_load
 
-    def _bh_vec_raw_load(self, struct, offset, descr, _count):
-        values = []
+    def bh_vec_store(self, struct, offset, newvalues, scale, disp, descr, count):
         stride = descr.get_item_size_in_bytes()
-        count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes()
-        assert _count == count
-        assert count > 0
-        for i in range(count):
-            val = self.bh_raw_load(struct, offset + i*stride, descr)
-            values.append(val)
-        return values
-
-    bh_vec_raw_load_i = _bh_vec_raw_load
-    bh_vec_raw_load_f = _bh_vec_raw_load
-
-    def bh_vec_raw_store(self, struct, offset, newvalues, descr, count):
-        stride = descr.get_item_size_in_bytes()
+        adr = struct + (offset * scale + disp)
+        a = support.cast_arg(lltype.Ptr(descr.A), adr)
+        array = a._obj
         for i,n in enumerate(newvalues):
-            self.bh_raw_store(struct, offset + i*stride, n, descr)
-
-    def bh_vec_setarrayitem_raw(self, struct, offset, newvalues, descr, count):
-        for i,n in enumerate(newvalues):
-            self.bh_setarrayitem_raw(struct, offset + i, n, descr)
-
-    def bh_vec_setarrayitem_gc(self, struct, offset, newvalues, descr, count):
-        for i,n in enumerate(newvalues):
-            self.bh_setarrayitem_gc(struct, offset + i, n, descr)
+            array.setitem(i, support.cast_arg(descr.A.OF, n))
 
     def store_fail_descr(self, deadframe, descr):
         pass # I *think*
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -9,6 +9,7 @@
 from rpython.jit.metainterp import pyjitpl, history, jitexc
 from rpython.jit.codewriter.policy import JitPolicy
 from rpython.jit.codewriter import codewriter, longlong
+from rpython.jit.backend.llsupport.vector_ext import VectorExt
 from rpython.rlib.rfloat import isnan
 from rpython.rlib.jit import ENABLE_ALL_OPTS
 from rpython.translator.backendopt.all import backend_optimizations
@@ -298,6 +299,9 @@
 class LLJitMixin(JitMixin):
     CPUClass = runner.LLGraphCPU
 
+    def supports_vector_ext(self):
+        return True
+
     @staticmethod
     def Ptr(T):
         return lltype.Ptr(T)

From pypy.commits at gmail.com  Mon Sep 12 08:28:27 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 12 Sep 2016 05:28:27 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57d69f6b.898b1c0a.2c724.0806@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87038:fa43a104e006
Date: 2016-09-12 14:19 +0200
http://bitbucket.org/pypy/pypy/changeset/fa43a104e006/

Log:	merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -12,4 +12,7 @@
 Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
 in numpypy
 
-
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,7 +119,7 @@
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
@@ -645,7 +645,7 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
-        ('_format', rffi.UCHAR),
+        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
         ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,9 +1,7 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib.rarithmetic import widen
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
-from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
+from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
@@ -14,102 +12,4 @@
         return 1
     return 0  
 
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    flags = widen(flags)
-    buf = space.buffer_w(w_obj, flags)
-    try:
-        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-    except ValueError:
-        raise BufferError("could not create buffer from object")
-    view.c_len = buf.getlength()
-    view.c_obj = make_ref(space, w_obj)
-    ndim = buf.getndim()
-    view.c_itemsize = buf.getitemsize()
-    rffi.setintfield(view, 'c_readonly', int(buf.readonly))
-    rffi.setintfield(view, 'c_ndim', ndim)
-    view.c_format = rffi.str2charp(buf.getformat())
-    view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
-    view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
-    shape = buf.getshape()
-    strides = buf.getstrides()
-    for i in range(ndim):
-        view.c_shape[i] = shape[i]
-        view.c_strides[i] = strides[i]
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    return 0
-
-def _IsFortranContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(view.c_ndim):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-
-def _IsCContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(ndim - 1, -1, -1):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-        
-
- at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fort):
-    """Return 1 if the memory defined by the view is C-style (fortran is
-    'C') or Fortran-style (fortran is 'F') contiguous or either one
-    (fortran is 'A').  Return 0 otherwise."""
-    # traverse the strides, checking for consistent stride increases from
-    # right-to-left (c) or left-to-right (fortran). Copied from cpython
-    if not view.c_suboffsets:
-        return 0
-    if (fort == 'C'):
-        return _IsCContiguous(view)
-    elif (fort == 'F'):
-        return _IsFortranContiguous(view)
-    elif (fort == 'A'):
-        return (_IsCContiguous(view) or _IsFortranContiguous(view))
-    return 0
-
     
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -144,6 +144,7 @@
 
 /* Py3k buffer interface, adapted for PyPy */
 #define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -158,7 +159,7 @@
     Py_ssize_t *shape;
     Py_ssize_t *strides;
     Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format;
+    unsigned char _format[Py_MAX_FMT];
     Py_ssize_t _strides[Py_MAX_NDIMS];
     Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,11 +1,125 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    view.c_obj = make_ref(space, w_obj)
+    return fill_Py_buffer(space, buf, view)
+
+def fill_Py_buffer(space, buf, view):    
+    # c_buf, c_obj have been filled in
+    ndim = buf.getndim()
+    view.c_len = buf.getlength()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        if ord(fmt[i]) > 255:
+            view.c_format[i] = '*'
+        else:
+            view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'        
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+ at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+    """Return 1 if the memory defined by the view is C-style (fortran is
+    'C') or Fortran-style (fortran is 'F') contiguous or either one
+    (fortran is 'A').  Return 0 otherwise."""
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
     return space.call_method(space.builtin, "memoryview", w_obj)
@@ -38,19 +152,6 @@
         view.c_obj = make_ref(space, w_s)
         rffi.setintfield(view, 'c_readonly', 1)
         isstr = True
-    view.c_len = w_obj.getlength()
-    view.c_itemsize = w_obj.buf.getitemsize()
-    rffi.setintfield(view, 'c_ndim', ndim)
-    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
-    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
-    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-    shape = w_obj.buf.getshape()
-    strides = w_obj.buf.getstrides()
-    for i in range(ndim):
-        view.c_shape[i] = shape[i]
-        view.c_strides[i] = strides[i]
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    fill_Py_buffer(space, w_obj.buf, view)     
     return view
 
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
 import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.api import PyObject
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
 
 from .support import c_compile
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0
@@ -53,7 +55,48 @@
         libraries=libraries)
     return soname
 
-def compile_extension_module(space, modname, include_dirs=[],
+class SystemCompilationInfo(object):
+    """Bundles all the generic information required to compile extensions.
+
+    Note: here, 'system' means OS + target interpreter + test config + ...
+    """
+    def __init__(self, include_extra=None, compile_extra=None, link_extra=None,
+            extra_libs=None, ext=None):
+        self.include_extra = include_extra or []
+        self.compile_extra = compile_extra
+        self.link_extra = link_extra
+        self.extra_libs = extra_libs
+        self.ext = ext
+
+def get_cpyext_info(space):
+    from pypy.module.imp.importing import get_so_extension
+    state = space.fromcache(State)
+    api_library = state.api_lib
+    if sys.platform == 'win32':
+        libraries = [api_library]
+        # '%s' undefined; assuming extern returning int
+        compile_extra = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+            (space.int_w(w_maj), space.int_w(w_min))]
+    else:
+        libraries = []
+        if sys.platform.startswith('linux'):
+            compile_extra = [
+                "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
+    return SystemCompilationInfo(
+        include_extra=api.include_dirs,
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        extra_libs=libraries,
+        ext=get_so_extension(space))
+
+
+def compile_extension_module(sys_info, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
@@ -65,35 +108,15 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    state = space.fromcache(State)
-    api_library = state.api_lib
-    if sys.platform == 'win32':
-        libraries = [api_library]
-        # '%s' undefined; assuming extern returning int
-        compile_extra = ["/we4013"]
-        # prevent linking with PythonXX.lib
-        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
-                              (space.int_w(w_maj), space.int_w(w_min))]
-    else:
-        libraries = []
-        if sys.platform.startswith('linux'):
-            compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
-            link_extra = ["-g"]
-        else:
-            compile_extra = link_extra = None
-
     modname = modname.split('.')[-1]
     soname = create_so(modname,
-            include_dirs=api.include_dirs + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra,
-            libraries=libraries)
-    from pypy.module.imp.importing import get_so_extension
-    ext = get_so_extension(space)
-    pydname = soname.new(purebasename=modname, ext=ext)
+        include_dirs=sys_info.include_extra + include_dirs,
+        source_files=source_files,
+        source_strings=source_strings,
+        compile_extra=sys_info.compile_extra,
+        link_extra=sys_info.link_extra,
+        libraries=sys_info.extra_libs)
+    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
     soname.rename(pydname)
     return str(pydname)
 
@@ -106,18 +129,8 @@
         raise RuntimeError("This interpreter does not define a filename "
             "suffix for C extensions!")
 
-def compile_extension_module_applevel(space, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
+def get_sys_info_app():
+    from distutils.sysconfig import get_python_inc
     if sys.platform == 'win32':
         compile_extra = ["/we4013"]
         link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
@@ -128,18 +141,13 @@
         compile_extra = [
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
         link_extra = None
+    ext = get_so_suffix()
+    return SystemCompilationInfo(
+        include_extra=[get_python_inc()],
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        ext=get_so_suffix())
 
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-            include_dirs=[space.include_dir] + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra)
-    ext = get_so_suffix()
-    pydname = soname.new(purebasename=modname, ext=ext)
-    soname.rename(pydname)
-    return str(pydname)
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -154,9 +162,7 @@
 class FakeSpace(object):
     """Like TinyObjSpace, but different"""
     def __init__(self, config):
-        from distutils.sysconfig import get_python_inc
         self.config = config
-        self.include_dir = get_python_inc()
 
     def passthrough(self, arg):
         return arg
@@ -271,11 +277,11 @@
             "the test actually passed in the first place; if it failed "
             "it is likely to reach this place.")
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_only_import(self):
         import cpyext
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_load_error(self):
         import cpyext
         raises(ImportError, cpyext.load_module, "missing.file", "foo")
@@ -303,27 +309,23 @@
     def setup_method(self, func):
         @gateway.unwrap_spec(name=str)
         def compile_module(space, name,
-                           w_separate_module_files=None,
-                           w_separate_module_sources=None):
+                           w_source_files=None,
+                           w_source_strings=None):
             """
             Build an extension module linked against the cpyext api library.
             """
-            if not space.is_none(w_separate_module_files):
-                separate_module_files = space.listview_bytes(
-                    w_separate_module_files)
-                assert separate_module_files is not None
+            if not space.is_none(w_source_files):
+                source_files = space.listview_bytes(w_source_files)
             else:
-                separate_module_files = []
-            if not space.is_none(w_separate_module_sources):
-                separate_module_sources = space.listview_bytes(
-                    w_separate_module_sources)
-                assert separate_module_sources is not None
+                source_files = None
+            if not space.is_none(w_source_strings):
+                source_strings = space.listview_bytes(w_source_strings)
             else:
-                separate_module_sources = []
-            pydname = self.compile_extension_module(
-                space, name,
-                source_files=separate_module_files,
-                source_strings=separate_module_sources)
+                source_strings = None
+            pydname = compile_extension_module(
+                self.sys_info, name,
+                source_files=source_files,
+                source_strings=source_strings)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -376,7 +378,7 @@
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
                 kwds = dict(source_files=[filename])
-            mod = self.compile_extension_module(space, name,
+            mod = compile_extension_module(self.sys_info, name,
                     include_dirs=include_dirs, **kwds)
 
             if load_it:
@@ -468,11 +470,11 @@
                 return run
             def wrap(func):
                 return func
-            self.compile_extension_module = compile_extension_module_applevel
+            self.sys_info = get_sys_info_app()
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
-            self.compile_extension_module = compile_extension_module
+            self.sys_info = get_cpyext_info(self.space)
         self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
         self.w_reimport_module = wrap(interp2app(reimport_module))
@@ -634,10 +636,10 @@
             skip('record_imported_module not supported in runappdirect mode')
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", separate_module_files=[self.here + 'banana.c'])
+            "apple.banana", source_files=[self.here + 'banana.c'])
         self.record_imported_module("apple.banana")
         date = self.compile_module(
-            "cherry.date", separate_module_files=[self.here + 'date.c'])
+            "cherry.date", source_files=[self.here + 'date.c'])
         self.record_imported_module("cherry.date")
 
         # Set up some package state so that the extensions can actually be
@@ -894,7 +896,7 @@
             ])
         raises(SystemError, mod.newexc, "name", Exception, {})
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy specific test')
+    @pytest.mark.skipif(only_pypy, reason='pypy specific test')
     def test_hash_pointer(self):
         mod = self.import_extension('foo', [
             ('get_hash', 'METH_NOARGS',
@@ -945,7 +947,7 @@
         print p
         assert 'py' in p
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_version(self):
         mod = self.import_extension('foo', [
             ('get_version', 'METH_NOARGS',
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,3 +1,4 @@
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
@@ -16,8 +17,12 @@
         w_buf = space.newbuffer(StringBuffer("hello"))
         w_memoryview = api.PyMemoryView_FromObject(w_buf)
         w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
-        ndim = w_view.c_ndim
-        assert ndim == 1
+        assert w_view.c_ndim == 1
+        f = rffi.charp2str(w_view.c_format)
+        assert f == 'B'
+        assert w_view.c_shape[0] == 5
+        assert w_view.c_strides[0] == 1
+        assert w_view.c_len == 5
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):
@@ -38,14 +43,10 @@
         from _numpypy import multiarray as np
         module = self.import_module(name='buffer_test')
         get_buffer_info = module.get_buffer_info
-        # test_export_flags from numpy test_multiarray
         raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
-        # test_relaxed_strides from numpy test_multiarray
-        arr = np.zeros((1, 10))
-        if arr.flags.f_contiguous:
-            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
-            assert strides[0] == 8
-            arr = np.ones((10, 1), order='F')
-            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
-            assert strides[-1] == 8
-
+        arr = np.zeros((1, 10), order='F')
+        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+        assert strides[0] == 8
+        arr = np.zeros((10, 1), order='C')
+        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+        assert strides[-1] == 8
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
         assert module.test_fast_sequence(s[0:-1])
         assert module.test_fast_sequence(s[::-1])
 
+    def test_fast_keyerror(self):
+        module = self.import_extension('foo', [
+            ("test_fast_sequence", "METH_VARARGS",
+             """
+                PyObject *foo;
+                PyObject * seq = PyTuple_GetItem(args, 0);
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "Could not convert object to sequence");
+                if (foo != NULL)
+                {
+                    return foo;
+                }
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                    return PyBool_FromLong(1);
+                }
+                return NULL;
+             """)])
+        class Map(object):
+            def __len__(self):
+                return 1
+
+            def __getitem__(self, index):
+                raise KeyError()
+
+        assert module.test_fast_sequence(Map()) is True
+
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -1,12 +1,13 @@
 import sys
 
-import py, pytest
+import pytest
 
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 class AppTestThread(AppTestCpythonExtensionBase):
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_thread_ident(self):
         module = self.import_extension('foo', [
             ("get_thread_ident", "METH_NOARGS",
@@ -33,7 +34,7 @@
 
         assert results[0][0] != results[1][0]
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_acquire_lock(self):
         module = self.import_extension('foo', [
             ("test_acquire_lock", "METH_NOARGS",
@@ -57,7 +58,7 @@
             ])
         module.test_acquire_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_release_lock(self):
         module = self.import_extension('foo', [
             ("test_release_lock", "METH_NOARGS",
@@ -79,7 +80,7 @@
             ])
         module.test_release_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_tls(self):
         module = self.import_extension('foo', [
             ("create_key", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,6 +3,7 @@
 import py, pytest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 def test_pragma_version():
     from pypy.module.sys.version import CPYTHON_VERSION
@@ -32,11 +33,9 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -358,6 +358,7 @@
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
                     w_fromlist = w_all
+                    length = space.len_w(w_fromlist)
                 else:
                     w_fromlist = None
                     # "from x import *" with x already imported and no x.__all__
@@ -409,6 +410,7 @@
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
                     w_fromlist = w_all
+                    length = space.len_w(w_fromlist)
                 else:
                     w_fromlist = None
             if w_fromlist is not None:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -65,8 +65,9 @@
              )
     setuppkg("pkg.pkg2", a='', b='')
     setuppkg("pkg.withall",
-             __init__  = "__all__ = ['foobar']",
-             foobar    = "found = 123")
+             __init__  = "__all__ = ['foobar', 'barbaz']",
+             foobar    = "found = 123",
+             barbaz    = "other = 543")
     setuppkg("pkg.withoutall",
              __init__  = "",
              foobar    = "found = 123")
@@ -707,6 +708,7 @@
             d = {}
             exec "from pkg.withall import *" in d
             assert d["foobar"].found == 123
+            assert d["barbaz"].other == 543
 
     def test_import_star_does_not_find_submodules_without___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -392,31 +392,39 @@
         extobj_w = space.newlist([space.wrap(8192), space.wrap(0), space.w_None])
         return extobj_w
 
+
+_reflected_ops = {
+        'add': 'radd',
+        'subtract': 'rsub',
+        'multiply': 'rmul',
+        'divide': 'rdiv',
+        'true_divide': 'rtruediv',
+        'floor_divide': 'rfloordiv',
+        'remainder': 'rmod',
+        'power': 'rpow',
+        'left_shift': 'rlshift',
+        'right_shift': 'rrshift',
+        'bitwise_and': 'rand',
+        'bitwise_xor': 'rxor',
+        'bitwise_or': 'ror',
+        #/* Comparisons */
+        'equal': 'eq',
+        'not_equal': 'ne',
+        'greater': 'lt',
+        'less': 'gt',
+        'greater_equal': 'le',
+        'less_equal': 'ge',
+}
+
+for key, value in _reflected_ops.items():
+    _reflected_ops[key] = "__" + value + "__"
+del key
+del value
+
 def _has_reflected_op(space, w_obj, op):
-    refops ={ 'add': 'radd',
-            'subtract': 'rsub',
-            'multiply': 'rmul',
-            'divide': 'rdiv',
-            'true_divide': 'rtruediv',
-            'floor_divide': 'rfloordiv',
-            'remainder': 'rmod',
-            'power': 'rpow',
-            'left_shift': 'rlshift',
-            'right_shift': 'rrshift',
-            'bitwise_and': 'rand',
-            'bitwise_xor': 'rxor',
-            'bitwise_or': 'ror',
-            #/* Comparisons */
-            'equal': 'eq',
-            'not_equal': 'ne',
-            'greater': 'lt',
-            'less': 'gt',
-            'greater_equal': 'le',
-            'less_equal': 'ge',
-        }
-    if op not in refops:
+    if op not in _reflected_ops:
         return False
-    return space.getattr(w_obj, space.wrap('__' + refops[op] + '__')) is not None
+    return space.getattr(w_obj, space.wrap(_reflected_ops[op])) is not None
 
 def safe_casting_mode(casting):
     assert casting is not None
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -364,7 +364,7 @@
         return getattr(self, 'handle_%s_indirect_call' % kind)(op)
 
     def rewrite_call(self, op, namebase, initialargs, args=None,
-                     calldescr=None):
+                     calldescr=None, force_ir=False):
         """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)'
            into 'i0 = xxx_call_ir_i(fn, descr, [i1,i2], [ref1,ref2])'.
            The name is one of '{residual,direct}_call_{r,ir,irf}_{i,r,f,v}'."""
@@ -374,8 +374,9 @@
         lst_i, lst_r, lst_f = self.make_three_lists(args)
         reskind = getkind(op.result.concretetype)[0]
         if lst_f or reskind == 'f': kinds = 'irf'
-        elif lst_i: kinds = 'ir'
+        elif lst_i or force_ir: kinds = 'ir'
         else: kinds = 'r'
+        if force_ir: assert kinds == 'ir'    # no 'f'
         sublists = []
         if 'i' in kinds: sublists.append(lst_i)
         if 'r' in kinds: sublists.append(lst_r)
@@ -1577,7 +1578,7 @@
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
         op1 = self.rewrite_call(op, 'conditional_call',
                                 op.args[:2], args=op.args[2:],
-                                calldescr=calldescr)
+                                calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1200,28 +1200,12 @@
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
     # conditional calls - note that they cannot return stuff
-    @arguments("cpu", "i", "i", "I", "d")
-    def bhimpl_conditional_call_i_v(cpu, condition, func, args_i, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, None, None, calldescr)
-
-    @arguments("cpu", "i", "i", "R", "d")
-    def bhimpl_conditional_call_r_v(cpu, condition, func, args_r, calldescr):
-        if condition:
-            cpu.bh_call_v(func, None, args_r, None, calldescr)
-
     @arguments("cpu", "i", "i", "I", "R", "d")
     def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
                                      calldescr):
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
-    @arguments("cpu", "i", "i", "I", "R", "F", "d")
-    def bhimpl_conditional_call_irf_v(cpu, condition, func, args_i, args_r,
-                                      args_f, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
-
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -365,6 +365,13 @@
     def visitor_dispatch_virtual_type(self, visitor):
         raise NotImplementedError("abstract")
 
+    def make_guards(self, op, short, optimizer):
+        from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
+        op = ResOperation(rop.INT_EQ, [op, CONST_0])
+        short.append(op)
+        op = ResOperation(rop.GUARD_FALSE, [op])
+        short.append(op)
+
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7592,7 +7592,7 @@
         ops = """
         [i0]
         p1 = new_with_vtable(descr=nodesize)
-        cond_call(1, 123, p1, descr=clear_vable)
+        cond_call(i0, 123, p1, descr=clear_vable)
         jump(i0)
         """
         expected = """
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,7 +16,7 @@
 from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
      have_debug_prints
 
-class UnrollableOptimizer(Optimizer):    
+class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             if self.optunroll.short_preamble_producer is None:
@@ -120,7 +120,8 @@
                 assert op.get_forwarded() is None
         if check_newops:
             assert not self.optimizer._newoperations
-    
+
+
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
@@ -156,7 +157,7 @@
         current_vs = self.get_virtual_state(end_jump.getarglist())
         # pick the vs we want to jump to
         assert isinstance(celltoken, JitCellToken)
-        
+
         target_virtual_state = self.pick_virtual_state(current_vs,
                                                        state.virtual_state,
                                                 celltoken.target_tokens)
@@ -180,17 +181,27 @@
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)            
+                    self.optimizer._newoperations)
 
         try:
-            new_virtual_state = self.jump_to_existing_trace(end_jump, label_op,
-                                                            state.runtime_boxes)
+            new_virtual_state = self.jump_to_existing_trace(
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
                     self.optimizer._newoperations)
+
+        if new_virtual_state is not None:
+            # Attempt to force virtual boxes in order to avoid jumping
+            # to the preamble.
+            try:
+                new_virtual_state = self.jump_to_existing_trace(
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+            except InvalidLoop:
+                pass
+
         if new_virtual_state is not None:
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
@@ -199,7 +210,7 @@
 
         self.disable_retracing_if_max_retrace_guards(
             self.optimizer._newoperations, target_token)
-        
+
         return (UnrollInfo(target_token, label_op, extra_same_as,
                            self.optimizer.quasi_immutable_deps),
                 self.optimizer._newoperations)
@@ -241,7 +252,8 @@
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
         try:
-            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes)
+            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                             force_boxes=False)
         except InvalidLoop:
             return self.jump_to_preamble(cell_token, jump_op, info)
         if vs is None:
@@ -252,6 +264,14 @@
             cell_token.retraced_count += 1
             debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
         else:
+            # Try forcing boxes to avoid jumping to the preamble
+            try:
+                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                                 force_boxes=True)
+            except InvalidLoop:
+                pass
+            if vs is None:
+                return info, self.optimizer._newoperations[:]
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
@@ -288,7 +308,7 @@
         return info, self.optimizer._newoperations[:]
 
 
-    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes):
+    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False):
         jitcelltoken = jump_op.getdescr()
         assert isinstance(jitcelltoken, JitCellToken)
         virtual_state = self.get_virtual_state(jump_op.getarglist())
@@ -299,7 +319,8 @@
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, args, runtime_boxes, self.optimizer)
+                    virtual_state, args, runtime_boxes, self.optimizer,
+                    force_boxes=force_boxes)
                 patchguardop = self.optimizer.patchguardop
                 for guard in extra_guards.extra_guards:
                     if isinstance(guard, GuardResOp):
@@ -308,8 +329,18 @@
                     self.send_extra_operation(guard)
             except VirtualStatesCantMatch:
                 continue
-            args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
-                args, self.optimizer)
+
+            # When force_boxes == True, creating the virtual args can fail when
+            # components of the virtual state alias. If this occurs, we must
+            # recompute the virtual state as boxes will have been forced.
+            try:
+                args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
+                    args, self.optimizer, force_boxes=force_boxes)
+            except VirtualStatesCantMatch:
+                assert force_boxes
+                virtual_state = self.get_virtual_state(args)
+                continue
+
             short_preamble = target_token.short_preamble
             try:
                 extra = self.inline_short_preamble(args + virtuals, args,
@@ -452,7 +483,7 @@
         # by short preamble
         label_args = exported_state.virtual_state.make_inputargs(
             targetargs, self.optimizer)
-        
+
         self.short_preamble_producer = ShortPreambleBuilder(
             label_args, exported_state.short_boxes,
             exported_state.short_inputargs, exported_state.exported_infos,
@@ -497,7 +528,7 @@
     * runtime_boxes - runtime values for boxes, necessary when generating
                       guards to jump to
     """
-    
+
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
                  short_inputargs, runtime_boxes, memo):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
      ArrayStructInfo, AbstractStructPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound, IntLowerBound
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation, \
      InputArgInt, InputArgRef, InputArgFloat
 from rpython.rlib.debug import debug_print
 
@@ -20,7 +20,7 @@
 
 
 class GenerateGuardState(object):
-    def __init__(self, optimizer=None, guards=None, renum=None, bad=None):
+    def __init__(self, optimizer=None, guards=None, renum=None, bad=None, force_boxes=False):
         self.optimizer = optimizer
         self.cpu = optimizer.cpu
         if guards is None:
@@ -32,6 +32,7 @@
         if bad is None:
             bad = {}
         self.bad = bad
+        self.force_boxes = force_boxes
 
     def get_runtime_item(self, box, descr, i):
         array = box.getref_base()
@@ -303,7 +304,7 @@
             opinfo = state.optimizer.getptrinfo(box)
             assert isinstance(opinfo, ArrayPtrInfo)
         else:
-            opinfo = None            
+            opinfo = None
         for i in range(self.length):
             for descr in self.fielddescrs:
                 index = i * len(self.fielddescrs) + descr.get_index()
@@ -514,6 +515,8 @@
         NotVirtualStateInfo.__init__(self, cpu, type, info)
 
     def _generate_guards(self, other, box, runtime_box, state):
+        if state.force_boxes and isinstance(other, VirtualStateInfo):
+            return self._generate_virtual_guards(other, box, runtime_box, state)
         if not isinstance(other, NotVirtualStateInfoPtr):
             raise VirtualStatesCantMatch(
                     'The VirtualStates does not match as a ' +
@@ -545,6 +548,23 @@
     # to an existing compiled loop or retracing the loop. Both alternatives
     # will always generate correct behaviour, but performance will differ.
 
+    def _generate_virtual_guards(self, other, box, runtime_box, state):
+        """
+        Generate the guards and add state information for unifying a virtual
+        object with a non-virtual. This involves forcing the object in the
+        event that unification can succeed. Since virtual objects cannot be null,
+        this method need only check that the virtual object has the expected type.
+        """
+        assert state.force_boxes and isinstance(other, VirtualStateInfo)
+
+        if self.level == LEVEL_CONSTANT:
+            raise VirtualStatesCantMatch(
+                    "cannot unify a constant value with a virtual object")
+
+        if self.level == LEVEL_KNOWNCLASS:
+            if not self.known_class.same_constant(other.known_class):
+                raise VirtualStatesCantMatch("classes don't match")
+
     def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
                                  state):
         if not isinstance(other, NotVirtualStateInfoPtr):
@@ -617,10 +637,10 @@
             return False
         return True
 
-    def generate_guards(self, other, boxes, runtime_boxes, optimizer):
+    def generate_guards(self, other, boxes, runtime_boxes, optimizer, force_boxes=False):
         assert (len(self.state) == len(other.state) == len(boxes) ==
                 len(runtime_boxes))
-        state = GenerateGuardState(optimizer)
+        state = GenerateGuardState(optimizer, force_boxes=force_boxes)
         for i in range(len(self.state)):
             self.state[i].generate_guards(other.state[i], boxes[i],
                                           runtime_boxes[i], state)
@@ -644,8 +664,8 @@
 
         return boxes
 
-    def make_inputargs_and_virtuals(self, inputargs, optimizer):
-        inpargs = self.make_inputargs(inputargs, optimizer)
+    def make_inputargs_and_virtuals(self, inputargs, optimizer, force_boxes=False):
+        inpargs = self.make_inputargs(inputargs, optimizer, force_boxes)
         # we append the virtuals here in case some stuff is proven
         # to be not a virtual and there are getfields in the short preamble
         # that will read items out of there
@@ -653,7 +673,7 @@
         for i in range(len(inputargs)):
             if not isinstance(self.state[i], NotVirtualStateInfo):
                 virtuals.append(inputargs[i])
-            
+
         return inpargs, virtuals
 
     def debug_print(self, hdr='', bad=None, metainterp_sd=None):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1056,23 +1056,11 @@
     opimpl_residual_call_irf_f = _opimpl_residual_call3
     opimpl_residual_call_irf_v = _opimpl_residual_call3
 
-    @arguments("box", "box", "boxes", "descr", "orgpc")
-    def opimpl_conditional_call_i_v(self, condbox, funcbox, argboxes, calldescr,
-                                    pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
-
-    opimpl_conditional_call_r_v = opimpl_conditional_call_i_v
-
     @arguments("box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, funcbox, argboxes,
                                      calldescr, pc):
         self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
 
-    @arguments("box", "box", "boxes3", "descr", "orgpc")
-    def opimpl_conditional_call_irf_v(self, condbox, funcbox, argboxes,
-                                      calldescr, pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
-
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
         targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4507,3 +4507,54 @@
                 i += 1
             return i
         self.meta_interp(f, [])
+
+    def test_round_trip_raw_pointer(self):
+        # The goal of this test to to get a raw pointer op into the short preamble
+        # so we can check that the proper guards are generated
+        # In this case, the resulting short preamble contains
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # i2 = int_eq(i1, 0)
+        # guard_false(i2)
+        #
+        # as opposed to what the JIT used to produce
+        #
+        # i1 = getfield_gc_i(p0, descr=inst__ptr)
+        # guard_nonnull(i1)
+        #
+        # Which will probably generate correct assembly, but the optimization
+        # pipline expects guard_nonnull arguments to be pointer ops and may crash
+        # and may crash on other input types.
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+
+        class Box(object):
+            _ptr = lltype.nullptr(rffi.CCHARP.TO)
+
+        def new_int_buffer(value):
+            data = lltype.malloc(rffi.CCHARP.TO, rffi.sizeof(rffi.INT), flavor='raw')
+            rffi.cast(rffi.INTP, data)[0] = rffi.cast(rffi.INT, value)
+            return data
+
+        def read_int_buffer(buf):
+            return rffi.cast(rffi.INTP, buf)[0]
+
+        def f():
+            i = 0
+            val = Box()
+            val._ptr = new_int_buffer(1)
+
+            set_param(None, 'retrace_limit', -1)
+            while i < 100:
+                driver.jit_merge_point(i=i, val=val)
+                driver.can_enter_jit(i=i, val=val)
+                # Just to produce a side exit
+                if i & 0b100:
+                    i += 1
+                i += int(read_int_buffer(val._ptr))
+                lltype.free(val._ptr, flavor='raw')
+                val._ptr = new_int_buffer(1)
+            lltype.free(val._ptr, flavor='raw')
+
+        self.meta_interp(f, [])
+        self.check_resops(guard_nonnull=0)
+
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1,8 +1,8 @@
 import py
-from rpython.rlib.jit import JitDriver, promote, dont_look_inside
+from rpython.rlib.jit import JitDriver, promote, dont_look_inside, set_param
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.jit.codewriter.policy import StopAtXPolicy
-from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.test.support import LLJitMixin, get_stats
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper import rclass
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -965,6 +965,82 @@
         self.check_aborted_count(0)
         self.check_target_token_count(4)
 
+    def test_avoid_preamble(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        class Box(object):
+            def __init__(self, v):
+                self.unbox = v
+
+        mask = -2
+        const = Box(X(5))
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            val   = X(0)
+            i     = 0
+            const.unbox = X(5)
+            while i < 17:
+                driver.can_enter_jit(i=i, val=val)
+                driver.jit_merge_point(i=i, val=val)
+                # Logical & rather than comparison to confuse range analysis.
+                # Test only succeeds on the first 2 iterations
+                if i & -2 == 0:
+                    val = const.unbox
+                else:
+                    val = X(i)
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
+
+        # With retracing disable, there will be one optimized loop expecting a
+        # non-virtual X object. The side exit creates a virtual object which must
+        # be allocated to jump to the optimized trace.
+        self.check_resops(jump=3, label=2, new_with_vtable=2)
+        self.check_target_token_count(2)
+        self.check_trace_count(3)
+
+    def test_conflated_virtual_states(self):
+        # All cases are covered when forcing one component of the virtual state
+        # also forces an as yet unseen component.
+        # i.e. expect [NotVirtual, Virtual] and given a pair of aliasing virtual
+        # objects
+        driver = JitDriver(greens=[], reds=['i', 'v1', 'v2'])
+        class Box(object):
+            def __init__(self, v):
+                self.v = v
+
+        class X(object):
+            def __init__(self, v):
+                self.v = v
+
+        const = Box(X(0))
+        def f():
+            set_param(None, 'retrace_limit', -1)
+            set_param(None, 'threshold', 1)
+            i = 0
+            v1 = X(0)
+            v2 = X(0)
+            const.v = X(0)
+            while i < 17:
+                driver.jit_merge_point(i=i, v1=v1, v2=v2)
+                driver.can_enter_jit(i=i, v1=v1, v2=v2)
+                if i & 1 == 0:
+                    v1 = const.v
+                    v2 = X(i)
+                else:
+                    v1 = v2 = X(i)
+                i += 1
+            return None
+        self.meta_interp(f, [])
+        # assert did not crash
+
 
 class VirtualMiscTests:
     def test_multiple_equal_virtuals(self):
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,7 +97,8 @@
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
-if OPENSSL_VERSION_NUMBER >= 0x10100000:
+if (OPENSSL_VERSION_NUMBER >= 0x10100000 and
+     OPENSSL_VERSION_NUMBER < 0x20000000):    # <= libressl :-(
     eci.pre_include_bits = ()
     eci.post_include_bits = ()
     raise Exception("""OpenSSL version >= 1.1 not supported yet.

From pypy.commits at gmail.com  Mon Sep 12 08:28:31 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 12 Sep 2016 05:28:31 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: resolve issues in test_schedule
 introduced by vec_load/store
Message-ID: <57d69f6f.ca11c30a.cfcd6.d32e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87039:29d7e657b3af
Date: 2016-09-12 14:27 +0200
http://bitbucket.org/pypy/pypy/changeset/29d7e657b3af/

Log:	resolve issues in test_schedule introduced by vec_load/store

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -6,6 +6,7 @@
 from rpython.jit.metainterp.optimizeopt.vector import (VecScheduleState,
         Pack, Pair, NotAProfitableLoop, VectorizingOptimizer, X86_CostModel,
         PackSet)
+from rpython.jit.backend.llsupport.vector_ext import VectorExt
 from rpython.jit.metainterp.optimizeopt.dependency import Node, DependencyGraph
 from rpython.jit.metainterp.optimizeopt.schedule import Scheduler
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
@@ -33,6 +34,10 @@
             'char': self.chararraydescr,
         }
 
+    def setup_method(self, name):
+        self.vector_ext = VectorExt()
+        self.vector_ext.enable(16, True)
+
     def pack(self, loop, l, r, input_type=None, output_type=None):
         return Pack(loop.graph.nodes[l:r])
 
@@ -92,7 +97,7 @@
         ps.packs = [self.pack(loop1, 0, 3)]
         op1 = ps.packs[0].operations[0]
         op2 = ps.packs[0].operations[1]
-        ps.split_overloaded_packs()
+        ps.split_overloaded_packs(self.vector_ext)
         assert len(ps.packs) == 1
         assert ps.packs[0].leftmost() is op1.getoperation()
         assert ps.packs[0].rightmost() is op2.getoperation()
@@ -109,7 +114,7 @@
         pack1 = self.pack(loop1, 0, 6)
         loop2 = self.schedule(loop1, [pack1])
         loop3 = self.parse_trace("""
-        v10[4xi32] = vec_raw_load_f(p0, i0, descr=float)
+        v10[4xi32] = vec_load_f(p0, i0, 1, 0, descr=float)
         f10 = raw_load_f(p0, i4, descr=float)
         f11 = raw_load_f(p0, i5, descr=float)
         """, False)
@@ -129,7 +134,7 @@
         pack3 = self.pack(loop1, 4, 6)
         loop2 = self.schedule(loop1, [pack1, pack2, pack3])
         loop3 = self.parse_trace("""
-        v10[2xi64] = vec_raw_load_i(p0, i0, descr=long)
+        v10[2xi64] = vec_load_i(p0, i0, 1, 0, descr=long)
         v20[2xi32] = vec_int_signext(v10[2xi64], 4)
         v30[2xf64] = vec_cast_int_to_float(v20[2xi32])
         """, False)
@@ -266,10 +271,10 @@
                                   '_prevent_signext': void
                               })
         loop3 = self.parse_trace("""
-        v10[2xf64] = vec_raw_load_f(p0, i1, descr=double)
-        v11[2xf64] = vec_raw_load_f(p0, i3, descr=double)
-        v12[2xf64] = vec_raw_load_f(p0, i5, descr=double)
-        v13[2xf64] = vec_raw_load_f(p0, i7, descr=double)
+        v10[2xf64] = vec_load_f(p0, i1, 1, 0, descr=double)
+        v11[2xf64] = vec_load_f(p0, i3, 1, 0, descr=double)
+        v12[2xf64] = vec_load_f(p0, i5, 1, 0, descr=double)
+        v13[2xf64] = vec_load_f(p0, i7, 1, 0, descr=double)
         v14[2xi32] = vec_cast_float_to_int(v10[2xf64])
         v15[2xi32] = vec_cast_float_to_int(v11[2xf64])
         v16[2xi32] = vec_cast_float_to_int(v12[2xf64])
@@ -279,7 +284,7 @@
         v23[6xi16] = vec_pack_i(v16[2xi32], v17[2xi32], 2, 2)
         v20[4xi16] = vec_int_signext(v23[4xi32],2)
         v24[8xi16] = vec_pack_i(v18[4xi16], v20[4xi16], 4, 4)
-        vec_raw_store(p1, i1, v24[8xi16], descr=short)
+        vec_store(p1, i1, v24[8xi16], 1, 0, descr=short)
         """, False)
         self.assert_equal(loop2, loop3)
 
@@ -305,12 +310,12 @@
         pack3 = self.pack(loop1, 8, 12)
         loop2 = self.schedule(loop1, [pack1,pack2,pack3])
         loop3 = self.parse_trace("""
-        v44[2xf64] = vec_raw_load_f(p0, i1, descr=double) 
-        v45[2xf64] = vec_raw_load_f(p0, i3, descr=double) 
+        v44[2xf64] = vec_load_f(p0, i1, 1, 0, descr=double) 
+        v45[2xf64] = vec_load_f(p0, i3, 1, 0, descr=double) 
         v46[2xi32] = vec_cast_float_to_singlefloat(v44[2xf64]) 
         v47[2xi32] = vec_cast_float_to_singlefloat(v45[2xf64]) 
         v41[4xi32] = vec_pack_i(v46[2xi32], v47[2xi32], 2, 2) 
-        vec_raw_store(p1, i1, v41[4xi32], descr=float)
+        vec_store(p1, i1, v41[4xi32], 1, 0, descr=float)
         """, False)
         self.assert_equal(loop2, loop3)
 
@@ -331,7 +336,7 @@
         loop2 = self.schedule(loop1, [pack1,pack2,pack3], prepend_invariant=True)
         loop3 = self.parse_trace("""
         v9[2xi64] = vec_expand_i(255)
-        v10[2xi64] = vec_raw_load_i(p0, i1, descr=long)
+        v10[2xi64] = vec_load_i(p0, i1, 1, 0, descr=long)
         v11[2xi64] = vec_int_and(v10[2xi64], v9[2xi64])
         vec_guard_true(v11[2xi64]) []
         """, False)
@@ -350,7 +355,7 @@
         pack2 = self.pack(loop1, 4, 6)
         loop2 = self.schedule(loop1, [pack1,pack2], prepend_invariant=True)
         loop3 = self.parse_trace("""
-        v1[4xi32] = vec_raw_load_i(p0, i1, descr=float)
+        v1[4xi32] = vec_load_i(p0, i1, 1, 0, descr=float)
         i10 = vec_unpack_i(v1[4xi32], 0, 1)
         raw_store(p0, i3, i10, descr=float)
         i11 = vec_unpack_i(v1[4xi32], 1, 1)
@@ -426,7 +431,7 @@
         """)
         pack = self.pack(trace, 0, 4)
         packs = []
-        pack.split(packs, 16)
+        pack.split(packs, 16, self.vector_ext)
         packs.append(pack)
         assert len(packs) == 2
 
@@ -438,7 +443,7 @@
         """)
         pack = self.pack(trace, 0, 2)
         packset = FakePackSet([pack])
-        packset.split_overloaded_packs()
+        packset.split_overloaded_packs(self.vector_ext)
         assert len(packset.packs) == 1
 
     def test_expand(self):

From pypy.commits at gmail.com  Mon Sep 12 08:57:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 12 Sep 2016 05:57:47 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: renamed costmodel class to
 generic one (it is used also for ppc, not only for x86)
Message-ID: <57d6a64b.28eac20a.666bd.d3b0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87040:15528006be4d
Date: 2016-09-12 14:57 +0200
http://bitbucket.org/pypy/pypy/changeset/15528006be4d/

Log:	renamed costmodel class to generic one (it is used also for ppc, not
	only for x86)

diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -268,6 +268,7 @@
         if user_code:
             self.eliminate_array_bound_checks(info, loop)
 
+
     def emit_operation(self, op):
         self.renamer.rename(op)
         self._newoperations.append(op)
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -58,11 +58,13 @@
                 del needs_resolving[op]
                 if op in indexvars:
                     indexvar = indexvars[op]
+                    last = None
                     for operation in indexvar.get_operations():
                         self.append_to_oplist(operation)
                         last = operation
-                    indexvars[last] = indexvar
-                    self.renamer.start_renaming(op, last)
+                    if last:
+                        indexvars[last] = indexvar
+                        self.renamer.start_renaming(op, last)
                 else: 
                     self.resolve_delayed(needs_resolving, delayed, op)
                     self.append_to_oplist(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -2,7 +2,7 @@
 
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.metainterp.optimizeopt.vector import (Pack, X86_CostModel,
+from rpython.jit.metainterp.optimizeopt.vector import (Pack, GenericCostModel,
         NotAProfitableLoop, VectorizingOptimizer, CostModel)
 from rpython.jit.metainterp.optimizeopt.schedule import VecScheduleState
 from rpython.jit.metainterp.optimizeopt.dependency import Node, DependencyGraph
@@ -101,7 +101,7 @@
             print "pack: \n   ",
             print '\n    '.join([str(op.getoperation()) for op in pack.operations])
             print
-        costmodel = FakeCostModel(X86_CostModel(self.cpu, 0))
+        costmodel = FakeCostModel(GenericCostModel(self.cpu, 0))
         costmodel.reset_savings()
         state = VecScheduleState(graph, opt.packset, self.cpu, costmodel)
         opt.schedule(state)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
 from rpython.jit.metainterp.optimizeopt.renamer import Renamer
 from rpython.jit.metainterp.optimizeopt.vector import (VecScheduleState,
-        Pack, Pair, NotAProfitableLoop, VectorizingOptimizer, X86_CostModel,
+        Pack, Pair, NotAProfitableLoop, VectorizingOptimizer, GenericCostModel,
         PackSet)
 from rpython.jit.backend.llsupport.vector_ext import VectorExt
 from rpython.jit.metainterp.optimizeopt.dependency import Node, DependencyGraph
@@ -43,7 +43,7 @@
 
     def schedule(self, loop, packs, vec_reg_size=16,
                  prepend_invariant=False, overwrite_funcs=None):
-        cm = X86_CostModel(self.cpu, 0)
+        cm = GenericCostModel(self.cpu, 0)
         cm.profitable = lambda: True
         pairs = []
         for pack in packs:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -13,7 +13,7 @@
 from rpython.jit.metainterp.optimizeopt.dependency import DependencyGraph
 from rpython.jit.metainterp.optimizeopt.vector import (VectorizingOptimizer,
         MemoryRef, isomorphic, Pair, NotAVectorizeableLoop, VectorLoop,
-        NotAProfitableLoop, GuardStrengthenOpt, CostModel, X86_CostModel,
+        NotAProfitableLoop, GuardStrengthenOpt, CostModel, GenericCostModel,
         PackSet, optimize_vector)
 from rpython.jit.metainterp.optimizeopt.schedule import (Scheduler,
         SchedulerState, VecScheduleState, Pack)
@@ -174,6 +174,14 @@
         if with_guard_opt:
             gso = GuardStrengthenOpt(graph.index_vars)
             gso.propagate_all_forward(info, loop)
+        # re-schedule
+        #vsize = self.cpu.vector_ext.vec_size()
+        #graph = DependencyGraph(loop)
+        #state = VecScheduleState(graph, PackSet(vsize), self.cpu, costmodel)
+        #state.prepare()
+        #scheduler = Scheduler()
+        #scheduler.walk_and_emit(state)
+        #state.post_schedule()
         return opt
 
     def vectorize(self, loop, unroll_factor = -1):
@@ -183,13 +191,23 @@
         opt.find_adjacent_memory_refs(graph)
         opt.extend_packset()
         opt.combine_packset()
-        costmodel = X86_CostModel(self.cpu, 0)
+        costmodel = GenericCostModel(self.cpu, 0)
         state = VecScheduleState(graph, opt.packset, self.cpu, costmodel)
         opt.schedule(state)
         if not costmodel.profitable():
             raise NotAProfitableLoop()
         gso = GuardStrengthenOpt(graph.index_vars)
         gso.propagate_all_forward(info, loop)
+        #
+        # re-schedule
+        #vsize = self.cpu.vector_ext.vec_size()
+        #graph = DependencyGraph(loop)
+        #state = VecScheduleState(graph, PackSet(vsize), self.cpu, costmodel)
+        #state.prepare()
+        #scheduler = Scheduler()
+        #scheduler.walk_and_emit(state)
+        #state.post_schedule()
+        #
         oplist = loop.operations
         loop.operations = loop.prefix[:]
         if loop.prefix_label:
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -133,9 +133,7 @@
         #
         start = time.clock()
         opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, warmstate.vec_cost)
-        index_vars = opt.run_optimization(metainterp_sd, info, loop)
-        gso = GuardStrengthenOpt(index_vars)
-        gso.propagate_all_forward(info, loop, user_code)
+        opt.run_optimization(metainterp_sd, info, loop)
         end = time.clock()
         #
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
@@ -254,13 +252,23 @@
         self.find_adjacent_memory_refs(graph)
         self.extend_packset()
         self.combine_packset()
-        # TODO move cost model to CPU
-        costmodel = X86_CostModel(self.cpu, self.cost_threshold)
+        costmodel = GenericCostModel(self.cpu, self.cost_threshold)
         state = VecScheduleState(graph, self.packset, self.cpu, costmodel)
         self.schedule(state)
         if not state.profitable():
             raise NotAProfitableLoop
-        return graph.index_vars
+        gso = GuardStrengthenOpt(graph.index_vars)
+        gso.propagate_all_forward(info, loop, user_code)
+
+        # re-schedule the trace -> removes index operations
+        # work in progress
+        #graph = DependencyGraph(loop)
+        #costmodel = GenericCostModel(self.cpu, self.cost_threshold)
+        #state = VecScheduleState(graph, PackSet(vsize), self.cpu, costmodel)
+        #state.prepare()
+        #scheduler = Scheduler()
+        #scheduler.walk_and_emit(state)
+        #state.post_schedule()
 
     def unroll_loop_iterations(self, loop, unroll_count):
         """ Unroll the loop X times. unroll_count + 1 = unroll_factor """
@@ -600,7 +608,7 @@
     def profitable(self):
         return self.savings >= 0
 
-class X86_CostModel(CostModel):
+class GenericCostModel(CostModel):
     def record_pack_savings(self, pack, times):
         cost, benefit_factor = (1,1)
         node = pack.operations[0]

From pypy.commits at gmail.com  Mon Sep 12 09:58:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 06:58:49 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: oops
Message-ID: <57d6b499.43921c0a.2ecee.2720@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87041:8be6ad8bbb8d
Date: 2016-09-12 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/8be6ad8bbb8d/

Log:	oops

diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -776,7 +776,7 @@
 for i in range(2):
     OPERATIONS.append(GuardClassOperation(rop.GUARD_CLASS))
     OPERATIONS.append(CondCallOperation(rop.COND_CALL))
-    OPERATIONS.append(CondCallValueOperation(rop.COND_CALL_VALUE_I))
+    OPERATIONS.append(CondCallOperation(rop.COND_CALL_VALUE_I))
     OPERATIONS.append(RaisingCallOperation(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationGuardNoException(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationWrongGuardException(rop.CALL_N))

From pypy.commits at gmail.com  Mon Sep 12 10:03:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 07:03:44 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: fix tests
Message-ID: <57d6b5c0.898b1c0a.2c724.2f75@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87042:97042f59665b
Date: 2016-09-12 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/97042f59665b/

Log:	fix tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,7 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(ll_strhash), p10, descr=<Calli . r EF=0>)
+            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -195,7 +195,8 @@
                            'new_with_vtable': 2, 'getinteriorfield_gc_i': 2,
                            'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
                            'call_i': 6, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
-                           'guard_no_exception': 8, 'new': 2})
+                           'guard_no_exception': 8, 'new': 2,
+                           'guard_nonnull': 2})
 
     def test_unrolling_of_dict_iter(self):
         driver = JitDriver(greens = [], reds = ['n'])

From pypy.commits at gmail.com  Mon Sep 12 10:32:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 07:32:15 -0700 (PDT)
Subject: [pypy-commit] pypy conditional_call_value_3: close branch,
 ready to merge: gives 11% improvements on microbenchmarks
Message-ID: <57d6bc6f.43921c0a.2ecee.3494@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_3
Changeset: r87043:27150117815e
Date: 2016-09-12 16:29 +0200
http://bitbucket.org/pypy/pypy/changeset/27150117815e/

Log:	close branch, ready to merge: gives 11% improvements on
	microbenchmarks


From pypy.commits at gmail.com  Mon Sep 12 10:32:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 07:32:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge conditional_call_value_3
Message-ID: <57d6bc71.452ec20a.bd23e.fb4d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87044:6305cfb3bad2
Date: 2016-09-12 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/6305cfb3bad2/

Log:	hg merge conditional_call_value_3

	JIT residual calls: if the called function starts with a fast-path
	like "if x.foo != 0: return x.foo", then inline the check before
	doing the CALL.

	Right now only implemented on the x86 backend. Other backends
	specify supports_cond_call_value = False.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,7 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(ll_strhash), p10, descr=<Calli . r EF=0>)
+            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1002,6 +1002,9 @@
     prepare_op_cond_call_gc_wb_array = prepare_op_cond_call_gc_wb
 
     def prepare_op_cond_call(self, op, fcond):
+        # XXX don't force the arguments to be loaded in specific
+        # locations before knowing if we can take the fast path
+        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -325,6 +325,7 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
+    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_extension = True
@@ -1334,6 +1335,16 @@
         # cond_call can't have a return value
         self.execute_call_n(calldescr, func, *args)
 
+    def execute_cond_call_value_i(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_i(calldescr, func, *args)
+        return value
+
+    def execute_cond_call_value_r(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_r(calldescr, func, *args)
+        return value
+
     def _execute_call(self, calldescr, func, *args):
         effectinfo = calldescr.get_extra_info()
         if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'):
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -759,6 +759,8 @@
         if (opnum != rop.GUARD_TRUE and opnum != rop.GUARD_FALSE
                                     and opnum != rop.COND_CALL):
             return False
+        # NB: don't list COND_CALL_VALUE_I/R here, these two variants
+        # of COND_CALL don't accept a cc as input
         if next_op.getarg(0) is not op:
             return False
         if self.longevity[op][1] > i + 1:
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -11,7 +11,7 @@
 from rpython.jit.backend.llsupport.symbolic import (WORD,
         get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER
+     FLAG_POINTER, CallDescr
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -370,7 +370,9 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- call assembler -----------
+            # ---------- calls -----------
+            if OpHelpers.is_plain_call(op.getopnum()):
+                self.expand_call_shortcut(op)
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -616,6 +618,30 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
+    def expand_call_shortcut(self, op):
+        if not self.cpu.supports_cond_call_value:
+            return
+        descr = op.getdescr()
+        if descr is None:
+            return
+        assert isinstance(descr, CallDescr)
+        effectinfo = descr.get_extra_info()
+        if effectinfo is None or effectinfo.call_shortcut is None:
+            return
+        if op.type == 'r':
+            cond_call_opnum = rop.COND_CALL_VALUE_R
+        elif op.type == 'i':
+            cond_call_opnum = rop.COND_CALL_VALUE_I
+        else:
+            return
+        cs = effectinfo.call_shortcut
+        ptr_box = op.getarg(1 + cs.argnum)
+        value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
+                                       raw=(ptr_box.type == 'i'))
+        self.replace_op_with(op, ResOperation(cond_call_opnum,
+                                              [value_box] + op.getarglist(),
+                                              descr=descr))
+
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,7 +1,8 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr
+     SizeDescr, get_interiorfield_descr, get_call_descr
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -80,6 +81,14 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
+        call_shortcut = CallShortcut(0, tzdescr)
+        effectinfo = EffectInfo(None, None, None, None, None, None,
+                                EffectInfo.EF_RANDOM_EFFECTS,
+                                call_shortcut=call_shortcut)
+        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
+            [lltype.Ptr(T)], lltype.Signed,
+            effectinfo)
+        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -200,6 +209,7 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
+    supports_cond_call_value = True
 
     translate_support_code = None
 
@@ -1429,3 +1439,15 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
+
+    def test_handle_call_shortcut(self):
+        self.check_rewrite("""
+            [p0]
+            i1 = call_i(123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """, """
+            [p0]
+            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
+            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,6 +16,7 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
+    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2389,7 +2389,7 @@
             f2 = longlong.getfloatstorage(3.4)
             frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5, f1, f2)
             assert not called
-            for j in range(5):
+            for j in range(6):
                 assert self.cpu.get_int_value(frame, j) == j
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 6)) == 1.2
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 3.4
@@ -2447,6 +2447,54 @@
                                            67, 89)
             assert called == [(67, 89)]
 
+    def test_cond_call_value(self):
+        if not self.cpu.supports_cond_call_value:
+            py.test.skip("missing supports_cond_call_value")
+
+        def func_int(*args):
+            called.append(args)
+            return len(args) * 100 + 1000
+
+        for i in range(5):
+            called = []
+
+            FUNC = self.FuncType([lltype.Signed] * i, lltype.Signed)
+            func_ptr = llhelper(lltype.Ptr(FUNC), func_int)
+            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                             EffectInfo.MOST_GENERAL)
+
+            ops = '''
+            [i0, i1, i2, i3, i4, i5, i6, f0, f1]
+            i15 = cond_call_value_i(i1, ConstClass(func_ptr), %s)
+            guard_false(i0, descr=faildescr) [i1,i2,i3,i4,i5,i6,i15, f0,f1]
+            finish(i15)
+            ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] +
+                            ["descr=calldescr"])
+            loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
+                                         'func_ptr': func_ptr,
+                                         'calldescr': calldescr})
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+            f1 = longlong.getfloatstorage(1.2)
+            f2 = longlong.getfloatstorage(3.4)
+            frame = self.cpu.execute_token(looptoken, 1, 50, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert not called
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        50, 1, 2, 3, 4, 5, 50]
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 8)) == 3.4
+            #
+            frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert called == [(1, 2, 3, 4)[:i]]
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        0, 1, 2, 3, 4, 5, i * 100 + 1000]
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 8)) == 3.4
+
     def test_force_operations_returning_void(self):
         values = []
         def maybe_force(token, flag):
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -594,7 +594,7 @@
         return subset, d['f'], vtableptr
 
     def getresulttype(self):
-        if self.opnum == rop.CALL_I:
+        if self.opnum == rop.CALL_I or self.opnum == rop.COND_CALL_VALUE_I:
             return lltype.Signed
         elif self.opnum == rop.CALL_F:
             return lltype.Float
@@ -712,7 +712,12 @@
 class CondCallOperation(BaseCallOperation):
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
-        v_cond = builder.get_bool_var(r)
+        if self.opnum == rop.COND_CALL:
+            RESULT_TYPE = lltype.Void
+            v_cond = builder.get_bool_var(r)
+        else:
+            RESULT_TYPE = lltype.Signed
+            v_cond = r.choice(builder.intvars)
         subset = builder.subset_of_intvars(r)[:4]
         for i in range(len(subset)):
             if r.random() < 0.35:
@@ -724,8 +729,10 @@
                 seen.append(args)
             else:
                 assert seen[0] == args
+            if RESULT_TYPE is lltype.Signed:
+                return len(args) - 42000
         #
-        TP = lltype.FuncType([lltype.Signed] * len(subset), lltype.Void)
+        TP = lltype.FuncType([lltype.Signed] * len(subset), RESULT_TYPE)
         ptr = llhelper(lltype.Ptr(TP), call_me)
         c_addr = ConstAddr(llmemory.cast_ptr_to_adr(ptr), builder.cpu)
         args = [v_cond, c_addr] + subset
@@ -769,6 +776,7 @@
 for i in range(2):
     OPERATIONS.append(GuardClassOperation(rop.GUARD_CLASS))
     OPERATIONS.append(CondCallOperation(rop.COND_CALL))
+    OPERATIONS.append(CondCallOperation(rop.COND_CALL_VALUE_I))
     OPERATIONS.append(RaisingCallOperation(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationGuardNoException(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationWrongGuardException(rop.CALL_N))
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -174,8 +174,8 @@
         # copy registers to the frame, with the exception of the
         # 'cond_call_register_arguments' and eax, because these have already
         # been saved by the caller.  Note that this is not symmetrical:
-        # these 5 registers are saved by the caller but restored here at
-        # the end of this function.
+        # these 5 registers are saved by the caller but 4 of them are
+        # restored here at the end of this function.
         self._push_all_regs_to_frame(mc, cond_call_register_arguments + [eax],
                                      supports_floats, callee_only)
         # the caller already did push_gcmap(store=True)
@@ -198,7 +198,7 @@
             mc.ADD(esp, imm(WORD * 7))
         self.set_extra_stack_depth(mc, 0)
         self.pop_gcmap(mc)   # cancel the push_gcmap(store=True) in the caller
-        self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only)
+        self._pop_all_regs_from_frame(mc, [eax], supports_floats, callee_only)
         mc.RET()
         return mc.materialize(self.cpu, [])
 
@@ -1703,7 +1703,8 @@
         self.implement_guard(guard_token)
         # If the previous operation was a COND_CALL, overwrite its conditional
         # jump to jump over this GUARD_NO_EXCEPTION as well, if we can
-        if self._find_nearby_operation(-1).getopnum() == rop.COND_CALL:
+        if self._find_nearby_operation(-1).getopnum() in (
+                rop.COND_CALL, rop.COND_CALL_VALUE_I, rop.COND_CALL_VALUE_R):
             jmp_adr = self.previous_cond_call_jcond
             offset = self.mc.get_relative_pos() - jmp_adr
             if offset <= 127:
@@ -2381,7 +2382,7 @@
     def label(self):
         self._check_frame_depth_debug(self.mc)
 
-    def cond_call(self, op, gcmap, imm_func, arglocs):
+    def cond_call(self, gcmap, imm_func, arglocs, resloc=None):
         assert self.guard_success_cc >= 0
         self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0)
                                                             # patched later
@@ -2394,11 +2395,14 @@
         # plus the register 'eax'
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         should_be_saved = self._regalloc.rm.reg_bindings.values()
+        restore_eax = False
         for gpr in cond_call_register_arguments + [eax]:
-            if gpr not in should_be_saved:
+            if gpr not in should_be_saved or gpr is resloc:
                 continue
             v = gpr_reg_mgr_cls.all_reg_indexes[gpr.value]
             self.mc.MOV_br(v * WORD + base_ofs, gpr.value)
+            if gpr is eax:
+                restore_eax = True
         #
         # load the 0-to-4 arguments into these registers
         from rpython.jit.backend.x86.jump import remap_frame_layout
@@ -2422,8 +2426,16 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.CALL(imm(follow_jump(cond_call_adr)))
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        if resloc is not None and resloc is not eax:
+            self.mc.MOV(resloc, eax)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  We must only restore eax, if
+        # needed.
+        if restore_eax:
+            v = gpr_reg_mgr_cls.all_reg_indexes[eax.value]
+            self.mc.MOV_rb(eax.value, v * WORD + base_ofs)
+        #
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -938,16 +938,45 @@
                     self.rm.force_spill_var(box)
                     assert box not in self.rm.reg_bindings
         #
-        assert op.type == 'v'
         args = op.getarglist()
         assert 2 <= len(args) <= 4 + 2     # maximum 4 arguments
-        v = args[1]
-        assert isinstance(v, Const)
-        imm_func = self.rm.convert_to_imm(v)
+        v_func = args[1]
+        assert isinstance(v_func, Const)
+        imm_func = self.rm.convert_to_imm(v_func)
+
+        # Delicate ordering here.  First get the argument's locations.
+        # If this also contains args[0], this returns the current
+        # location too.
         arglocs = [self.loc(args[i]) for i in range(2, len(args))]
         gcmap = self.get_gcmap()
-        self.load_condition_into_cc(op.getarg(0))
-        self.assembler.cond_call(op, gcmap, imm_func, arglocs)
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            resloc = None
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+
+            # Test the register for the result.
+            self.assembler.test_location(resloc)
+            self.assembler.guard_success_cc = rx86.Conditions['Z']
+
+        self.assembler.cond_call(gcmap, imm_func, arglocs, resloc)
+
+    consider_cond_call_value_i = consider_cond_call
+    consider_cond_call_value_r = consider_cond_call
 
     def consider_call_malloc_nursery(self, op):
         size_box = op.getarg(0)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -15,6 +15,7 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
+    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,9 +7,10 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection)
+    EffectInfo, CallInfoCollection, CallShortcut)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
+from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -214,6 +215,7 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
+        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -228,6 +230,12 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
+            if hasattr(funcobj, 'graph'):
+                call_shortcut = self.find_call_shortcut(funcobj.graph)
+            if getattr(func, "_call_shortcut_", False):
+                assert call_shortcut is not None, (
+                    "%r: marked as @jit.call_shortcut but shortcut not found"
+                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -298,6 +306,7 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
+            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -368,3 +377,65 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
+
+    def find_call_shortcut(self, graph):
+        """Identifies graphs that start like this:
+
+           def graph(x, y, z):         def graph(x, y, z):
+               if y.field:                 r = y.field
+                   return y.field          if r: return r
+        """
+        block = graph.startblock
+        if len(block.operations) == 0:
+            return
+        op = block.operations[0]
+        if op.opname != 'getfield':
+            return
+        [v_inst, c_fieldname] = op.args
+        if not isinstance(v_inst, Variable):
+            return
+        v_result = op.result
+        if v_result.concretetype != graph.getreturnvar().concretetype:
+            return
+        if v_result.concretetype == lltype.Void:
+            return
+        argnum = i = 0
+        while block.inputargs[i] is not v_inst:
+            if block.inputargs[i].concretetype != lltype.Void:
+                argnum += 1
+            i += 1
+        PSTRUCT = v_inst.concretetype
+        v_check = v_result
+        fastcase = True
+        for op in block.operations[1:]:
+            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
+                    and v_check is op.args[0]):
+                v_check = op.result
+            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
+                v_check = op.result
+                fastcase = not fastcase
+            elif (op.opname in ('int_eq', 'int_ne')
+                    and v_check is op.args[0]
+                    and isinstance(op.args[1], Constant)
+                    and op.args[1].value == 0):
+                v_check = op.result
+                if op.opname == 'int_eq':
+                    fastcase = not fastcase
+            else:
+                return
+        if v_check.concretetype is not lltype.Bool:
+            return
+        if block.exitswitch is not v_check:
+            return
+
+        links = [link for link in block.exits if link.exitcase == fastcase]
+        if len(links) != 1:
+            return
+        [link] = links
+        if link.args != [v_result]:
+            return
+        if not link.target.is_final_block():
+            return
+
+        fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,7 +117,8 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True):
+                can_collect=True,
+                call_shortcut=None):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -135,7 +136,8 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect)
+               can_collect,
+               call_shortcut)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -190,6 +192,7 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
+        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -275,7 +278,8 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True):
+                                 can_collect=True,
+                                 call_shortcut=None):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -364,7 +368,8 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect)
+                      can_collect,
+                      call_shortcut)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -387,6 +392,24 @@
 
 # ____________________________________________________________
 
+
+class CallShortcut(object):
+    def __init__(self, argnum, fielddescr):
+        self.argnum = argnum
+        self.fielddescr = fielddescr
+
+    def __eq__(self, other):
+        return (isinstance(other, CallShortcut) and
+                self.argnum == other.argnum and
+                self.fielddescr == other.fielddescr)
+    def __ne__(self, other):
+        return not (self == other)
+    def __hash__(self):
+        return hash((self.argnum, self.fielddescr))
+
+# ____________________________________________________________
+
+
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl
-from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 
 
 class FakePolicy:
@@ -368,3 +368,100 @@
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.check_can_collect() == expected
+
+def test_find_call_shortcut():
+    class FakeCPU:
+        def fielddescrof(self, TYPE, fieldname):
+            if isinstance(TYPE, lltype.GcStruct):
+                if fieldname == 'inst_foobar':
+                    return 'foobardescr'
+                if fieldname == 'inst_fooref':
+                    return 'foorefdescr'
+            if TYPE == RAW and fieldname == 'x':
+                return 'xdescr'
+            assert False, (TYPE, fieldname)
+    cc = CallControl(FakeCPU())
+
+    class B(object):
+        foobar = 0
+        fooref = None
+
+    def f1(a, b, c):
+        if b.foobar:
+            return b.foobar
+        b.foobar = a + c
+        return b.foobar
+
+    def f2(x, y, z, b):
+        r = b.fooref
+        if r is not None:
+            return r
+        r = b.fooref = B()
+        return r
+
+    class Space(object):
+        def _freeze_(self):
+            return True
+    space = Space()
+
+    def f3(space, b):
+        r = b.foobar
+        if not r:
+            r = b.foobar = 123
+        return r
+
+    def f4(raw):
+        r = raw.x
+        if r != 0:
+            return r
+        raw.x = 123
+        return 123
+    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
+
+    def f5(b):
+        r = b.foobar
+        if r == 0:
+            r = b.foobar = 123
+        return r
+
+    def f(a, c):
+        b = B()
+        f1(a, b, c)
+        f2(a, c, a, b)
+        f3(space, b)
+        r = lltype.malloc(RAW, flavor='raw')
+        f4(r)
+        f5(b)
+
+    rtyper = support.annotate(f, [10, 20])
+    f1_graph = rtyper.annotator.translator._graphof(f1)
+    assert cc.find_call_shortcut(f1_graph) == CallShortcut(1, "foobardescr")
+    f2_graph = rtyper.annotator.translator._graphof(f2)
+    assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
+    f3_graph = rtyper.annotator.translator._graphof(f3)
+    assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
+    f4_graph = rtyper.annotator.translator._graphof(f4)
+    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
+    f5_graph = rtyper.annotator.translator._graphof(f5)
+    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
+
+def test_cant_find_call_shortcut():
+    from rpython.jit.backend.llgraph.runner import LLGraphCPU
+
+    @jit.dont_look_inside
+    @jit.call_shortcut
+    def f1(n):
+        return n + 17   # no call shortcut found
+
+    def f(n):
+        return f1(n)
+
+    rtyper = support.annotate(f, [1])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
+    res = cc.find_all_graphs(FakePolicy())
+    [f_graph] = [x for x in res if x.func is f]
+    call_op = f_graph.startblock.operations[0]
+    assert call_op.opname == 'direct_call'
+    e = py.test.raises(AssertionError, cc.getcalldescr, call_op)
+    assert "shortcut not found" in str(e.value)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -101,6 +101,18 @@
     if condbox.getint():
         do_call_n(cpu, metainterp, argboxes[1:], descr)
 
+def do_cond_call_value_i(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getint()
+    if value == 0:
+        value = do_call_i(cpu, metainterp, argboxes[1:], descr)
+    return value
+
+def do_cond_call_value_r(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getref_base()
+    if not value:
+        value = do_call_r(cpu, metainterp, argboxes[1:], descr)
+    return value
+
 def do_getarrayitem_gc_i(cpu, _, arraybox, indexbox, arraydescr):
     array = arraybox.getref_base()
     index = indexbox.getint()
@@ -366,6 +378,8 @@
                          rop.CALL_ASSEMBLER_I,
                          rop.CALL_ASSEMBLER_N,
                          rop.INCREMENT_DEBUG_COUNTER,
+                         rop.COND_CALL_VALUE_R,
+                         rop.COND_CALL_VALUE_I,
                          rop.COND_CALL_GC_WB,
                          rop.COND_CALL_GC_WB_ARRAY,
                          rop.ZERO_ARRAY,
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1149,8 +1149,8 @@
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
     'CALL/*d/rfin',
-    'COND_CALL/*d/n',
-    # a conditional call, with first argument as a condition
+    'COND_CALL/*d/n',   # a conditional call, with first argument as a condition
+    'COND_CALL_VALUE/*d/ri',  # same but returns a result; emitted by rewrite
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',
     'CALL_LOOPINVARIANT/*d/rfin',
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -195,7 +195,8 @@
                            'new_with_vtable': 2, 'getinteriorfield_gc_i': 2,
                            'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
                            'call_i': 6, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
-                           'guard_no_exception': 8, 'new': 2})
+                           'guard_no_exception': 8, 'new': 2,
+                           'guard_nonnull': 2})
 
     def test_unrolling_of_dict_iter(self):
         driver = JitDriver(greens = [], reds = ['n'])
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -257,6 +257,26 @@
     func.oopspec = "jit.not_in_trace()"   # note that 'func' may take arguments
     return func
 
+def call_shortcut(func):
+    """A decorator to ensure that a function has a fast-path.
+    Only useful on functions that the JIT doesn't normally look inside.
+    It still replaces residual calls to that function with inline code
+    that checks for a fast path, and only does the call if not.  For
+    now, graphs made by the following kinds of functions are detected:
+
+           def func(x, y, z):         def func(x, y, z):
+               if y.field:                 r = y.field
+                   return y.field          if r is None:
+               ...                             ...
+                                           return r
+
+    Fast-path detection is always on, but this decorator makes the
+    codewriter complain if it cannot find the promized fast-path.
+    """
+    func._call_shortcut_ = True
+    return func
+
+
 @oopspec("jit.isconstant(value)")
 def isconstant(value):
     """
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -369,13 +369,19 @@
         return b
 
     @staticmethod
+    def ll_strhash(s):
+        if s:
+            return LLHelpers._ll_strhash(s)
+        else:
+            return 0
+
+    @staticmethod
     @jit.elidable
-    def ll_strhash(s):
+    @jit.call_shortcut
+    def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the
         # special non-computed-yet value.
-        if not s:
-            return 0
         x = s.hash
         if x == 0:
             x = _hash_string(s.chars)

From pypy.commits at gmail.com  Mon Sep 12 10:48:26 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 12 Sep 2016 07:48:26 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: vector operations (load/store)
 where not considered in dep. construction (was not needed up to now)
Message-ID: <57d6c03a.6222c20a.1432.e58c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87045:f7f829beb5bb
Date: 2016-09-12 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/f7f829beb5bb/

Log:	vector operations (load/store) where not considered in dep.
	construction (was not needed up to now)

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -13,6 +13,7 @@
 MODIFY_COMPLEX_OBJ = [ (rop.SETARRAYITEM_GC, 0, 1)
                      , (rop.SETARRAYITEM_RAW, 0, 1)
                      , (rop.RAW_STORE, 0, 1)
+                     , (rop.VEC_STORE, 0, 1)
                      , (rop.SETINTERIORFIELD_GC, 0, -1)
                      , (rop.SETINTERIORFIELD_RAW, 0, -1)
                      , (rop.SETFIELD_GC, 0, -1)
@@ -31,6 +32,8 @@
                    , (rop.GETARRAYITEM_RAW_F, 0, 1)
                    , (rop.RAW_LOAD_I, 0, 1)
                    , (rop.RAW_LOAD_F, 0, 1)
+                   , (rop.VEC_LOAD_I, 0, 1)
+                   , (rop.VEC_LOAD_F, 0, 1)
                    , (rop.GETINTERIORFIELD_GC_I, 0, 1)
                    , (rop.GETINTERIORFIELD_GC_F, 0, 1)
                    , (rop.GETINTERIORFIELD_GC_R, 0, 1)
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -43,11 +43,14 @@
         if not delayed:
             return
         args = op.getarglist()
+        if op.is_guard():
+            args = args[:] + op.getfailargs()
         for arg in args:
             if arg.is_constant() or arg.is_inputarg():
                 continue
             if arg not in self.seen:
                 needs_resolving[arg] = None
+
         indexvars = self.graph.index_vars
         i = len(delayed)-1
         while i >= 0:
@@ -118,8 +121,6 @@
                 self.worklist.insert(0, node)
 
     def try_emit_or_delay(self, node):
-        # implement me in subclass. e.g. as in VecScheduleState
-
         if not node.is_imaginary() and node.is_pure():
             # this operation might never be emitted. only if it is really needed
             self.delay_emit(node)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -540,6 +540,15 @@
         """)
         self.assert_dependencies(graph, full_check=True)
 
+    def test_dep_on_vector_op(self):
+        graph = self.build_dependency("""
+        [p0, i1] # 0: 1,2,3
+        i19 = int_mul(i1, 8) # 1: 2
+        v20[2xi64] = vec_load_i(p0, i19, 1, 0, descr=arraydescr) # 2:
+        jump(p0, i1) # 3:
+        """)
+        self.assert_dependencies(graph, full_check=True)
+
 
     def test_iterate(self):
         n1,n2,n3,n4,n5 = [FakeNode(i+1) for i in range(5)]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -175,13 +175,11 @@
             gso = GuardStrengthenOpt(graph.index_vars)
             gso.propagate_all_forward(info, loop)
         # re-schedule
-        #vsize = self.cpu.vector_ext.vec_size()
-        #graph = DependencyGraph(loop)
-        #state = VecScheduleState(graph, PackSet(vsize), self.cpu, costmodel)
-        #state.prepare()
-        #scheduler = Scheduler()
-        #scheduler.walk_and_emit(state)
-        #state.post_schedule()
+        graph = DependencyGraph(loop)
+        state = SchedulerState(self.cpu, graph)
+        state.prepare()
+        Scheduler().walk_and_emit(state)
+        state.post_schedule()
         return opt
 
     def vectorize(self, loop, unroll_factor = -1):
@@ -200,13 +198,11 @@
         gso.propagate_all_forward(info, loop)
         #
         # re-schedule
-        #vsize = self.cpu.vector_ext.vec_size()
-        #graph = DependencyGraph(loop)
-        #state = VecScheduleState(graph, PackSet(vsize), self.cpu, costmodel)
-        #state.prepare()
-        #scheduler = Scheduler()
-        #scheduler.walk_and_emit(state)
-        #state.post_schedule()
+        graph = DependencyGraph(loop)
+        state = SchedulerState(self.cpu, graph)
+        state.prepare()
+        Scheduler().walk_and_emit(state)
+        state.post_schedule()
         #
         oplist = loop.operations
         loop.operations = loop.prefix[:]
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -261,14 +261,13 @@
         gso.propagate_all_forward(info, loop, user_code)
 
         # re-schedule the trace -> removes index operations
-        # work in progress
-        #graph = DependencyGraph(loop)
-        #costmodel = GenericCostModel(self.cpu, self.cost_threshold)
-        #state = VecScheduleState(graph, PackSet(vsize), self.cpu, costmodel)
-        #state.prepare()
-        #scheduler = Scheduler()
-        #scheduler.walk_and_emit(state)
-        #state.post_schedule()
+        graph = DependencyGraph(loop)
+        costmodel = GenericCostModel(self.cpu, self.cost_threshold)
+        state = ScheduleState(self.cpu, graph)
+        state.prepare()
+        scheduler = Scheduler()
+        scheduler.walk_and_emit(state)
+        state.post_schedule()
 
     def unroll_loop_iterations(self, loop, unroll_count):
         """ Unroll the loop X times. unroll_count + 1 = unroll_factor """

From pypy.commits at gmail.com  Mon Sep 12 10:51:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 07:51:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Add explicitly @jit.call_shortcut here
Message-ID: <57d6c102.4398c20a.783a0.8fa7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87047:bd9d1dd19c2c
Date: 2016-09-12 16:51 +0200
http://bitbucket.org/pypy/pypy/changeset/bd9d1dd19c2c/

Log:	Add explicitly @jit.call_shortcut here

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -78,6 +78,7 @@
         return self._value
 
     @jit.elidable
+    @jit.call_shortcut
     def identifier_w(self, space):
         identifier = self._utf8
         if identifier is not None:

From pypy.commits at gmail.com  Mon Sep 12 10:51:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 07:51:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57d6c100.255ac20a.7ccce.ffac@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87046:81579d630803
Date: 2016-09-12 16:49 +0200
http://bitbucket.org/pypy/pypy/changeset/81579d630803/

Log:	hg merge default

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
         assert module.test_fast_sequence(s[0:-1])
         assert module.test_fast_sequence(s[::-1])
 
+    def test_fast_keyerror(self):
+        module = self.import_extension('foo', [
+            ("test_fast_sequence", "METH_VARARGS",
+             """
+                PyObject *foo;
+                PyObject * seq = PyTuple_GetItem(args, 0);
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "Could not convert object to sequence");
+                if (foo != NULL)
+                {
+                    return foo;
+                }
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                    return PyBool_FromLong(1);
+                }
+                return NULL;
+             """)])
+        class Map(object):
+            def __len__(self):
+                return 1
+
+            def __getitem__(self, index):
+                raise KeyError()
+
+        assert module.test_fast_sequence(Map()) is True
+
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,7 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(ll_strhash), p10, descr=<Calli . r EF=0>)
+            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1002,6 +1002,9 @@
     prepare_op_cond_call_gc_wb_array = prepare_op_cond_call_gc_wb
 
     def prepare_op_cond_call(self, op, fcond):
+        # XXX don't force the arguments to be loaded in specific
+        # locations before knowing if we can take the fast path
+        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -325,6 +325,7 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
+    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_extension = True
@@ -1334,6 +1335,16 @@
         # cond_call can't have a return value
         self.execute_call_n(calldescr, func, *args)
 
+    def execute_cond_call_value_i(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_i(calldescr, func, *args)
+        return value
+
+    def execute_cond_call_value_r(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_r(calldescr, func, *args)
+        return value
+
     def _execute_call(self, calldescr, func, *args):
         effectinfo = calldescr.get_extra_info()
         if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'):
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -759,6 +759,8 @@
         if (opnum != rop.GUARD_TRUE and opnum != rop.GUARD_FALSE
                                     and opnum != rop.COND_CALL):
             return False
+        # NB: don't list COND_CALL_VALUE_I/R here, these two variants
+        # of COND_CALL don't accept a cc as input
         if next_op.getarg(0) is not op:
             return False
         if self.longevity[op][1] > i + 1:
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -11,7 +11,7 @@
 from rpython.jit.backend.llsupport.symbolic import (WORD,
         get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER
+     FLAG_POINTER, CallDescr
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -370,7 +370,9 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- call assembler -----------
+            # ---------- calls -----------
+            if OpHelpers.is_plain_call(op.getopnum()):
+                self.expand_call_shortcut(op)
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -616,6 +618,30 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
+    def expand_call_shortcut(self, op):
+        if not self.cpu.supports_cond_call_value:
+            return
+        descr = op.getdescr()
+        if descr is None:
+            return
+        assert isinstance(descr, CallDescr)
+        effectinfo = descr.get_extra_info()
+        if effectinfo is None or effectinfo.call_shortcut is None:
+            return
+        if op.type == 'r':
+            cond_call_opnum = rop.COND_CALL_VALUE_R
+        elif op.type == 'i':
+            cond_call_opnum = rop.COND_CALL_VALUE_I
+        else:
+            return
+        cs = effectinfo.call_shortcut
+        ptr_box = op.getarg(1 + cs.argnum)
+        value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
+                                       raw=(ptr_box.type == 'i'))
+        self.replace_op_with(op, ResOperation(cond_call_opnum,
+                                              [value_box] + op.getarglist(),
+                                              descr=descr))
+
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,7 +1,8 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr
+     SizeDescr, get_interiorfield_descr, get_call_descr
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -80,6 +81,14 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
+        call_shortcut = CallShortcut(0, tzdescr)
+        effectinfo = EffectInfo(None, None, None, None, None, None,
+                                EffectInfo.EF_RANDOM_EFFECTS,
+                                call_shortcut=call_shortcut)
+        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
+            [lltype.Ptr(T)], lltype.Signed,
+            effectinfo)
+        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -200,6 +209,7 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
+    supports_cond_call_value = True
 
     translate_support_code = None
 
@@ -1429,3 +1439,15 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
+
+    def test_handle_call_shortcut(self):
+        self.check_rewrite("""
+            [p0]
+            i1 = call_i(123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """, """
+            [p0]
+            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
+            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,6 +16,7 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
+    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2389,7 +2389,7 @@
             f2 = longlong.getfloatstorage(3.4)
             frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5, f1, f2)
             assert not called
-            for j in range(5):
+            for j in range(6):
                 assert self.cpu.get_int_value(frame, j) == j
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 6)) == 1.2
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 3.4
@@ -2447,6 +2447,54 @@
                                            67, 89)
             assert called == [(67, 89)]
 
+    def test_cond_call_value(self):
+        if not self.cpu.supports_cond_call_value:
+            py.test.skip("missing supports_cond_call_value")
+
+        def func_int(*args):
+            called.append(args)
+            return len(args) * 100 + 1000
+
+        for i in range(5):
+            called = []
+
+            FUNC = self.FuncType([lltype.Signed] * i, lltype.Signed)
+            func_ptr = llhelper(lltype.Ptr(FUNC), func_int)
+            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                             EffectInfo.MOST_GENERAL)
+
+            ops = '''
+            [i0, i1, i2, i3, i4, i5, i6, f0, f1]
+            i15 = cond_call_value_i(i1, ConstClass(func_ptr), %s)
+            guard_false(i0, descr=faildescr) [i1,i2,i3,i4,i5,i6,i15, f0,f1]
+            finish(i15)
+            ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] +
+                            ["descr=calldescr"])
+            loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
+                                         'func_ptr': func_ptr,
+                                         'calldescr': calldescr})
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+            f1 = longlong.getfloatstorage(1.2)
+            f2 = longlong.getfloatstorage(3.4)
+            frame = self.cpu.execute_token(looptoken, 1, 50, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert not called
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        50, 1, 2, 3, 4, 5, 50]
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 8)) == 3.4
+            #
+            frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert called == [(1, 2, 3, 4)[:i]]
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        0, 1, 2, 3, 4, 5, i * 100 + 1000]
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 8)) == 3.4
+
     def test_force_operations_returning_void(self):
         values = []
         def maybe_force(token, flag):
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -594,7 +594,7 @@
         return subset, d['f'], vtableptr
 
     def getresulttype(self):
-        if self.opnum == rop.CALL_I:
+        if self.opnum == rop.CALL_I or self.opnum == rop.COND_CALL_VALUE_I:
             return lltype.Signed
         elif self.opnum == rop.CALL_F:
             return lltype.Float
@@ -712,7 +712,12 @@
 class CondCallOperation(BaseCallOperation):
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
-        v_cond = builder.get_bool_var(r)
+        if self.opnum == rop.COND_CALL:
+            RESULT_TYPE = lltype.Void
+            v_cond = builder.get_bool_var(r)
+        else:
+            RESULT_TYPE = lltype.Signed
+            v_cond = r.choice(builder.intvars)
         subset = builder.subset_of_intvars(r)[:4]
         for i in range(len(subset)):
             if r.random() < 0.35:
@@ -724,8 +729,10 @@
                 seen.append(args)
             else:
                 assert seen[0] == args
+            if RESULT_TYPE is lltype.Signed:
+                return len(args) - 42000
         #
-        TP = lltype.FuncType([lltype.Signed] * len(subset), lltype.Void)
+        TP = lltype.FuncType([lltype.Signed] * len(subset), RESULT_TYPE)
         ptr = llhelper(lltype.Ptr(TP), call_me)
         c_addr = ConstAddr(llmemory.cast_ptr_to_adr(ptr), builder.cpu)
         args = [v_cond, c_addr] + subset
@@ -769,6 +776,7 @@
 for i in range(2):
     OPERATIONS.append(GuardClassOperation(rop.GUARD_CLASS))
     OPERATIONS.append(CondCallOperation(rop.COND_CALL))
+    OPERATIONS.append(CondCallOperation(rop.COND_CALL_VALUE_I))
     OPERATIONS.append(RaisingCallOperation(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationGuardNoException(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationWrongGuardException(rop.CALL_N))
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -174,8 +174,8 @@
         # copy registers to the frame, with the exception of the
         # 'cond_call_register_arguments' and eax, because these have already
         # been saved by the caller.  Note that this is not symmetrical:
-        # these 5 registers are saved by the caller but restored here at
-        # the end of this function.
+        # these 5 registers are saved by the caller but 4 of them are
+        # restored here at the end of this function.
         self._push_all_regs_to_frame(mc, cond_call_register_arguments + [eax],
                                      supports_floats, callee_only)
         # the caller already did push_gcmap(store=True)
@@ -198,7 +198,7 @@
             mc.ADD(esp, imm(WORD * 7))
         self.set_extra_stack_depth(mc, 0)
         self.pop_gcmap(mc)   # cancel the push_gcmap(store=True) in the caller
-        self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only)
+        self._pop_all_regs_from_frame(mc, [eax], supports_floats, callee_only)
         mc.RET()
         return mc.materialize(self.cpu, [])
 
@@ -1703,7 +1703,8 @@
         self.implement_guard(guard_token)
         # If the previous operation was a COND_CALL, overwrite its conditional
         # jump to jump over this GUARD_NO_EXCEPTION as well, if we can
-        if self._find_nearby_operation(-1).getopnum() == rop.COND_CALL:
+        if self._find_nearby_operation(-1).getopnum() in (
+                rop.COND_CALL, rop.COND_CALL_VALUE_I, rop.COND_CALL_VALUE_R):
             jmp_adr = self.previous_cond_call_jcond
             offset = self.mc.get_relative_pos() - jmp_adr
             if offset <= 127:
@@ -2381,7 +2382,7 @@
     def label(self):
         self._check_frame_depth_debug(self.mc)
 
-    def cond_call(self, op, gcmap, imm_func, arglocs):
+    def cond_call(self, gcmap, imm_func, arglocs, resloc=None):
         assert self.guard_success_cc >= 0
         self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0)
                                                             # patched later
@@ -2394,11 +2395,14 @@
         # plus the register 'eax'
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         should_be_saved = self._regalloc.rm.reg_bindings.values()
+        restore_eax = False
         for gpr in cond_call_register_arguments + [eax]:
-            if gpr not in should_be_saved:
+            if gpr not in should_be_saved or gpr is resloc:
                 continue
             v = gpr_reg_mgr_cls.all_reg_indexes[gpr.value]
             self.mc.MOV_br(v * WORD + base_ofs, gpr.value)
+            if gpr is eax:
+                restore_eax = True
         #
         # load the 0-to-4 arguments into these registers
         from rpython.jit.backend.x86.jump import remap_frame_layout
@@ -2422,8 +2426,16 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.CALL(imm(follow_jump(cond_call_adr)))
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        if resloc is not None and resloc is not eax:
+            self.mc.MOV(resloc, eax)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  We must only restore eax, if
+        # needed.
+        if restore_eax:
+            v = gpr_reg_mgr_cls.all_reg_indexes[eax.value]
+            self.mc.MOV_rb(eax.value, v * WORD + base_ofs)
+        #
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -938,16 +938,45 @@
                     self.rm.force_spill_var(box)
                     assert box not in self.rm.reg_bindings
         #
-        assert op.type == 'v'
         args = op.getarglist()
         assert 2 <= len(args) <= 4 + 2     # maximum 4 arguments
-        v = args[1]
-        assert isinstance(v, Const)
-        imm_func = self.rm.convert_to_imm(v)
+        v_func = args[1]
+        assert isinstance(v_func, Const)
+        imm_func = self.rm.convert_to_imm(v_func)
+
+        # Delicate ordering here.  First get the argument's locations.
+        # If this also contains args[0], this returns the current
+        # location too.
         arglocs = [self.loc(args[i]) for i in range(2, len(args))]
         gcmap = self.get_gcmap()
-        self.load_condition_into_cc(op.getarg(0))
-        self.assembler.cond_call(op, gcmap, imm_func, arglocs)
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            resloc = None
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+
+            # Test the register for the result.
+            self.assembler.test_location(resloc)
+            self.assembler.guard_success_cc = rx86.Conditions['Z']
+
+        self.assembler.cond_call(gcmap, imm_func, arglocs, resloc)
+
+    consider_cond_call_value_i = consider_cond_call
+    consider_cond_call_value_r = consider_cond_call
 
     def consider_call_malloc_nursery(self, op):
         size_box = op.getarg(0)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -15,6 +15,7 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
+    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,9 +7,10 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection)
+    EffectInfo, CallInfoCollection, CallShortcut)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
+from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -214,6 +215,7 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
+        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -228,6 +230,12 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
+            if hasattr(funcobj, 'graph'):
+                call_shortcut = self.find_call_shortcut(funcobj.graph)
+            if getattr(func, "_call_shortcut_", False):
+                assert call_shortcut is not None, (
+                    "%r: marked as @jit.call_shortcut but shortcut not found"
+                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -298,6 +306,7 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
+            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -368,3 +377,65 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
+
+    def find_call_shortcut(self, graph):
+        """Identifies graphs that start like this:
+
+           def graph(x, y, z):         def graph(x, y, z):
+               if y.field:                 r = y.field
+                   return y.field          if r: return r
+        """
+        block = graph.startblock
+        if len(block.operations) == 0:
+            return
+        op = block.operations[0]
+        if op.opname != 'getfield':
+            return
+        [v_inst, c_fieldname] = op.args
+        if not isinstance(v_inst, Variable):
+            return
+        v_result = op.result
+        if v_result.concretetype != graph.getreturnvar().concretetype:
+            return
+        if v_result.concretetype == lltype.Void:
+            return
+        argnum = i = 0
+        while block.inputargs[i] is not v_inst:
+            if block.inputargs[i].concretetype != lltype.Void:
+                argnum += 1
+            i += 1
+        PSTRUCT = v_inst.concretetype
+        v_check = v_result
+        fastcase = True
+        for op in block.operations[1:]:
+            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
+                    and v_check is op.args[0]):
+                v_check = op.result
+            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
+                v_check = op.result
+                fastcase = not fastcase
+            elif (op.opname in ('int_eq', 'int_ne')
+                    and v_check is op.args[0]
+                    and isinstance(op.args[1], Constant)
+                    and op.args[1].value == 0):
+                v_check = op.result
+                if op.opname == 'int_eq':
+                    fastcase = not fastcase
+            else:
+                return
+        if v_check.concretetype is not lltype.Bool:
+            return
+        if block.exitswitch is not v_check:
+            return
+
+        links = [link for link in block.exits if link.exitcase == fastcase]
+        if len(links) != 1:
+            return
+        [link] = links
+        if link.args != [v_result]:
+            return
+        if not link.target.is_final_block():
+            return
+
+        fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,7 +117,8 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True):
+                can_collect=True,
+                call_shortcut=None):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -135,7 +136,8 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect)
+               can_collect,
+               call_shortcut)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -190,6 +192,7 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
+        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -275,7 +278,8 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True):
+                                 can_collect=True,
+                                 call_shortcut=None):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -364,7 +368,8 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect)
+                      can_collect,
+                      call_shortcut)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -387,6 +392,24 @@
 
 # ____________________________________________________________
 
+
+class CallShortcut(object):
+    def __init__(self, argnum, fielddescr):
+        self.argnum = argnum
+        self.fielddescr = fielddescr
+
+    def __eq__(self, other):
+        return (isinstance(other, CallShortcut) and
+                self.argnum == other.argnum and
+                self.fielddescr == other.fielddescr)
+    def __ne__(self, other):
+        return not (self == other)
+    def __hash__(self):
+        return hash((self.argnum, self.fielddescr))
+
+# ____________________________________________________________
+
+
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -364,7 +364,7 @@
         return getattr(self, 'handle_%s_indirect_call' % kind)(op)
 
     def rewrite_call(self, op, namebase, initialargs, args=None,
-                     calldescr=None):
+                     calldescr=None, force_ir=False):
         """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)'
            into 'i0 = xxx_call_ir_i(fn, descr, [i1,i2], [ref1,ref2])'.
            The name is one of '{residual,direct}_call_{r,ir,irf}_{i,r,f,v}'."""
@@ -374,8 +374,9 @@
         lst_i, lst_r, lst_f = self.make_three_lists(args)
         reskind = getkind(op.result.concretetype)[0]
         if lst_f or reskind == 'f': kinds = 'irf'
-        elif lst_i: kinds = 'ir'
+        elif lst_i or force_ir: kinds = 'ir'
         else: kinds = 'r'
+        if force_ir: assert kinds == 'ir'    # no 'f'
         sublists = []
         if 'i' in kinds: sublists.append(lst_i)
         if 'r' in kinds: sublists.append(lst_r)
@@ -1577,7 +1578,7 @@
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
         op1 = self.rewrite_call(op, 'conditional_call',
                                 op.args[:2], args=op.args[2:],
-                                calldescr=calldescr)
+                                calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl
-from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 
 
 class FakePolicy:
@@ -368,3 +368,100 @@
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.check_can_collect() == expected
+
+def test_find_call_shortcut():
+    class FakeCPU:
+        def fielddescrof(self, TYPE, fieldname):
+            if isinstance(TYPE, lltype.GcStruct):
+                if fieldname == 'inst_foobar':
+                    return 'foobardescr'
+                if fieldname == 'inst_fooref':
+                    return 'foorefdescr'
+            if TYPE == RAW and fieldname == 'x':
+                return 'xdescr'
+            assert False, (TYPE, fieldname)
+    cc = CallControl(FakeCPU())
+
+    class B(object):
+        foobar = 0
+        fooref = None
+
+    def f1(a, b, c):
+        if b.foobar:
+            return b.foobar
+        b.foobar = a + c
+        return b.foobar
+
+    def f2(x, y, z, b):
+        r = b.fooref
+        if r is not None:
+            return r
+        r = b.fooref = B()
+        return r
+
+    class Space(object):
+        def _freeze_(self):
+            return True
+    space = Space()
+
+    def f3(space, b):
+        r = b.foobar
+        if not r:
+            r = b.foobar = 123
+        return r
+
+    def f4(raw):
+        r = raw.x
+        if r != 0:
+            return r
+        raw.x = 123
+        return 123
+    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
+
+    def f5(b):
+        r = b.foobar
+        if r == 0:
+            r = b.foobar = 123
+        return r
+
+    def f(a, c):
+        b = B()
+        f1(a, b, c)
+        f2(a, c, a, b)
+        f3(space, b)
+        r = lltype.malloc(RAW, flavor='raw')
+        f4(r)
+        f5(b)
+
+    rtyper = support.annotate(f, [10, 20])
+    f1_graph = rtyper.annotator.translator._graphof(f1)
+    assert cc.find_call_shortcut(f1_graph) == CallShortcut(1, "foobardescr")
+    f2_graph = rtyper.annotator.translator._graphof(f2)
+    assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
+    f3_graph = rtyper.annotator.translator._graphof(f3)
+    assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
+    f4_graph = rtyper.annotator.translator._graphof(f4)
+    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
+    f5_graph = rtyper.annotator.translator._graphof(f5)
+    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
+
+def test_cant_find_call_shortcut():
+    from rpython.jit.backend.llgraph.runner import LLGraphCPU
+
+    @jit.dont_look_inside
+    @jit.call_shortcut
+    def f1(n):
+        return n + 17   # no call shortcut found
+
+    def f(n):
+        return f1(n)
+
+    rtyper = support.annotate(f, [1])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
+    res = cc.find_all_graphs(FakePolicy())
+    [f_graph] = [x for x in res if x.func is f]
+    call_op = f_graph.startblock.operations[0]
+    assert call_op.opname == 'direct_call'
+    e = py.test.raises(AssertionError, cc.getcalldescr, call_op)
+    assert "shortcut not found" in str(e.value)
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1200,28 +1200,12 @@
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
     # conditional calls - note that they cannot return stuff
-    @arguments("cpu", "i", "i", "I", "d")
-    def bhimpl_conditional_call_i_v(cpu, condition, func, args_i, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, None, None, calldescr)
-
-    @arguments("cpu", "i", "i", "R", "d")
-    def bhimpl_conditional_call_r_v(cpu, condition, func, args_r, calldescr):
-        if condition:
-            cpu.bh_call_v(func, None, args_r, None, calldescr)
-
     @arguments("cpu", "i", "i", "I", "R", "d")
     def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
                                      calldescr):
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
-    @arguments("cpu", "i", "i", "I", "R", "F", "d")
-    def bhimpl_conditional_call_irf_v(cpu, condition, func, args_i, args_r,
-                                      args_f, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
-
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -101,6 +101,18 @@
     if condbox.getint():
         do_call_n(cpu, metainterp, argboxes[1:], descr)
 
+def do_cond_call_value_i(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getint()
+    if value == 0:
+        value = do_call_i(cpu, metainterp, argboxes[1:], descr)
+    return value
+
+def do_cond_call_value_r(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getref_base()
+    if not value:
+        value = do_call_r(cpu, metainterp, argboxes[1:], descr)
+    return value
+
 def do_getarrayitem_gc_i(cpu, _, arraybox, indexbox, arraydescr):
     array = arraybox.getref_base()
     index = indexbox.getint()
@@ -366,6 +378,8 @@
                          rop.CALL_ASSEMBLER_I,
                          rop.CALL_ASSEMBLER_N,
                          rop.INCREMENT_DEBUG_COUNTER,
+                         rop.COND_CALL_VALUE_R,
+                         rop.COND_CALL_VALUE_I,
                          rop.COND_CALL_GC_WB,
                          rop.COND_CALL_GC_WB_ARRAY,
                          rop.ZERO_ARRAY,
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1056,23 +1056,11 @@
     opimpl_residual_call_irf_f = _opimpl_residual_call3
     opimpl_residual_call_irf_v = _opimpl_residual_call3
 
-    @arguments("box", "box", "boxes", "descr", "orgpc")
-    def opimpl_conditional_call_i_v(self, condbox, funcbox, argboxes, calldescr,
-                                    pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
-
-    opimpl_conditional_call_r_v = opimpl_conditional_call_i_v
-
     @arguments("box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, funcbox, argboxes,
                                      calldescr, pc):
         self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
 
-    @arguments("box", "box", "boxes3", "descr", "orgpc")
-    def opimpl_conditional_call_irf_v(self, condbox, funcbox, argboxes,
-                                      calldescr, pc):
-        self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
-
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
         targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1149,8 +1149,8 @@
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
     'CALL/*d/rfin',
-    'COND_CALL/*d/n',
-    # a conditional call, with first argument as a condition
+    'COND_CALL/*d/n',   # a conditional call, with first argument as a condition
+    'COND_CALL_VALUE/*d/ri',  # same but returns a result; emitted by rewrite
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',
     'CALL_LOOPINVARIANT/*d/rfin',
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -195,7 +195,8 @@
                            'new_with_vtable': 2, 'getinteriorfield_gc_i': 2,
                            'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
                            'call_i': 6, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
-                           'guard_no_exception': 8, 'new': 2})
+                           'guard_no_exception': 8, 'new': 2,
+                           'guard_nonnull': 2})
 
     def test_unrolling_of_dict_iter(self):
         driver = JitDriver(greens = [], reds = ['n'])
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -257,6 +257,26 @@
     func.oopspec = "jit.not_in_trace()"   # note that 'func' may take arguments
     return func
 
+def call_shortcut(func):
+    """A decorator to ensure that a function has a fast-path.
+    Only useful on functions that the JIT doesn't normally look inside.
+    It still replaces residual calls to that function with inline code
+    that checks for a fast path, and only does the call if not.  For
+    now, graphs made by the following kinds of functions are detected:
+
+           def func(x, y, z):         def func(x, y, z):
+               if y.field:                 r = y.field
+                   return y.field          if r is None:
+               ...                             ...
+                                           return r
+
+    Fast-path detection is always on, but this decorator makes the
+    codewriter complain if it cannot find the promized fast-path.
+    """
+    func._call_shortcut_ = True
+    return func
+
+
 @oopspec("jit.isconstant(value)")
 def isconstant(value):
     """
diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -369,13 +369,19 @@
         return b
 
     @staticmethod
+    def ll_strhash(s):
+        if s:
+            return LLHelpers._ll_strhash(s)
+        else:
+            return 0
+
+    @staticmethod
     @jit.elidable
-    def ll_strhash(s):
+    @jit.call_shortcut
+    def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the
         # special non-computed-yet value.
-        if not s:
-            return 0
         x = s.hash
         if x == 0:
             x = _hash_string(s.chars)

From pypy.commits at gmail.com  Mon Sep 12 11:03:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 08:03:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix the initial value returned by
 sys.getrecursionlimit(). Used to be
Message-ID: <57d6c3be.0575c20a.8f0a7.04d0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87048:0165d335e184
Date: 2016-09-12 17:02 +0200
http://bitbucket.org/pypy/pypy/changeset/0165d335e184/

Log:	Fix the initial value returned by sys.getrecursionlimit(). Used to
	be 100, even though it was internally configured like it is after
	sys.setrecursionlimit(1000).

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -15,7 +15,7 @@
         if space.config.translating:
             del self.__class__.interpleveldefs['pypy_getudir']
         super(Module, self).__init__(space, w_name)
-        self.recursionlimit = 100
+        self.recursionlimit = 1000
         self.w_default_encoder = None
         self.defaultencoding = "ascii"
         self.filesystemencoding = None

From pypy.commits at gmail.com  Mon Sep 12 11:29:13 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 12 Sep 2016 08:29:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix -A segfault in
 test_tp_new_in_subclass_type(), but now it fails in regular testing
Message-ID: <57d6c9c9.c19d1c0a.68f16.4cc2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r87049:6ea4b78fa64b
Date: 2016-09-12 16:28 +0100
http://bitbucket.org/pypy/pypy/changeset/6ea4b78fa64b/

Log:	Fix -A segfault in test_tp_new_in_subclass_type(), but now it fails
	in regular testing

diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,9 +4,7 @@
 PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
 {
     PyObject* newType;
-    /*printf("in foo3type_tp_new, preprocessing...\n"); */
-    newType = PyType_Type.tp_new(metatype, args, kwds);
-    /*printf("in foo3type_tp_new, postprocessing...\n"); */
+    newType = PyType_GenericNew(metatype, args, kwds);
     return newType;
 }
 
@@ -73,6 +71,7 @@
     PyObject *mod, *d;
     footype.tp_base = &PyType_Type;
     PyType_Ready(&footype);
+    Py_INCREF(&footype);
     mod = Py_InitModule("foo3", sbkMethods);
     if (mod == NULL)
         return;
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -961,7 +961,6 @@
 
     def test_tp_new_in_subclass_of_type(self):
         module = self.import_module(name='foo3')
-        #print('calling module.footype()...')
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):

From pypy.commits at gmail.com  Mon Sep 12 12:15:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 09:15:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for argument clinic's use of the
 "bool" specifier, which now maps
Message-ID: <57d6d498.d32d1c0a.60292.5baf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87050:a8137a62856a
Date: 2016-09-12 18:14 +0200
http://bitbucket.org/pypy/pypy/changeset/a8137a62856a/

Log:	Fix for argument clinic's use of the "bool" specifier, which now
	maps to "unwrap_spec(...=bool)".

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1232,7 +1232,7 @@
             if e.match(self, self.w_AttributeError):
                 return False
             raise
-        return self.bool_w(self.nonzero(w_result))
+        return self.is_true(self.nonzero(w_result))
 
     # CPython rules allows subclasses of BaseExceptions to be exceptions.
     # This is slightly less general than the case above, so we prefix
@@ -1683,7 +1683,9 @@
 
     def bool_w(self, w_obj):
         # Unwraps a bool, also accepting an int for compatibility.
-        # This is here mostly just for gateway.int_unwrapping_space_method().
+        # For cases where you need to accept bools and ints and nothing
+        # else.  Note that saying 'bool' in unwrap_spec() doesn't call
+        # this, but the general is_true(),  accepting any object.
         return bool(self.int_w(w_obj))
 
     def ord(self, w_obj):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -520,8 +520,17 @@
     assert typ in (int, str, float, unicode, r_longlong, r_uint, r_ulonglong, bool)
     if typ is r_int is r_longlong:
         return 'gateway_r_longlong_w'
-    elif typ in (str, unicode, bool):
+    elif typ in (str, unicode):
         return typ.__name__ + '_w'
+    elif typ is bool:
+        # For argument clinic's "bool" specifier: accept any object, and
+        # convert it to a boolean value.  If you don't want this
+        # behavior, you need to say "int" in the unwrap_spec().  Please
+        # use only to emulate "bool" in argument clinic or the 'p'
+        # letter in PyArg_ParseTuple().  Accepting *anything* when a
+        # boolean flag is expected feels like it comes straight from
+        # JavaScript: it is a sure way to hide bugs imho <arigo>.
+        return 'is_true'
     else:
         return 'gateway_' + typ.__name__ + '_w'
 
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -21,7 +21,7 @@
 
 @unwrap_spec(mode=str, buffering=int,
              encoding="str_or_None", errors="str_or_None",
-             newline="str_or_None", closefd=bool)
+             newline="str_or_None", closefd=int)
 def open(space, w_file, mode="r", buffering=-1, encoding=None, errors=None,
          newline=None, closefd=True, w_opener=None):
     from pypy.module._io.interp_bufferedio import (W_BufferedRandom,
@@ -88,7 +88,7 @@
                     "binary mode doesn't take a newline argument")
     w_raw = space.call_function(
         space.gettypefor(W_FileIO), w_file, space.wrap(rawmode),
-        space.wrap(closefd), w_opener)
+        space.wrap(bool(closefd)), w_opener)
 
     isatty = space.is_true(space.call_method(w_raw, "isatty"))
     line_buffering = buffering == 1 or (buffering < 0 and isatty)
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -434,11 +434,11 @@
         return stats(space, self.data.values() + self.builtin_data.values(),
                      factor)
 
- at unwrap_spec(time_unit=float, subcalls=bool, builtins=bool)
+ at unwrap_spec(time_unit=float, subcalls=int, builtins=int)
 def descr_new_profile(space, w_type, w_callable=None, time_unit=0.0,
-                      subcalls=True, builtins=True):
+                      subcalls=1, builtins=1):
     p = space.allocate_instance(W_Profiler, w_type)
-    p.__init__(space, w_callable, time_unit, subcalls, builtins)
+    p.__init__(space, w_callable, time_unit, bool(subcalls), bool(builtins))
     return space.wrap(p)
 
 W_Profiler.typedef = TypeDef(
diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py
--- a/pypy/module/_multibytecodec/interp_incremental.py
+++ b/pypy/module/_multibytecodec/interp_incremental.py
@@ -48,8 +48,9 @@
             c_codecs.pypy_cjk_dec_free(self.decodebuf)
             self.decodebuf = lltype.nullptr(c_codecs.DECODEBUF_P.TO)
 
-    @unwrap_spec(object='bufferstr', final=bool)
-    def decode_w(self, object, final=False):
+    @unwrap_spec(object='bufferstr', final=int)
+    def decode_w(self, object, final=0):
+        final = bool(final)
         space = self.space
         state = space.fromcache(CodecState)
         if len(self.pending) > 0:
@@ -96,8 +97,9 @@
             c_codecs.pypy_cjk_enc_free(self.encodebuf)
             self.encodebuf = lltype.nullptr(c_codecs.ENCODEBUF_P.TO)
 
-    @unwrap_spec(object=unicode, final=bool)
-    def encode_w(self, object, final=False):
+    @unwrap_spec(object=unicode, final=int)
+    def encode_w(self, object, final=0):
+        final = bool(final)
         space = self.space
         state = space.fromcache(CodecState)
         if len(self.pending) > 0:
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -36,9 +36,9 @@
     def get_basic_ffi_type(self):
         return self.basicffitype
 
-    @unwrap_spec(length=int, autofree=bool)
+    @unwrap_spec(length=int, autofree=int)
     def descr_call(self, space, length, w_items=None, autofree=False):
-        result = self.allocate(space, length, autofree)
+        result = self.allocate(space, length, bool(autofree))
         if not space.is_none(w_items):
             items_w = space.unpackiterable(w_items)
             iterlength = len(items_w)
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -189,9 +189,9 @@
             raise oefmt(space.w_AttributeError,
                         "C Structure has no attribute %s", attr)
 
-    @unwrap_spec(autofree=bool)
+    @unwrap_spec(autofree=int)
     def descr_call(self, space, autofree=False):
-        return space.wrap(self.allocate(space, 1, autofree))
+        return space.wrap(self.allocate(space, 1, bool(autofree)))
 
     def descr_repr(self, space):
         fieldnames = ' '.join(["'%s'" % name for name, _, _ in self.fields])
@@ -248,8 +248,9 @@
             lltype.free(self.ffi_struct, flavor='raw')
 
 
- at unwrap_spec(union=bool, pack=int)
-def descr_new_structure(space, w_type, w_shapeinfo, union=False, pack=0):
+ at unwrap_spec(union=int, pack=int)
+def descr_new_structure(space, w_type, w_shapeinfo, union=0, pack=0):
+    union = bool(union)
     if pack < 0:
         raise oefmt(space.w_ValueError,
                     "_pack_ must be a non-negative integer")
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -453,7 +453,7 @@
             raise converted_error(space, e)
         return space.wrap(count)
 
-    @unwrap_spec(flag=bool)
+    @unwrap_spec(flag=int)
     def setblocking_w(self, flag):
         """setblocking(flag)
 
@@ -461,7 +461,7 @@
         setblocking(True) is equivalent to settimeout(None);
         setblocking(False) is equivalent to settimeout(0.0).
         """
-        self.sock.setblocking(flag)
+        self.sock.setblocking(bool(flag))
 
     @unwrap_spec(level=int, optname=int)
     def setsockopt_w(self, space, level, optname, w_optval):
diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -267,13 +267,13 @@
             return known_type == rposix_scandir.DT_LNK
         return self.check_mode(follow_symlinks=False) == stat.S_IFLNK
 
-    @unwrap_spec(follow_symlinks=int)
-    def descr_is_dir(self, space, __kwonly__, follow_symlinks=1):
+    @unwrap_spec(follow_symlinks=bool)
+    def descr_is_dir(self, space, __kwonly__, follow_symlinks=True):
         """return True if the entry is a directory; cached per entry"""
         return space.wrap(self.is_dir(follow_symlinks))
 
-    @unwrap_spec(follow_symlinks=int)
-    def descr_is_file(self, space, __kwonly__, follow_symlinks=1):
+    @unwrap_spec(follow_symlinks=bool)
+    def descr_is_file(self, space, __kwonly__, follow_symlinks=True):
         """return True if the entry is a file; cached per entry"""
         return space.wrap(self.is_file(follow_symlinks))
 
@@ -281,8 +281,8 @@
         """return True if the entry is a symbolic link; cached per entry"""
         return space.wrap(self.is_symlink())
 
-    @unwrap_spec(follow_symlinks=int)
-    def descr_stat(self, space, __kwonly__, follow_symlinks=1):
+    @unwrap_spec(follow_symlinks=bool)
+    def descr_stat(self, space, __kwonly__, follow_symlinks=True):
         """return stat_result object for the entry; cached per entry"""
         st = self.get_stat_or_lstat(follow_symlinks)
         return build_stat_result(self.space, st)
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -634,7 +634,7 @@
 
     # Parse methods
 
-    @unwrap_spec(isfinal=bool)
+    @unwrap_spec(isfinal=int)
     def Parse(self, space, w_data, isfinal=False):
         """Parse(data[, isfinal])
 Parse XML data.  `isfinal' should be true at end of input."""
@@ -645,6 +645,7 @@
             XML_SetEncoding(self.itself, "utf-8")
         else:
             data = space.bufferstr_w(w_data)
+        isfinal = bool(isfinal)
         res = XML_Parse(self.itself, data, len(data), isfinal)
         if self._exc_info:
             e = self._exc_info
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -210,27 +210,27 @@
 )
 W_NotFromAssembler.typedef.acceptable_as_base_class = False
 
- at unwrap_spec(next_instr=int, is_being_profiled=bool, w_pycode=PyCode)
+ at unwrap_spec(next_instr=int, is_being_profiled=int, w_pycode=PyCode)
 @dont_look_inside
 def get_jitcell_at_key(space, next_instr, is_being_profiled, w_pycode):
     ll_pycode = cast_instance_to_gcref(w_pycode)
     return space.wrap(bool(jit_hooks.get_jitcell_at_key(
-        'pypyjit', r_uint(next_instr), int(is_being_profiled), ll_pycode)))
+       'pypyjit', r_uint(next_instr), int(bool(is_being_profiled)), ll_pycode)))
 
- at unwrap_spec(next_instr=int, is_being_profiled=bool, w_pycode=PyCode)
+ at unwrap_spec(next_instr=int, is_being_profiled=int, w_pycode=PyCode)
 @dont_look_inside
 def dont_trace_here(space, next_instr, is_being_profiled, w_pycode):
     ll_pycode = cast_instance_to_gcref(w_pycode)
     jit_hooks.dont_trace_here(
-        'pypyjit', r_uint(next_instr), int(is_being_profiled), ll_pycode)
+        'pypyjit', r_uint(next_instr), int(bool(is_being_profiled)), ll_pycode)
     return space.w_None
 
- at unwrap_spec(next_instr=int, is_being_profiled=bool, w_pycode=PyCode)
+ at unwrap_spec(next_instr=int, is_being_profiled=int, w_pycode=PyCode)
 @dont_look_inside
 def trace_next_iteration(space, next_instr, is_being_profiled, w_pycode):
     ll_pycode = cast_instance_to_gcref(w_pycode)
     jit_hooks.trace_next_iteration(
-        'pypyjit', r_uint(next_instr), int(is_being_profiled), ll_pycode)
+        'pypyjit', r_uint(next_instr), int(bool(is_being_profiled)), ll_pycode)
     return space.w_None
 
 @unwrap_spec(hash=r_uint)
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -44,8 +44,8 @@
     else:
         return space.wrap(greenkey_repr)
 
- at unwrap_spec(operations=bool)
-def set_compile_hook(space, w_hook, operations=True):
+ at unwrap_spec(operations=int)
+def set_compile_hook(space, w_hook, operations=1):
     """ set_compile_hook(hook, operations=True)
 
     Set a compiling hook that will be called each time a loop is compiled.
@@ -60,7 +60,7 @@
     cache = space.fromcache(Cache)
     assert w_hook is not None
     cache.w_compile_hook = w_hook
-    cache.compile_hook_with_ops = operations
+    cache.compile_hook_with_ops = bool(operations)
     cache.in_recursion = NonConstant(False)
 
 def set_abort_hook(space, w_hook):
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -169,7 +169,7 @@
         return space.wrap(u"<%s owner=%d count=%d>" % (
                 typename, self.rlock_owner, self.rlock_count))
 
-    @unwrap_spec(blocking=bool, timeout=float)
+    @unwrap_spec(blocking=int, timeout=float)
     def acquire_w(self, space, blocking=True, timeout=-1.0):
         """Lock the lock.  `blocking` indicates whether we should wait
         for the lock to be available or not.  If `blocking` is False
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -677,7 +677,7 @@
             raise oefmt(space.w_ValueError, "%R is not in list", w_value)
         return space.wrap(i)
 
-    @unwrap_spec(reverse=bool)
+    @unwrap_spec(reverse=int)
     def descr_sort(self, space, w_key=None, reverse=False):
         """ L.sort(key=None, reverse=False) -- stable
         sort *IN PLACE*"""
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -564,7 +564,7 @@
 
         return self._newlist_unwrapped(space, res)
 
-    @unwrap_spec(keepends=bool)
+    @unwrap_spec(keepends=int)
     def descr_splitlines(self, space, keepends=False):
         value = self._val(space)
         length = len(value)

From pypy.commits at gmail.com  Mon Sep 12 12:15:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 09:15:47 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: done unwrap_spec(bool)
Message-ID: <57d6d4b3.c19d1c0a.68f16.5e75@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5701:b87aed34f87f
Date: 2016-09-12 18:15 +0200
http://bitbucket.org/pypy/extradoc/changeset/b87aed34f87f/

Log:	done unwrap_spec(bool)

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -40,7 +40,7 @@
 * review all unwrap_spec() that are meant to pass booleans (now
   with '=int').  Argument clinic turns these to PyObject_IsTrue(), i.e.
   accepting any object whatsoever(?!), which is supposedly a feature
-  (see http://bugs.python.org/issue14705).
+  (see http://bugs.python.org/issue14705).  DONE
 
 * ``math.isclose()``
 

From pypy.commits at gmail.com  Mon Sep 12 12:16:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 09:16:53 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: _utf8 DONE in bd9d1dd19c2c
Message-ID: <57d6d4f5.cbc11c0a.10597.8092@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5702:9ccc0fd6b163
Date: 2016-09-12 18:16 +0200
http://bitbucket.org/pypy/extradoc/changeset/9ccc0fd6b163/

Log:	_utf8 DONE in bd9d1dd19c2c

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -52,7 +52,7 @@
   ``SyntaxError: Missing parentheses in call to 'print'``
 
 * ``_utf8`` in W_UnicodeObject used to be quasi-immutable,
-  document why it doesn't work and do a proper fix
+  document why it doesn't work and do a proper fix.  DONE
 
 
 Milestone 1 (Aug-Sep-Oct 2016)

From pypy.commits at gmail.com  Mon Sep 12 12:58:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 09:58:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Another one
Message-ID: <57d6dec9.81091c0a.16a13.3988@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87051:e9ce8f174cd4
Date: 2016-09-12 18:58 +0200
http://bitbucket.org/pypy/pypy/changeset/e9ce8f174cd4/

Log:	Another one

diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -113,8 +113,9 @@
 def newlist_hint(space, sizehint):
     return space.newlist_hint(sizehint)
 
- at unwrap_spec(debug=bool)
+ at unwrap_spec(debug=int)
 def set_debug(space, debug):
+    debug = bool(debug)
     space.sys.debug = debug
     space.setitem(space.builtin.w_dict,
                   space.wrap('__debug__'),

From pypy.commits at gmail.com  Mon Sep 12 13:40:31 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 12 Sep 2016 10:40:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: Using PyType_GenericNew is wrong for a
 metaclass (effectively reverts 6ea4b78fa64b)
Message-ID: <57d6e88f.07c41c0a.297ec.46f2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r87052:9a5ae30e19ff
Date: 2016-09-12 18:39 +0100
http://bitbucket.org/pypy/pypy/changeset/9a5ae30e19ff/

Log:	Using PyType_GenericNew is wrong for a metaclass (effectively
	reverts 6ea4b78fa64b)

diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,7 +4,7 @@
 PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
 {
     PyObject* newType;
-    newType = PyType_GenericNew(metatype, args, kwds);
+    newType = PyType_Type.tp_new(metatype, args, kwds);
     return newType;
 }
 
@@ -71,7 +71,6 @@
     PyObject *mod, *d;
     footype.tp_base = &PyType_Type;
     PyType_Ready(&footype);
-    Py_INCREF(&footype);
     mod = Py_InitModule("foo3", sbkMethods);
     if (mod == NULL)
         return;
@@ -80,4 +79,5 @@
         return;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return;
+    Py_INCREF(&footype);
 }

From pypy.commits at gmail.com  Mon Sep 12 14:04:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 11:04:37 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57d6ee35.e2cdc20a.e94b1.5286@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r797:0148b71de2ee
Date: 2016-09-12 20:04 +0200
http://bitbucket.org/pypy/pypy.org/changeset/0148b71de2ee/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $65099 of $105000 (62.0%)
+   $65104 of $105000 (62.0%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5553 left
+    we have $5557 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Mon Sep 12 15:07:33 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 12 Sep 2016 12:07:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove unecessary check (arigato)
Message-ID: <57d6fcf5.04a81c0a.67389.7048@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87053:59d1974a870e
Date: 2016-09-12 19:32 +0300
http://bitbucket.org/pypy/pypy/changeset/59d1974a870e/

Log:	remove unecessary check (arigato)

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -53,10 +53,7 @@
     else:
         n = len(fmt)
     for i in range(n):
-        if ord(fmt[i]) > 255:
-            view.c_format[i] = '*'
-        else:
-            view.c_format[i] = fmt[i]
+        view.c_format[i] = fmt[i]
     view.c_format[n] = '\x00'        
     shape = buf.getshape()
     strides = buf.getstrides()

From pypy.commits at gmail.com  Mon Sep 12 15:25:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 12:25:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Uh
Message-ID: <57d7013a.04141c0a.488f8.a54c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87054:d79485f5ea36
Date: 2016-09-12 21:25 +0200
http://bitbucket.org/pypy/pypy/changeset/d79485f5ea36/

Log:	Uh

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1232,7 +1232,7 @@
             if e.match(self, self.w_AttributeError):
                 return False
             raise
-        return self.is_true(self.nonzero(w_result))
+        return self.is_true(w_result)
 
     # CPython rules allows subclasses of BaseExceptions to be exceptions.
     # This is slightly less general than the case above, so we prefix

From pypy.commits at gmail.com  Mon Sep 12 15:31:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 12:31:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57d7028a.04141c0a.488f8.a728@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87055:50677989e503
Date: 2016-09-12 20:40 +0100
http://bitbucket.org/pypy/pypy/changeset/50677989e503/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 2301 lines

diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
--- a/pypy/doc/release-pypy2.7-v5.4.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -9,16 +9,16 @@
     this was unfortunately left out of 5.4.0. My apologies to the new
     contributors
 
-  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+  * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a
     dynamically loadable file
 
-  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+  * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython
 
-  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+  * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
 
-  * Fix for an issue where os.access() accepted a float for mode
+  * Fix for an issue for translating the sandbox
 
-  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+  * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
     the last byte of a unicode string sometimes
 
   * Update built-in cffi_ to version 1.8.1
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -12,4 +12,7 @@
 Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
 in numpypy
 
-
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
diff --git a/pypy/module/__pypy__/test/test_bytebuffer.py b/pypy/module/__pypy__/test/test_bytebuffer.py
--- a/pypy/module/__pypy__/test/test_bytebuffer.py
+++ b/pypy/module/__pypy__/test/test_bytebuffer.py
@@ -14,12 +14,9 @@
         assert b[-3] == ord(b'+')
         exc = raises(ValueError, "b[3:5] = b'abc'")
         assert str(exc.value) == "cannot modify size of memoryview object"
-        raises(NotImplementedError, "b[3:7:2] = b'abc'")
 
         b = bytebuffer(10)
         b[1:3] = b'xy'
         assert bytes(b) == b"\x00xy" + b"\x00" * 7
-        # XXX: supported in 3.3
-        raises(NotImplementedError, "b[4:8:2] = b'zw'")
-        #b[4:8:2] = b'zw'
-        #assert bytes(b) == b"\x00xy\x00z\x00w" + b"\x00" * 3
+        b[4:8:2] = b'zw'
+        assert bytes(b) == b"\x00xy\x00z\x00w" + b"\x00" * 3
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -118,7 +118,7 @@
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
@@ -647,7 +647,7 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
-        ('_format', rffi.UCHAR),
+        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
         ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,66 +1,8 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import buffer
-from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer)
-from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
-
-def _IsFortranContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(view.c_ndim):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-
-def _IsCContiguous(view):
-    ndim = widen(view.c_ndim)
-    if ndim == 0:
-        return 1
-    if not view.c_strides:
-        return ndim == 1
-    sd = view.c_itemsize
-    if ndim == 1:
-        return view.c_shape[0] == 1 or sd == view.c_strides[0]
-    for i in range(ndim - 1, -1, -1):
-        dim = view.c_shape[i]
-        if dim == 0:
-            return 1
-        if view.c_strides[i] != sd:
-            return 0
-        sd *= dim
-    return 1
-        
-
- at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fort):
-    """Return 1 if the memory defined by the view is C-style (fortran is
-    'C') or Fortran-style (fortran is 'F') contiguous or either one
-    (fortran is 'A').  Return 0 otherwise."""
-    # traverse the strides, checking for consistent stride increases from
-    # right-to-left (c) or left-to-right (fortran). Copied from cpython
-    if not view.c_suboffsets:
-        return 0
-    if (fort == 'C'):
-        return _IsCContiguous(view)
-    elif (fort == 'F'):
-        return _IsFortranContiguous(view)
-    elif (fort == 'A'):
-        return (_IsCContiguous(view) or _IsFortranContiguous(view))
-    return 0
-
-    
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
+from pypy.module.cpyext.pyobject import PyObject
 
 class CBuffer(buffer.Buffer):
 
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -133,6 +133,7 @@
 
 /* Py3k buffer interface, adapted for PyPy */
 #define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -147,7 +148,7 @@
     Py_ssize_t *shape;
     Py_ssize_t *strides;
     Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
-    unsigned char _format;
+    unsigned char _format[Py_MAX_FMT];
     Py_ssize_t _strides[Py_MAX_NDIMS];
     Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,7 +1,8 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                               Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
 
 from pypy.interpreter.error import oefmt
@@ -10,6 +11,90 @@
 from pypy.objspace.std.memoryobject import W_MemoryView
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
+def fill_Py_buffer(space, buf, view):
+    # c_buf, c_obj have been filled in
+    ndim = buf.getndim()
+    view.c_len = buf.getlength()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        if ord(fmt[i]) > 255:
+            view.c_format[i] = '*'
+        else:
+            view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+ at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+    """Return 1 if the memory defined by the view is C-style (fortran is
+    'C') or Fortran-style (fortran is 'F') contiguous or either one
+    (fortran is 'A').  Return 0 otherwise."""
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
     return space.call_method(space.builtin, "memoryview", w_obj)
@@ -42,20 +127,7 @@
         view.c_obj = make_ref(space, w_s)
         rffi.setintfield(view, 'c_readonly', 1)
         isstr = True
-    view.c_len = w_obj.getlength()
-    view.c_itemsize = w_obj.buf.getitemsize()
-    rffi.setintfield(view, 'c_ndim', ndim)
-    view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
-    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
-    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
-    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
-    shape = w_obj.buf.getshape()
-    strides = w_obj.buf.getstrides()
-    for i in range(ndim):
-        view.c_shape[i] = shape[i]
-        view.c_strides[i] = strides[i]
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    fill_Py_buffer(space, w_obj.buf, view)
     return view
 
 @cpython_api([lltype.Ptr(Py_buffer)], PyObject)
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -192,10 +192,10 @@
     PyObject* obj = PyTuple_GetItem(args, 0);
     PyObject* memoryview = PyMemoryView_FromObject(obj);
     if (memoryview == NULL)
-        return PyInt_FromLong(-1);
+        return PyLong_FromLong(-1);
     view = PyMemoryView_GET_BUFFER(memoryview);
     Py_DECREF(memoryview);
-    return PyInt_FromLong(view->len);
+    return PyLong_FromLong(view->len);
 }
 
 /* Copied from numpy tests */
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
 import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
 
 from .support import c_compile
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
+
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0
@@ -53,7 +55,48 @@
         libraries=libraries)
     return soname
 
-def compile_extension_module(space, modname, include_dirs=[],
+class SystemCompilationInfo(object):
+    """Bundles all the generic information required to compile extensions.
+
+    Note: here, 'system' means OS + target interpreter + test config + ...
+    """
+    def __init__(self, include_extra=None, compile_extra=None, link_extra=None,
+            extra_libs=None, ext=None):
+        self.include_extra = include_extra or []
+        self.compile_extra = compile_extra
+        self.link_extra = link_extra
+        self.extra_libs = extra_libs
+        self.ext = ext
+
+def get_cpyext_info(space):
+    from pypy.module.imp.importing import get_so_extension
+    state = space.fromcache(State)
+    api_library = state.api_lib
+    if sys.platform == 'win32':
+        libraries = [api_library]
+        # '%s' undefined; assuming extern returning int
+        compile_extra = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+            (space.int_w(w_maj), space.int_w(w_min))]
+    else:
+        libraries = []
+        if sys.platform.startswith('linux'):
+            compile_extra = [
+                "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
+    return SystemCompilationInfo(
+        include_extra=api.include_dirs,
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        extra_libs=libraries,
+        ext=get_so_extension(space))
+
+
+def compile_extension_module(sys_info, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
@@ -65,35 +108,15 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    state = space.fromcache(State)
-    api_library = state.api_lib
-    if sys.platform == 'win32':
-        libraries = [api_library]
-        # '%s' undefined; assuming extern returning int
-        compile_extra = ["/we4013"]
-        # prevent linking with PythonXX.lib
-        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
-                              (space.int_w(w_maj), space.int_w(w_min))]
-    else:
-        libraries = []
-        if sys.platform.startswith('linux'):
-            compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
-            link_extra = ["-g"]
-        else:
-            compile_extra = link_extra = None
-
     modname = modname.split('.')[-1]
     soname = create_so(modname,
-            include_dirs=api.include_dirs + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra,
-            libraries=libraries)
-    from pypy.module.imp.importing import get_so_extension
-    ext = get_so_extension(space)
-    pydname = soname.new(purebasename=modname, ext=ext)
+        include_dirs=sys_info.include_extra + include_dirs,
+        source_files=source_files,
+        source_strings=source_strings,
+        compile_extra=sys_info.compile_extra,
+        link_extra=sys_info.link_extra,
+        libraries=sys_info.extra_libs)
+    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
     soname.rename(pydname)
     return str(pydname)
 
@@ -106,18 +129,8 @@
         raise RuntimeError("This interpreter does not define a filename "
             "suffix for C extensions!")
 
-def compile_extension_module_applevel(space, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
+def get_sys_info_app():
+    from distutils.sysconfig import get_python_inc
     if sys.platform == 'win32':
         compile_extra = ["/we4013"]
         link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
@@ -128,18 +141,13 @@
         compile_extra = [
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
         link_extra = None
+    ext = get_so_suffix()
+    return SystemCompilationInfo(
+        include_extra=[get_python_inc()],
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        ext=get_so_suffix())
 
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-            include_dirs=[space.include_dir] + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra)
-    ext = get_so_suffix()
-    pydname = soname.new(purebasename=modname, ext=ext)
-    soname.rename(pydname)
-    return str(pydname)
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -154,9 +162,7 @@
 class FakeSpace(object):
     """Like TinyObjSpace, but different"""
     def __init__(self, config):
-        from distutils.sysconfig import get_python_inc
         self.config = config
-        self.include_dir = get_python_inc()
 
     def passthrough(self, arg):
         return arg
@@ -249,9 +255,9 @@
         # enabled automatically by pypy.conftest.
         return leaking
 
+ at pytest.mark.xfail(reason="Skipped until other tests in this file are unskipped")
 class AppTestApi(LeakCheckingTest):
     def setup_class(cls):
-        skip("Skipped until other tests in this file are unskipped")
         from rpython.rlib.clibffi import get_libc_name
         if cls.runappdirect:
             cls.libc = get_libc_name()
@@ -273,11 +279,11 @@
             "the test actually passed in the first place; if it failed "
             "it is likely to reach this place.")
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_only_import(self):
         import cpyext
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_load_error(self):
         import cpyext
         raises(ImportError, cpyext.load_module, "missing.file", "foo")
@@ -305,26 +311,23 @@
     def setup_method(self, func):
         @gateway.unwrap_spec(name=str)
         def compile_module(space, name,
-                           w_separate_module_files=None,
-                           w_separate_module_sources=None):
+                           w_source_files=None,
+                           w_source_strings=None):
             """
             Build an extension module linked against the cpyext api library.
             """
-            if not space.is_none(w_separate_module_files):
-                separate_module_files = space.unwrap(w_separate_module_files)
-                assert separate_module_files is not None
+            if not space.is_none(w_source_files):
+                source_files = space.unwrap(w_source_files)
             else:
-                separate_module_files = []
-            if not space.is_none(w_separate_module_sources):
-                separate_module_sources = space.listview_bytes(
-                    w_separate_module_sources)
-                assert separate_module_sources is not None
+                source_files = None
+            if not space.is_none(w_source_strings):
+                source_strings = space.listview_bytes(w_source_strings)
             else:
-                separate_module_sources = []
-            pydname = self.compile_extension_module(
-                space, name,
-                source_files=separate_module_files,
-                source_strings=separate_module_sources)
+                source_strings = None
+            pydname = compile_extension_module(
+                self.sys_info, name,
+                source_files=source_files,
+                source_strings=source_strings)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -375,7 +378,7 @@
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
                 kwds = dict(source_files=[filename])
-            mod = self.compile_extension_module(space, name,
+            mod = compile_extension_module(self.sys_info, name,
                     include_dirs=include_dirs, **kwds)
 
             if load_it:
@@ -475,11 +478,11 @@
                 return run
             def wrap(func):
                 return func
-            self.compile_extension_module = compile_extension_module_applevel
+            self.sys_info = get_sys_info_app()
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
-            self.compile_extension_module = compile_extension_module
+            self.sys_info = get_cpyext_info(self.space)
         self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
         self.w_reimport_module = wrap(interp2app(reimport_module))
@@ -629,10 +632,10 @@
             skip('record_imported_module not supported in runappdirect mode')
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", separate_module_files=[self.here + 'banana.c'])
+            "apple.banana", source_files=[self.here + 'banana.c'])
         self.record_imported_module("apple.banana")
         date = self.compile_module(
-            "cherry.date", separate_module_files=[self.here + 'date.c'])
+            "cherry.date", source_files=[self.here + 'date.c'])
         self.record_imported_module("cherry.date")
 
         # Set up some package state so that the extensions can actually be
@@ -901,7 +904,7 @@
             ])
         raises(SystemError, mod.newexc, "name", Exception, {})
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy specific test')
+    @pytest.mark.skipif(only_pypy, reason='pypy specific test')
     def test_hash_pointer(self):
         mod = self.import_extension('foo', [
             ('get_hash', 'METH_NOARGS',
@@ -952,7 +955,7 @@
         print(p)
         assert 'py' in p
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_version(self):
         mod = self.import_extension('foo', [
             ('get_version', 'METH_NOARGS',
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,9 @@
+import pytest
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rlib.buffer import StringBuffer
+
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
         w_hello = space.newbytes("hello")
@@ -15,8 +18,12 @@
         w_buf = space.newbuffer(StringBuffer("hello"))
         w_memoryview = api.PyMemoryView_FromObject(w_buf)
         w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
-        ndim = w_view.c_ndim
-        assert ndim == 1
+        assert w_view.c_ndim == 1
+        f = rffi.charp2str(w_view.c_format)
+        assert f == 'B'
+        assert w_view.c_shape[0] == 5
+        assert w_view.c_strides[0] == 1
+        assert w_view.c_len == 5
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     def test_fillWithObject(self):
@@ -76,18 +83,15 @@
         viewlen = module.test_buffer(arr)
         assert viewlen == y.itemsize * len(y)
 
+    @pytest.mark.skipif(True, reason="no _numpypy on py3k")
     def test_buffer_info(self):
         from _numpypy import multiarray as np
         module = self.import_module(name='buffer_test')
         get_buffer_info = module.get_buffer_info
-        # test_export_flags from numpy test_multiarray
         raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
-        # test_relaxed_strides from numpy test_multiarray
-        arr = np.zeros((1, 10))
-        if arr.flags.f_contiguous:
-            shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
-            assert strides[0] == 8
-            arr = np.ones((10, 1), order='F')
-            shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
-            assert strides[-1] == 8
-
+        arr = np.zeros((1, 10), order='F')
+        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+        assert strides[0] == 8
+        arr = np.zeros((10, 1), order='C')
+        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+        assert strides[-1] == 8
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
         assert module.test_fast_sequence(s[0:-1])
         assert module.test_fast_sequence(s[::-1])
 
+    def test_fast_keyerror(self):
+        module = self.import_extension('foo', [
+            ("test_fast_sequence", "METH_VARARGS",
+             """
+                PyObject *foo;
+                PyObject * seq = PyTuple_GetItem(args, 0);
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "Could not convert object to sequence");
+                if (foo != NULL)
+                {
+                    return foo;
+                }
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                    return PyBool_FromLong(1);
+                }
+                return NULL;
+             """)])
+        class Map(object):
+            def __len__(self):
+                return 1
+
+            def __getitem__(self, index):
+                raise KeyError()
+
+        assert module.test_fast_sequence(Map()) is True
+
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -1,12 +1,13 @@
 import sys
 
-import py, pytest
+import pytest
 
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 class AppTestThread(AppTestCpythonExtensionBase):
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_thread_ident(self):
         module = self.import_extension('foo', [
             ("get_thread_ident", "METH_NOARGS",
@@ -33,7 +34,7 @@
 
         assert results[0][0] != results[1][0]
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_acquire_lock(self):
         module = self.import_extension('foo', [
             ("test_acquire_lock", "METH_NOARGS",
@@ -57,7 +58,7 @@
             ])
         module.test_acquire_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_release_lock(self):
         module = self.import_extension('foo', [
             ("test_release_lock", "METH_NOARGS",
@@ -79,7 +80,7 @@
             ])
         module.test_release_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_tls(self):
         module = self.import_extension('foo', [
             ("create_key", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,6 +3,7 @@
 import py, pytest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 def test_pragma_version():
     from pypy.module.sys.version import CPYTHON_VERSION
@@ -41,11 +42,9 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -95,8 +95,9 @@
              )
     setuppkg("pkg.pkg2", a='', b='')
     setuppkg("pkg.withall",
-             __init__  = "__all__ = ['foobar']",
-             foobar    = "found = 123")
+             __init__  = "__all__ = ['foobar', 'barbaz']",
+             foobar    = "found = 123",
+             barbaz    = "other = 543")
     setuppkg("pkg.withoutall",
              __init__  = "",
              foobar    = "found = 123")
@@ -724,6 +725,7 @@
             d = {}
             exec("from pkg.withall import *", d)
             assert d["foobar"].found == 123
+            assert d["barbaz"].other == 543
 
     def test_import_star_does_not_find_submodules_without___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -392,31 +392,39 @@
         extobj_w = space.newlist([space.wrap(8192), space.wrap(0), space.w_None])
         return extobj_w
 
+
+_reflected_ops = {
+        'add': 'radd',
+        'subtract': 'rsub',
+        'multiply': 'rmul',
+        'divide': 'rdiv',
+        'true_divide': 'rtruediv',
+        'floor_divide': 'rfloordiv',
+        'remainder': 'rmod',
+        'power': 'rpow',
+        'left_shift': 'rlshift',
+        'right_shift': 'rrshift',
+        'bitwise_and': 'rand',
+        'bitwise_xor': 'rxor',
+        'bitwise_or': 'ror',
+        #/* Comparisons */
+        'equal': 'eq',
+        'not_equal': 'ne',
+        'greater': 'lt',
+        'less': 'gt',
+        'greater_equal': 'le',
+        'less_equal': 'ge',
+}
+
+for key, value in _reflected_ops.items():
+    _reflected_ops[key] = "__" + value + "__"
+del key
+del value
+
 def _has_reflected_op(space, w_obj, op):
-    refops ={ 'add': 'radd',
-            'subtract': 'rsub',
-            'multiply': 'rmul',
-            'divide': 'rdiv',
-            'true_divide': 'rtruediv',
-            'floor_divide': 'rfloordiv',
-            'remainder': 'rmod',
-            'power': 'rpow',
-            'left_shift': 'rlshift',
-            'right_shift': 'rrshift',
-            'bitwise_and': 'rand',
-            'bitwise_xor': 'rxor',
-            'bitwise_or': 'ror',
-            #/* Comparisons */
-            'equal': 'eq',
-            'not_equal': 'ne',
-            'greater': 'lt',
-            'less': 'gt',
-            'greater_equal': 'le',
-            'less_equal': 'ge',
-        }
-    if op not in refops:
+    if op not in _reflected_ops:
         return False
-    return space.getattr(w_obj, space.wrap('__' + refops[op] + '__')) is not None
+    return space.getattr(w_obj, space.wrap(_reflected_ops[op])) is not None
 
 def safe_casting_mode(casting):
     assert casting is not None
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,7 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(ll_strhash), p10, descr=<Calli . r EF=0>)
+            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -7,6 +7,7 @@
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
     unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii)
+from rpython.rlib import jit
 
 from pypy.interpreter import unicodehelper
 from pypy.interpreter.baseobjspace import W_Root
@@ -25,7 +26,7 @@
 
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
-    _immutable_fields_ = ['_value', '_utf8?']
+    _immutable_fields_ = ['_value']
 
     def __init__(self, unistr):
         assert isinstance(unistr, unicode)
@@ -76,6 +77,8 @@
     def unicode_w(self, space):
         return self._value
 
+    @jit.elidable
+    @jit.call_shortcut
     def identifier_w(self, space):
         identifier = self._utf8
         if identifier is not None:
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1002,6 +1002,9 @@
     prepare_op_cond_call_gc_wb_array = prepare_op_cond_call_gc_wb
 
     def prepare_op_cond_call(self, op, fcond):
+        # XXX don't force the arguments to be loaded in specific
+        # locations before knowing if we can take the fast path
+        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -325,6 +325,7 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
+    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_extension = True
@@ -1334,6 +1335,16 @@
         # cond_call can't have a return value
         self.execute_call_n(calldescr, func, *args)
 
+    def execute_cond_call_value_i(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_i(calldescr, func, *args)
+        return value
+
+    def execute_cond_call_value_r(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_r(calldescr, func, *args)
+        return value
+
     def _execute_call(self, calldescr, func, *args):
         effectinfo = calldescr.get_extra_info()
         if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'):
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -759,6 +759,8 @@
         if (opnum != rop.GUARD_TRUE and opnum != rop.GUARD_FALSE
                                     and opnum != rop.COND_CALL):
             return False
+        # NB: don't list COND_CALL_VALUE_I/R here, these two variants
+        # of COND_CALL don't accept a cc as input
         if next_op.getarg(0) is not op:
             return False
         if self.longevity[op][1] > i + 1:
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -11,7 +11,7 @@
 from rpython.jit.backend.llsupport.symbolic import (WORD,
         get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER
+     FLAG_POINTER, CallDescr
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -370,7 +370,9 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- call assembler -----------
+            # ---------- calls -----------
+            if OpHelpers.is_plain_call(op.getopnum()):
+                self.expand_call_shortcut(op)
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -616,6 +618,30 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
+    def expand_call_shortcut(self, op):
+        if not self.cpu.supports_cond_call_value:
+            return
+        descr = op.getdescr()
+        if descr is None:
+            return
+        assert isinstance(descr, CallDescr)
+        effectinfo = descr.get_extra_info()
+        if effectinfo is None or effectinfo.call_shortcut is None:
+            return
+        if op.type == 'r':
+            cond_call_opnum = rop.COND_CALL_VALUE_R
+        elif op.type == 'i':
+            cond_call_opnum = rop.COND_CALL_VALUE_I
+        else:
+            return
+        cs = effectinfo.call_shortcut
+        ptr_box = op.getarg(1 + cs.argnum)
+        value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
+                                       raw=(ptr_box.type == 'i'))
+        self.replace_op_with(op, ResOperation(cond_call_opnum,
+                                              [value_box] + op.getarglist(),
+                                              descr=descr))
+
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,7 +1,8 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr
+     SizeDescr, get_interiorfield_descr, get_call_descr
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -80,6 +81,14 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
+        call_shortcut = CallShortcut(0, tzdescr)
+        effectinfo = EffectInfo(None, None, None, None, None, None,
+                                EffectInfo.EF_RANDOM_EFFECTS,
+                                call_shortcut=call_shortcut)
+        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
+            [lltype.Ptr(T)], lltype.Signed,
+            effectinfo)
+        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -200,6 +209,7 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
+    supports_cond_call_value = True
 
     translate_support_code = None
 
@@ -1429,3 +1439,15 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
+
+    def test_handle_call_shortcut(self):
+        self.check_rewrite("""
+            [p0]
+            i1 = call_i(123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """, """
+            [p0]
+            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
+            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,6 +16,7 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
+    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2389,7 +2389,7 @@
             f2 = longlong.getfloatstorage(3.4)
             frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5, f1, f2)
             assert not called
-            for j in range(5):
+            for j in range(6):
                 assert self.cpu.get_int_value(frame, j) == j
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 6)) == 1.2
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 3.4
@@ -2447,6 +2447,54 @@
                                            67, 89)
             assert called == [(67, 89)]
 
+    def test_cond_call_value(self):
+        if not self.cpu.supports_cond_call_value:
+            py.test.skip("missing supports_cond_call_value")
+
+        def func_int(*args):
+            called.append(args)
+            return len(args) * 100 + 1000
+
+        for i in range(5):
+            called = []
+
+            FUNC = self.FuncType([lltype.Signed] * i, lltype.Signed)
+            func_ptr = llhelper(lltype.Ptr(FUNC), func_int)
+            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                             EffectInfo.MOST_GENERAL)
+
+            ops = '''
+            [i0, i1, i2, i3, i4, i5, i6, f0, f1]
+            i15 = cond_call_value_i(i1, ConstClass(func_ptr), %s)
+            guard_false(i0, descr=faildescr) [i1,i2,i3,i4,i5,i6,i15, f0,f1]
+            finish(i15)
+            ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] +
+                            ["descr=calldescr"])
+            loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
+                                         'func_ptr': func_ptr,
+                                         'calldescr': calldescr})
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+            f1 = longlong.getfloatstorage(1.2)
+            f2 = longlong.getfloatstorage(3.4)
+            frame = self.cpu.execute_token(looptoken, 1, 50, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert not called
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        50, 1, 2, 3, 4, 5, 50]
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 8)) == 3.4
+            #
+            frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert called == [(1, 2, 3, 4)[:i]]
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        0, 1, 2, 3, 4, 5, i * 100 + 1000]
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 8)) == 3.4
+
     def test_force_operations_returning_void(self):
         values = []
         def maybe_force(token, flag):
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -594,7 +594,7 @@
         return subset, d['f'], vtableptr
 
     def getresulttype(self):
-        if self.opnum == rop.CALL_I:
+        if self.opnum == rop.CALL_I or self.opnum == rop.COND_CALL_VALUE_I:
             return lltype.Signed
         elif self.opnum == rop.CALL_F:
             return lltype.Float
@@ -712,7 +712,12 @@
 class CondCallOperation(BaseCallOperation):
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
-        v_cond = builder.get_bool_var(r)
+        if self.opnum == rop.COND_CALL:
+            RESULT_TYPE = lltype.Void
+            v_cond = builder.get_bool_var(r)
+        else:
+            RESULT_TYPE = lltype.Signed
+            v_cond = r.choice(builder.intvars)
         subset = builder.subset_of_intvars(r)[:4]
         for i in range(len(subset)):
             if r.random() < 0.35:
@@ -724,8 +729,10 @@
                 seen.append(args)
             else:
                 assert seen[0] == args
+            if RESULT_TYPE is lltype.Signed:
+                return len(args) - 42000
         #
-        TP = lltype.FuncType([lltype.Signed] * len(subset), lltype.Void)
+        TP = lltype.FuncType([lltype.Signed] * len(subset), RESULT_TYPE)
         ptr = llhelper(lltype.Ptr(TP), call_me)
         c_addr = ConstAddr(llmemory.cast_ptr_to_adr(ptr), builder.cpu)
         args = [v_cond, c_addr] + subset
@@ -769,6 +776,7 @@
 for i in range(2):
     OPERATIONS.append(GuardClassOperation(rop.GUARD_CLASS))
     OPERATIONS.append(CondCallOperation(rop.COND_CALL))
+    OPERATIONS.append(CondCallOperation(rop.COND_CALL_VALUE_I))
     OPERATIONS.append(RaisingCallOperation(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationGuardNoException(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationWrongGuardException(rop.CALL_N))
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -174,8 +174,8 @@
         # copy registers to the frame, with the exception of the
         # 'cond_call_register_arguments' and eax, because these have already
         # been saved by the caller.  Note that this is not symmetrical:
-        # these 5 registers are saved by the caller but restored here at
-        # the end of this function.
+        # these 5 registers are saved by the caller but 4 of them are
+        # restored here at the end of this function.
         self._push_all_regs_to_frame(mc, cond_call_register_arguments + [eax],
                                      supports_floats, callee_only)
         # the caller already did push_gcmap(store=True)
@@ -198,7 +198,7 @@
             mc.ADD(esp, imm(WORD * 7))
         self.set_extra_stack_depth(mc, 0)
         self.pop_gcmap(mc)   # cancel the push_gcmap(store=True) in the caller
-        self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only)
+        self._pop_all_regs_from_frame(mc, [eax], supports_floats, callee_only)
         mc.RET()
         return mc.materialize(self.cpu, [])
 
@@ -1703,7 +1703,8 @@
         self.implement_guard(guard_token)
         # If the previous operation was a COND_CALL, overwrite its conditional
         # jump to jump over this GUARD_NO_EXCEPTION as well, if we can
-        if self._find_nearby_operation(-1).getopnum() == rop.COND_CALL:
+        if self._find_nearby_operation(-1).getopnum() in (
+                rop.COND_CALL, rop.COND_CALL_VALUE_I, rop.COND_CALL_VALUE_R):
             jmp_adr = self.previous_cond_call_jcond
             offset = self.mc.get_relative_pos() - jmp_adr
             if offset <= 127:
@@ -2381,7 +2382,7 @@
     def label(self):
         self._check_frame_depth_debug(self.mc)
 
-    def cond_call(self, op, gcmap, imm_func, arglocs):
+    def cond_call(self, gcmap, imm_func, arglocs, resloc=None):
         assert self.guard_success_cc >= 0
         self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0)
                                                             # patched later
@@ -2394,11 +2395,14 @@
         # plus the register 'eax'
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         should_be_saved = self._regalloc.rm.reg_bindings.values()
+        restore_eax = False
         for gpr in cond_call_register_arguments + [eax]:
-            if gpr not in should_be_saved:
+            if gpr not in should_be_saved or gpr is resloc:
                 continue
             v = gpr_reg_mgr_cls.all_reg_indexes[gpr.value]
             self.mc.MOV_br(v * WORD + base_ofs, gpr.value)
+            if gpr is eax:
+                restore_eax = True
         #
         # load the 0-to-4 arguments into these registers
         from rpython.jit.backend.x86.jump import remap_frame_layout
@@ -2422,8 +2426,16 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.CALL(imm(follow_jump(cond_call_adr)))
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        if resloc is not None and resloc is not eax:
+            self.mc.MOV(resloc, eax)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  We must only restore eax, if
+        # needed.
+        if restore_eax:
+            v = gpr_reg_mgr_cls.all_reg_indexes[eax.value]
+            self.mc.MOV_rb(eax.value, v * WORD + base_ofs)
+        #
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -938,16 +938,45 @@
                     self.rm.force_spill_var(box)
                     assert box not in self.rm.reg_bindings
         #
-        assert op.type == 'v'
         args = op.getarglist()
         assert 2 <= len(args) <= 4 + 2     # maximum 4 arguments
-        v = args[1]
-        assert isinstance(v, Const)
-        imm_func = self.rm.convert_to_imm(v)
+        v_func = args[1]
+        assert isinstance(v_func, Const)
+        imm_func = self.rm.convert_to_imm(v_func)
+
+        # Delicate ordering here.  First get the argument's locations.
+        # If this also contains args[0], this returns the current
+        # location too.
         arglocs = [self.loc(args[i]) for i in range(2, len(args))]
         gcmap = self.get_gcmap()
-        self.load_condition_into_cc(op.getarg(0))
-        self.assembler.cond_call(op, gcmap, imm_func, arglocs)
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            resloc = None
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+
+            # Test the register for the result.
+            self.assembler.test_location(resloc)
+            self.assembler.guard_success_cc = rx86.Conditions['Z']
+
+        self.assembler.cond_call(gcmap, imm_func, arglocs, resloc)
+
+    consider_cond_call_value_i = consider_cond_call
+    consider_cond_call_value_r = consider_cond_call
 
     def consider_call_malloc_nursery(self, op):
         size_box = op.getarg(0)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -15,6 +15,7 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
+    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,9 +7,10 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection)
+    EffectInfo, CallInfoCollection, CallShortcut)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
+from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -214,6 +215,7 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
+        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -228,6 +230,12 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
+            if hasattr(funcobj, 'graph'):
+                call_shortcut = self.find_call_shortcut(funcobj.graph)
+            if getattr(func, "_call_shortcut_", False):
+                assert call_shortcut is not None, (
+                    "%r: marked as @jit.call_shortcut but shortcut not found"
+                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -298,6 +306,7 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
+            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -368,3 +377,65 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
+
+    def find_call_shortcut(self, graph):
+        """Identifies graphs that start like this:
+
+           def graph(x, y, z):         def graph(x, y, z):
+               if y.field:                 r = y.field
+                   return y.field          if r: return r
+        """
+        block = graph.startblock
+        if len(block.operations) == 0:
+            return
+        op = block.operations[0]
+        if op.opname != 'getfield':
+            return
+        [v_inst, c_fieldname] = op.args
+        if not isinstance(v_inst, Variable):
+            return
+        v_result = op.result
+        if v_result.concretetype != graph.getreturnvar().concretetype:
+            return
+        if v_result.concretetype == lltype.Void:
+            return
+        argnum = i = 0
+        while block.inputargs[i] is not v_inst:
+            if block.inputargs[i].concretetype != lltype.Void:
+                argnum += 1
+            i += 1
+        PSTRUCT = v_inst.concretetype
+        v_check = v_result
+        fastcase = True
+        for op in block.operations[1:]:
+            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
+                    and v_check is op.args[0]):
+                v_check = op.result
+            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
+                v_check = op.result
+                fastcase = not fastcase
+            elif (op.opname in ('int_eq', 'int_ne')
+                    and v_check is op.args[0]
+                    and isinstance(op.args[1], Constant)
+                    and op.args[1].value == 0):
+                v_check = op.result
+                if op.opname == 'int_eq':
+                    fastcase = not fastcase
+            else:
+                return
+        if v_check.concretetype is not lltype.Bool:
+            return
+        if block.exitswitch is not v_check:
+            return
+
+        links = [link for link in block.exits if link.exitcase == fastcase]
+        if len(links) != 1:
+            return
+        [link] = links
+        if link.args != [v_result]:
+            return
+        if not link.target.is_final_block():
+            return
+
+        fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,7 +117,8 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True):
+                can_collect=True,
+                call_shortcut=None):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -135,7 +136,8 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect)
+               can_collect,
+               call_shortcut)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -190,6 +192,7 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
+        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -275,7 +278,8 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True):
+                                 can_collect=True,
+                                 call_shortcut=None):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -364,7 +368,8 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect)
+                      can_collect,
+                      call_shortcut)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -387,6 +392,24 @@
 
 # ____________________________________________________________
 
+
+class CallShortcut(object):
+    def __init__(self, argnum, fielddescr):
+        self.argnum = argnum
+        self.fielddescr = fielddescr
+
+    def __eq__(self, other):
+        return (isinstance(other, CallShortcut) and
+                self.argnum == other.argnum and
+                self.fielddescr == other.fielddescr)
+    def __ne__(self, other):
+        return not (self == other)
+    def __hash__(self):
+        return hash((self.argnum, self.fielddescr))
+
+# ____________________________________________________________
+
+
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -364,7 +364,7 @@
         return getattr(self, 'handle_%s_indirect_call' % kind)(op)
 
     def rewrite_call(self, op, namebase, initialargs, args=None,
-                     calldescr=None):
+                     calldescr=None, force_ir=False):
         """Turn 'i0 = direct_call(fn, i1, i2, ref1, ref2)'
            into 'i0 = xxx_call_ir_i(fn, descr, [i1,i2], [ref1,ref2])'.
            The name is one of '{residual,direct}_call_{r,ir,irf}_{i,r,f,v}'."""
@@ -374,8 +374,9 @@
         lst_i, lst_r, lst_f = self.make_three_lists(args)
         reskind = getkind(op.result.concretetype)[0]
         if lst_f or reskind == 'f': kinds = 'irf'
-        elif lst_i: kinds = 'ir'
+        elif lst_i or force_ir: kinds = 'ir'
         else: kinds = 'r'
+        if force_ir: assert kinds == 'ir'    # no 'f'
         sublists = []
         if 'i' in kinds: sublists.append(lst_i)
         if 'r' in kinds: sublists.append(lst_r)
@@ -1577,7 +1578,7 @@
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
         op1 = self.rewrite_call(op, 'conditional_call',
                                 op.args[:2], args=op.args[2:],
-                                calldescr=calldescr)
+                                calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl
-from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 
 
 class FakePolicy:
@@ -368,3 +368,100 @@
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.check_can_collect() == expected
+
+def test_find_call_shortcut():
+    class FakeCPU:
+        def fielddescrof(self, TYPE, fieldname):
+            if isinstance(TYPE, lltype.GcStruct):
+                if fieldname == 'inst_foobar':
+                    return 'foobardescr'
+                if fieldname == 'inst_fooref':
+                    return 'foorefdescr'
+            if TYPE == RAW and fieldname == 'x':
+                return 'xdescr'
+            assert False, (TYPE, fieldname)
+    cc = CallControl(FakeCPU())
+
+    class B(object):
+        foobar = 0
+        fooref = None
+
+    def f1(a, b, c):
+        if b.foobar:
+            return b.foobar
+        b.foobar = a + c
+        return b.foobar
+
+    def f2(x, y, z, b):
+        r = b.fooref
+        if r is not None:
+            return r
+        r = b.fooref = B()
+        return r
+
+    class Space(object):
+        def _freeze_(self):
+            return True
+    space = Space()
+
+    def f3(space, b):
+        r = b.foobar
+        if not r:
+            r = b.foobar = 123
+        return r
+
+    def f4(raw):
+        r = raw.x
+        if r != 0:
+            return r
+        raw.x = 123
+        return 123
+    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
+
+    def f5(b):
+        r = b.foobar
+        if r == 0:
+            r = b.foobar = 123
+        return r
+
+    def f(a, c):
+        b = B()
+        f1(a, b, c)
+        f2(a, c, a, b)
+        f3(space, b)
+        r = lltype.malloc(RAW, flavor='raw')
+        f4(r)
+        f5(b)
+
+    rtyper = support.annotate(f, [10, 20])
+    f1_graph = rtyper.annotator.translator._graphof(f1)
+    assert cc.find_call_shortcut(f1_graph) == CallShortcut(1, "foobardescr")
+    f2_graph = rtyper.annotator.translator._graphof(f2)
+    assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
+    f3_graph = rtyper.annotator.translator._graphof(f3)
+    assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
+    f4_graph = rtyper.annotator.translator._graphof(f4)
+    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
+    f5_graph = rtyper.annotator.translator._graphof(f5)
+    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
+
+def test_cant_find_call_shortcut():
+    from rpython.jit.backend.llgraph.runner import LLGraphCPU
+
+    @jit.dont_look_inside
+    @jit.call_shortcut
+    def f1(n):
+        return n + 17   # no call shortcut found
+
+    def f(n):
+        return f1(n)
+
+    rtyper = support.annotate(f, [1])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
+    res = cc.find_all_graphs(FakePolicy())
+    [f_graph] = [x for x in res if x.func is f]
+    call_op = f_graph.startblock.operations[0]
+    assert call_op.opname == 'direct_call'
+    e = py.test.raises(AssertionError, cc.getcalldescr, call_op)
+    assert "shortcut not found" in str(e.value)
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1200,28 +1200,12 @@
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
     # conditional calls - note that they cannot return stuff
-    @arguments("cpu", "i", "i", "I", "d")
-    def bhimpl_conditional_call_i_v(cpu, condition, func, args_i, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, None, None, calldescr)
-
-    @arguments("cpu", "i", "i", "R", "d")
-    def bhimpl_conditional_call_r_v(cpu, condition, func, args_r, calldescr):
-        if condition:
-            cpu.bh_call_v(func, None, args_r, None, calldescr)
-
     @arguments("cpu", "i", "i", "I", "R", "d")
     def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
                                      calldescr):
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
-    @arguments("cpu", "i", "i", "I", "R", "F", "d")
-    def bhimpl_conditional_call_irf_v(cpu, condition, func, args_i, args_r,
-                                      args_f, calldescr):
-        if condition:
-            cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
-
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -101,6 +101,18 @@
     if condbox.getint():
         do_call_n(cpu, metainterp, argboxes[1:], descr)
 
+def do_cond_call_value_i(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getint()
+    if value == 0:
+        value = do_call_i(cpu, metainterp, argboxes[1:], descr)
+    return value
+
+def do_cond_call_value_r(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getref_base()
+    if not value:
+        value = do_call_r(cpu, metainterp, argboxes[1:], descr)
+    return value
+
 def do_getarrayitem_gc_i(cpu, _, arraybox, indexbox, arraydescr):
     array = arraybox.getref_base()
     index = indexbox.getint()
@@ -366,6 +378,8 @@
                          rop.CALL_ASSEMBLER_I,
                          rop.CALL_ASSEMBLER_N,
                          rop.INCREMENT_DEBUG_COUNTER,
+                         rop.COND_CALL_VALUE_R,
+                         rop.COND_CALL_VALUE_I,
                          rop.COND_CALL_GC_WB,
                          rop.COND_CALL_GC_WB_ARRAY,
                          rop.ZERO_ARRAY,
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -365,6 +365,13 @@
     def visitor_dispatch_virtual_type(self, visitor):
         raise NotImplementedError("abstract")
 
+    def make_guards(self, op, short, optimizer):
+        from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
+        op = ResOperation(rop.INT_EQ, [op, CONST_0])
+        short.append(op)
+        op = ResOperation(rop.GUARD_FALSE, [op])
+        short.append(op)
+
 class RawBufferPtrInfo(AbstractRawPtrInfo):
     buffer = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7592,7 +7592,7 @@
         ops = """
         [i0]
         p1 = new_with_vtable(descr=nodesize)
-        cond_call(1, 123, p1, descr=clear_vable)
+        cond_call(i0, 123, p1, descr=clear_vable)
         jump(i0)
         """
         expected = """
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,7 +16,7 @@
 from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
      have_debug_prints
 
-class UnrollableOptimizer(Optimizer):    
+class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             if self.optunroll.short_preamble_producer is None:
@@ -120,7 +120,8 @@
                 assert op.get_forwarded() is None
         if check_newops:
             assert not self.optimizer._newoperations
-    
+
+
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
@@ -156,7 +157,7 @@
         current_vs = self.get_virtual_state(end_jump.getarglist())
         # pick the vs we want to jump to
         assert isinstance(celltoken, JitCellToken)
-        
+
         target_virtual_state = self.pick_virtual_state(current_vs,
                                                        state.virtual_state,
                                                 celltoken.target_tokens)
@@ -180,17 +181,27 @@
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
-                    self.optimizer._newoperations)            
+                    self.optimizer._newoperations)
 
         try:
-            new_virtual_state = self.jump_to_existing_trace(end_jump, label_op,
-                                                            state.runtime_boxes)
+            new_virtual_state = self.jump_to_existing_trace(
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
                                self.optimizer.quasi_immutable_deps),
                     self.optimizer._newoperations)
+
+        if new_virtual_state is not None:
+            # Attempt to force virtual boxes in order to avoid jumping
+            # to the preamble.
+            try:
+                new_virtual_state = self.jump_to_existing_trace(
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+            except InvalidLoop:
+                pass
+
         if new_virtual_state is not None:
             self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, extra_same_as,
@@ -199,7 +210,7 @@
 
         self.disable_retracing_if_max_retrace_guards(
             self.optimizer._newoperations, target_token)
-        
+
         return (UnrollInfo(target_token, label_op, extra_same_as,
                            self.optimizer.quasi_immutable_deps),
                 self.optimizer._newoperations)
@@ -241,7 +252,8 @@
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
         try:
-            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes)
+            vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                             force_boxes=False)
         except InvalidLoop:
             return self.jump_to_preamble(cell_token, jump_op, info)
         if vs is None:
@@ -252,6 +264,14 @@
             cell_token.retraced_count += 1
             debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
         else:
+            # Try forcing boxes to avoid jumping to the preamble
+            try:
+                vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+                                                 force_boxes=True)
+            except InvalidLoop:
+                pass
+            if vs is None:
+                return info, self.optimizer._newoperations[:]
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
@@ -288,7 +308,7 @@
         return info, self.optimizer._newoperations[:]
 
 
-    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes):
+    def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False):
         jitcelltoken = jump_op.getdescr()
         assert isinstance(jitcelltoken, JitCellToken)
         virtual_state = self.get_virtual_state(jump_op.getarglist())
@@ -299,7 +319,8 @@
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, args, runtime_boxes, self.optimizer)
+                    virtual_state, args, runtime_boxes, self.optimizer,
+                    force_boxes=force_boxes)
                 patchguardop = self.optimizer.patchguardop
                 for guard in extra_guards.extra_guards:
                     if isinstance(guard, GuardResOp):
@@ -308,8 +329,18 @@
                     self.send_extra_operation(guard)
             except VirtualStatesCantMatch:
                 continue
-            args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
-                args, self.optimizer)
+
+            # When force_boxes == True, creating the virtual args can fail when
+            # components of the virtual state alias. If this occurs, we must
+            # recompute the virtual state as boxes will have been forced.
+            try:
+                args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
+                    args, self.optimizer, force_boxes=force_boxes)
+            except VirtualStatesCantMatch:
+                assert force_boxes
+                virtual_state = self.get_virtual_state(args)
+                continue
+
             short_preamble = target_token.short_preamble
             try:
                 extra = self.inline_short_preamble(args + virtuals, args,
@@ -452,7 +483,7 @@
         # by short preamble
         label_args = exported_state.virtual_state.make_inputargs(
             targetargs, self.optimizer)
-        
+
         self.short_preamble_producer = ShortPreambleBuilder(
             label_args, exported_state.short_boxes,
             exported_state.short_inputargs, exported_state.exported_infos,
@@ -497,7 +528,7 @@
     * runtime_boxes - runtime values for boxes, necessary when generating
                       guards to jump to
     """
-    
+
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
                  short_inputargs, runtime_boxes, memo):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
      ArrayStructInfo, AbstractStructPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound, IntLowerBound
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation, \
      InputArgInt, InputArgRef, InputArgFloat
 from rpython.rlib.debug import debug_print
 
@@ -20,7 +20,7 @@
 
 
 class GenerateGuardState(object):
-    def __init__(self, optimizer=None, guards=None, renum=None, bad=None):
+    def __init__(self, optimizer=None, guards=None, renum=None, bad=None, force_boxes=False):
         self.optimizer = optimizer
         self.cpu = optimizer.cpu
         if guards is None:
@@ -32,6 +32,7 @@
         if bad is None:
             bad = {}
         self.bad = bad
+        self.force_boxes = force_boxes
 
     def get_runtime_item(self, box, descr, i):
         array = box.getref_base()
@@ -303,7 +304,7 @@
             opinfo = state.optimizer.getptrinfo(box)
             assert isinstance(opinfo, ArrayPtrInfo)
         else:
-            opinfo = None            
+            opinfo = None
         for i in range(self.length):
             for descr in self.fielddescrs:
                 index = i * len(self.fielddescrs) + descr.get_index()
@@ -514,6 +515,8 @@
         NotVirtualStateInfo.__init__(self, cpu, type, info)
 
     def _generate_guards(self, other, box, runtime_box, state):
+        if state.force_boxes and isinstance(other, VirtualStateInfo):
+            return self._generate_virtual_guards(other, box, runtime_box, state)
         if not isinstance(other, NotVirtualStateInfoPtr):
             raise VirtualStatesCantMatch(
                     'The VirtualStates does not match as a ' +
@@ -545,6 +548,23 @@
     # to an existing compiled loop or retracing the loop. Both alternatives
     # will always generate correct behaviour, but performance will differ.
 
+    def _generate_virtual_guards(self, other, box, runtime_box, state):
+        """
+        Generate the guards and add state information for unifying a virtual
+        object with a non-virtual. This involves forcing the object in the
+        event that unification can succeed. Since virtual objects cannot be null,
+        this method need only check that the virtual object has the expected type.
+        """
+        assert state.force_boxes and isinstance(other, VirtualStateInfo)
+
+        if self.level == LEVEL_CONSTANT:
+            raise VirtualStatesCantMatch(
+                    "cannot unify a constant value with a virtual object")
+
+        if self.level == LEVEL_KNOWNCLASS:
+            if not self.known_class.same_constant(other.known_class):
+                raise VirtualStatesCantMatch("classes don't match")
+
     def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
                                  state):
         if not isinstance(other, NotVirtualStateInfoPtr):
@@ -617,10 +637,10 @@
             return False
         return True
 
-    def generate_guards(self, other, boxes, runtime_boxes, optimizer):
+    def generate_guards(self, other, boxes, runtime_boxes, optimizer, force_boxes=False):
         assert (len(self.state) == len(other.state) == len(boxes) ==
                 len(runtime_boxes))
-        state = GenerateGuardState(optimizer)
+        state = GenerateGuardState(optimizer, force_boxes=force_boxes)

From pypy.commits at gmail.com  Mon Sep 12 15:48:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Sep 2016 12:48:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Restore missing import
Message-ID: <57d70695.cbc11c0a.10597.cf26@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87056:243dab483e12
Date: 2016-09-12 20:57 +0100
http://bitbucket.org/pypy/pypy/changeset/243dab483e12/

Log:	Restore missing import

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -2,7 +2,7 @@
 from rpython.rlib import buffer
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
 class CBuffer(buffer.Buffer):
 

From pypy.commits at gmail.com  Tue Sep 13 03:31:23 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 13 Sep 2016 00:31:23 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase again
Message-ID: <57d7ab4b.6222c20a.1432.001a@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87057:2b705e789377
Date: 2016-09-12 21:13 +0200
http://bitbucket.org/pypy/pypy/changeset/2b705e789377/

Log:	Update qcgc codebase again

diff too long, truncating to 2000 out of 5319 lines

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -898,9 +898,7 @@
             includes = ["qcgc.h"],
             separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
-                ["qcgc.c", "arena.c", "allocator.c", "bag.c", "event_logger.c",
-                    "gray_stack.c", "shadow_stack.c", "hugeblocktable.c",
-                    "signal_handler.c"]],
+                ["qcgc.c"]],
             )
     return eci
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -946,10 +946,10 @@
 
     def OP_QCGC_PUSH_ROOT(self, op):
         obj = self.expr(op.args[0])
-        return 'qcgc_shadowstack_push((object_t *) %s);' % (obj,)
+        return 'qcgc_push_root((object_t *) %s);' % (obj,)
 
     def OP_QCGC_POP_ROOT(self, op):
-        return 'qcgc_shadowstack_pop();'
+        return 'qcgc_pop_root();'
 
     def OP_QCGC_ALLOCATE(self, op):
         # XXX: SET typeid
@@ -967,4 +967,4 @@
     def OP_QCGC_REGISTER_WEAKREF(self, op):
         weakref = self.expr(op.args[0])
         fieldaddr = self.expr(op.args[1])
-        return 'qcgc_register_weakref(%s, %s);' % (weakref, fieldaddr)
+        return 'qcgc_register_weakref((object_t *)%s, (object_t **)%s);' % (weakref, fieldaddr)
diff --git a/rpython/translator/c/src/qcgc/allocator.c b/rpython/translator/c/src/qcgc/allocator.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/allocator.c
+++ /dev/null
@@ -1,428 +0,0 @@
-#include "allocator.h"
-
-#include <assert.h>
-#include <stdbool.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "hugeblocktable.h"
-
-QCGC_STATIC size_t bytes_to_cells(size_t bytes);
-
-QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells);
-QCGC_STATIC void bump_allocator_advance(size_t cells);
-QCGC_STATIC void bump_allocator_renew_block(void);
-
-QCGC_STATIC bool is_small(size_t cells);
-QCGC_STATIC size_t small_index(size_t cells);
-QCGC_STATIC size_t large_index(size_t cells);
-QCGC_STATIC size_t small_index_to_cells(size_t index);
-
-QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells);
-QCGC_STATIC cell_t *fit_allocator_large_fit(size_t index, size_t cells);
-QCGC_STATIC cell_t *fit_allocator_large_first_fit(size_t index, size_t cells);
-
-QCGC_STATIC bool valid_block(cell_t *ptr, size_t cells);
-QCGC_STATIC void free_list_consistency_check(void);
-
-void qcgc_allocator_initialize(void) {
-	qcgc_allocator_state.arenas =
-		qcgc_arena_bag_create(QCGC_ARENA_BAG_INIT_SIZE);
-	qcgc_allocator_state.free_arenas = qcgc_arena_bag_create(4); // XXX
-
-	qcgc_allocator_state.use_bump_allocator = true;
-
-	// Bump Allocator
-	qcgc_allocator_state.bump_state.bump_ptr = NULL;
-	qcgc_allocator_state.bump_state.remaining_cells = 0;
-
-	// Fit Allocator
-	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
-		qcgc_allocator_state.fit_state.small_free_list[i] =
-			qcgc_linear_free_list_create(QCGC_SMALL_FREE_LIST_INIT_SIZE);
-	}
-
-	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
-		qcgc_allocator_state.fit_state.large_free_list[i] =
-			qcgc_exp_free_list_create(QCGC_LARGE_FREE_LIST_INIT_SIZE);
-	}
-}
-
-void qcgc_allocator_destroy(void) {
-	// Fit Allocator
-	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
-		free(qcgc_allocator_state.fit_state.small_free_list[i]);
-	}
-
-	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
-		free(qcgc_allocator_state.fit_state.large_free_list[i]);
-	}
-
-	// Arenas
-	size_t arena_count = qcgc_allocator_state.arenas->count;
-	for (size_t i = 0; i < arena_count; i++) {
-		qcgc_arena_destroy(qcgc_allocator_state.arenas->items[i]);
-	}
-
-	arena_count = qcgc_allocator_state.free_arenas->count;
-	for (size_t i = 0; i < arena_count; i++) {
-		qcgc_arena_destroy(qcgc_allocator_state.free_arenas->items[i]);
-	}
-
-	free(qcgc_allocator_state.arenas);
-	free(qcgc_allocator_state.free_arenas);
-}
-
-void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
-#if CHECKED
-	if (cells > 0) {
-		assert((((object_t *)ptr)->flags & QCGC_PREBUILT_OBJECT) == 0);
-		assert((cell_t *) qcgc_arena_addr(ptr) != ptr);
-		assert(qcgc_arena_get_blocktype(ptr) == BLOCK_FREE);
-		if (qcgc_arena_addr(ptr) == qcgc_arena_addr(ptr + cells)) {
-			assert(qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT);
-		}
-		for (size_t i = 1; i < cells; i++) {
-			assert(qcgc_arena_get_blocktype(ptr + i) == BLOCK_EXTENT);
-		}
-	}
-#endif
-	if (cells > 0) {
-		if (is_small(cells)) {
-			size_t index = small_index(cells);
-			qcgc_allocator_state.fit_state.small_free_list[index] =
-				qcgc_linear_free_list_add(
-						qcgc_allocator_state.fit_state.small_free_list[index],
-						ptr);
-		} else {
-			size_t index = large_index(cells);
-			qcgc_allocator_state.fit_state.large_free_list[index] =
-				qcgc_exp_free_list_add(
-						qcgc_allocator_state.fit_state.large_free_list[index],
-						(struct exp_free_list_item_s) {ptr, cells});
-		}
-	}
-}
-
-/*******************************************************************************
- * Bump Allocator                                                              *
- ******************************************************************************/
-
-object_t *qcgc_bump_allocate(size_t bytes) {
-#if CHECKED
-	assert(bytes <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP);
-#endif
-	size_t cells = bytes_to_cells(bytes);
-	if (cells > qcgc_allocator_state.bump_state.remaining_cells) {
-		bump_allocator_renew_block();
-	}
-	cell_t *mem = qcgc_allocator_state.bump_state.bump_ptr;
-	bump_allocator_advance(cells);
-
-	qcgc_arena_mark_allocated(mem, cells);
-	object_t *result = (object_t *) mem;
-
-#if QCGC_INIT_ZERO
-	memset(result, 0, cells * sizeof(cell_t));
-#endif
-
-	result->flags |= QCGC_GRAY_FLAG;
-#if CHECKED
-	assert(qcgc_arena_is_coalesced(qcgc_arena_addr((cell_t *)result)));
-	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
-		assert(qcgc_arena_get_blocktype(
-					qcgc_allocator_state.bump_state.bump_ptr) == BLOCK_FREE);
-		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
-				i++) {
-			assert(qcgc_arena_get_blocktype(
-						qcgc_allocator_state.bump_state.bump_ptr + i)
-					== BLOCK_EXTENT);
-		}
-	}
-#endif
-	return result;
-}
-
-QCGC_STATIC void bump_allocator_renew_block(void) {
-#if CHECKED
-	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
-		assert(qcgc_arena_get_blocktype(
-					qcgc_allocator_state.bump_state.bump_ptr) == BLOCK_FREE);
-		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
-				i++) {
-			assert(qcgc_arena_get_blocktype(
-						qcgc_allocator_state.bump_state.bump_ptr + i)
-					== BLOCK_EXTENT);
-		}
-	}
-#endif
-	// Add remaining memory to fit allocator
-	qcgc_fit_allocator_add(qcgc_allocator_state.bump_state.bump_ptr,
-			qcgc_allocator_state.bump_state.remaining_cells);
-
-	// Try finding some huge block from fit allocator
-	exp_free_list_t *free_list = qcgc_allocator_state.fit_state.
-		large_free_list[QCGC_LARGE_FREE_LISTS - 1];
-	while (free_list->count > 0 && !valid_block(free_list->items[0].ptr,
-				free_list->items[0].size)) {
-		free_list = qcgc_exp_free_list_remove_index(free_list, 0);
-	}
-
-	if (free_list->count > 0) {
-		// Assign huge block to bump allocator
-		bump_allocator_assign(free_list->items[0].ptr,
-				free_list->items[0].size);
-		free_list = qcgc_exp_free_list_remove_index(free_list, 0);
-	} else {
-		// Grab a new arena
-		arena_t *arena = qcgc_arena_create();
-		bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
-				QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
-		qcgc_allocator_state.arenas =
-			qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
-	}
-
-	qcgc_allocator_state.fit_state.
-		large_free_list[QCGC_LARGE_FREE_LISTS - 1] = free_list;
-#if CHECKED
-	assert(qcgc_allocator_state.bump_state.bump_ptr != NULL);
-	assert(qcgc_arena_get_blocktype(qcgc_allocator_state.bump_state.bump_ptr) ==
-			BLOCK_FREE);
-	for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
-			i++) {
-		assert(qcgc_arena_get_blocktype(
-					qcgc_allocator_state.bump_state.bump_ptr + i)
-				== BLOCK_EXTENT);
-	}
-#endif
-}
-
-QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells) {
-#if CHECKED
-	assert(qcgc_arena_get_blocktype(ptr) == BLOCK_FREE);
-	for (size_t i = 1; i < cells; i++) {
-		assert(qcgc_arena_get_blocktype(ptr + i) == BLOCK_EXTENT);
-	}
-#endif
-	qcgc_allocator_state.bump_state.bump_ptr = ptr;
-	qcgc_allocator_state.bump_state.remaining_cells = cells;
-}
-
-QCGC_STATIC void bump_allocator_advance(size_t cells) {
-	qcgc_allocator_state.bump_state.bump_ptr += cells;
-	qcgc_allocator_state.bump_state.remaining_cells -= cells;
-}
-
-object_t *qcgc_fit_allocate(size_t bytes) {
-#if CHECKED
-	//free_list_consistency_check();
-#endif
-	size_t cells = bytes_to_cells(bytes);
-	cell_t *mem;
-
-	if (is_small(cells)) {
-		size_t index = small_index(cells);
-		mem = fit_allocator_small_first_fit(index, cells);
-	} else {
-		size_t index = large_index(cells);
-		mem = fit_allocator_large_fit(index, cells);
-	}
-
-	if (mem == NULL) {
-		return NULL;
-	}
-
-	object_t *result = (object_t *) mem;
-
-#if QCGC_INIT_ZERO
-	memset(result, 0, cells * sizeof(cell_t));
-#endif
-
-	result->flags |= QCGC_GRAY_FLAG;
-	return result;
-}
-
-/**
- * Constraints:
- * - Zero initialized
- * - Aligned to arena size
- * - Multiple of arena size
- * - No header, metadata stored in hash-map
- */
-object_t *qcgc_large_allocate(size_t bytes) {
-	object_t *result = aligned_alloc(QCGC_ARENA_SIZE, bytes);
-#if QCGC_INIT_ZERO
-	memset(result, 0, bytes);
-#endif
-	qcgc_hbtable_insert(result);
-	return result;
-}
-
-void qcgc_fit_allocator_empty_lists(void) {
-	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
-		qcgc_allocator_state.fit_state.small_free_list[i]->count = 0;
-	}
-
-	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
-		qcgc_allocator_state.fit_state.large_free_list[i]->count = 0;
-	}
-}
-
-QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells) {
-#if CHECKED
-	assert(small_index_to_cells(index) >= cells);
-#endif
-	cell_t *result = NULL;
-	for ( ; index < QCGC_SMALL_FREE_LISTS; index++) {
-		size_t list_cell_size = small_index_to_cells(index);
-
-		if (qcgc_allocator_state.fit_state.small_free_list[index]->count > 0) {
-			result = qcgc_allocator_state.fit_state.small_free_list[index]->
-				items[0];
-			qcgc_allocator_state.fit_state.small_free_list[index] =
-				qcgc_linear_free_list_remove_index(
-						qcgc_allocator_state.fit_state.small_free_list[index],
-						0);
-			qcgc_arena_mark_allocated(result, cells);
-			qcgc_fit_allocator_add(result + cells, list_cell_size - cells);
-			return result;
-		}
-	}
-	return fit_allocator_large_first_fit(0, cells);
-}
-
-QCGC_STATIC cell_t *fit_allocator_large_fit(size_t index, size_t cells) {
-#if CHECKED
-	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP) <= cells);
-	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP + 1) > cells);
-#endif
-	size_t best_fit_index = qcgc_allocator_state.fit_state.
-		large_free_list[index]->count;
-
-	cell_t *result = NULL;
-	size_t best_fit_cells = SIZE_MAX;
-	size_t count = qcgc_allocator_state.fit_state.large_free_list[index]->count;
-	for (size_t i = 0; i < count; i++) {
-		if (qcgc_allocator_state.fit_state.large_free_list[index]->
-				items[i].size >= cells &&
-				qcgc_allocator_state.fit_state.large_free_list[index]->
-				items[i].size < best_fit_cells) {
-			result = qcgc_allocator_state.fit_state.large_free_list[index]->
-				items[i].ptr;
-			best_fit_cells = qcgc_allocator_state.fit_state.
-				large_free_list[index]->items[i].size;
-			best_fit_index = i;
-		}
-		if (best_fit_cells == cells) {
-			break;
-		}
-	}
-
-	if (result != NULL) {
-		// Best fit was found
-		assert(best_fit_index < qcgc_allocator_state.fit_state.
-				large_free_list[index]->count);
-		qcgc_allocator_state.fit_state.large_free_list[index] =
-			qcgc_exp_free_list_remove_index(qcgc_allocator_state.fit_state.
-					large_free_list[index], best_fit_index);
-		qcgc_arena_mark_allocated(result, cells);
-		qcgc_fit_allocator_add(result + cells, best_fit_cells - cells);
-	} else {
-		// No best fit, go for first fit
-		result = fit_allocator_large_first_fit(index + 1, cells);
-	}
-	return result;
-}
-
-QCGC_STATIC cell_t *fit_allocator_large_first_fit(size_t index, size_t cells) {
-#if CHECKED
-	assert(1u<<(index + QCGC_LARGE_FREE_LIST_FIRST_EXP) >= cells);
-#endif
-	for ( ; index < QCGC_LARGE_FREE_LISTS; index++) {
-		if (qcgc_allocator_state.fit_state.large_free_list[index]->count > 0) {
-			struct exp_free_list_item_s item =
-				qcgc_allocator_state.fit_state.large_free_list[index]->items[0];
-			qcgc_allocator_state.fit_state.large_free_list[index] =
-				qcgc_exp_free_list_remove_index(
-						qcgc_allocator_state.fit_state.large_free_list[index],
-						0);
-
-			qcgc_arena_mark_allocated(item.ptr, cells);
-			qcgc_fit_allocator_add(item.ptr + cells, item.size - cells);
-			return item.ptr;
-		}
-	}
-	return NULL;
-}
-
-QCGC_STATIC size_t bytes_to_cells(size_t bytes) {
-	return (bytes + sizeof(cell_t) - 1) / sizeof(cell_t);
-}
-
-QCGC_STATIC bool is_small(size_t cells) {
-	return cells <= QCGC_SMALL_FREE_LISTS;
-}
-
-QCGC_STATIC size_t small_index(size_t cells) {
-#if CHECKED
-	assert(is_small(cells));
-#endif
-	return cells - 1;
-}
-
-QCGC_STATIC size_t large_index(size_t cells) {
-#if CHECKED
-	assert(!is_small(cells));
-#endif
-	// shift such that the meaningless part disappears, i.e. everything that
-	// belongs into the first free list will become 1.
-	cells = cells >> QCGC_LARGE_FREE_LIST_FIRST_EXP;
-
-	// calculates floor(log(cells))
-	return MIN((8 * sizeof(unsigned long)) - __builtin_clzl(cells) - 1, QCGC_LARGE_FREE_LISTS - 1);
-}
-
-QCGC_STATIC size_t small_index_to_cells(size_t index) {
-#if CHECKED
-	assert(index < QCGC_SMALL_FREE_LISTS);
-#endif
-	return index + 1;
-}
-
-QCGC_STATIC bool valid_block(cell_t *ptr, size_t cells) {
-#if CHECKED
-	assert(ptr != NULL);
-	assert(cells > 0);
-#endif
-	return (qcgc_arena_get_blocktype(ptr) == BLOCK_FREE && (
-				((qcgc_arena_addr(ptr + cells)) == (arena_t *) (ptr + cells)) ||
-				qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT));
-}
-
-QCGC_STATIC void free_list_consistency_check(void) {
-	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
-		linear_free_list_t *free_list =
-			qcgc_allocator_state.fit_state.small_free_list[i];
-		for (size_t j = 0; j < free_list->count; j++) {
-			cell_t *item = free_list->items[j];
-			if  (qcgc_arena_get_blocktype(item) == BLOCK_FREE) {
-				for (size_t s = 1; s < small_index_to_cells(i); s++) {
-					assert(qcgc_arena_get_blocktype(item + s) == BLOCK_EXTENT);
-				}
-			}
-		}
-	}
-
-	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
-		exp_free_list_t *free_list =
-			qcgc_allocator_state.fit_state.large_free_list[i];
-		for (size_t j = 0; j < free_list->count; j++) {
-			struct exp_free_list_item_s item = free_list->items[j];
-			if  (qcgc_arena_get_blocktype(item.ptr) == BLOCK_FREE) {
-				for (size_t s = 1; s < item.size; s++) {
-					assert(qcgc_arena_get_blocktype(item.ptr + s) == BLOCK_EXTENT);
-				}
-			}
-		}
-	}
-}
diff --git a/rpython/translator/c/src/qcgc/allocator.h b/rpython/translator/c/src/qcgc/allocator.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/allocator.h
+++ /dev/null
@@ -1,104 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#include <stddef.h>
-
-#include "arena.h"
-#include "bag.h"
-#include "object.h"
-
-/**
- * Free lists:
- *
- * Small free lists:
- *                +---+---+-----+----+
- * index:         | 0 | 1 | ... | 30 |
- *                +---+---+-----+----+
- * size (cells):  | 1 | 2 | ... | 31 |
- *                +---+---+-----+----+
- * (31 is 2^QCGC_LARGE_FREE_LIST_FIRST_EXP - 1)
- *
- * Large free lists:
- *                        +-----+-----+-----+---------+
- * index:                 |  0  |  1  | ... |    x    |
- *                        +-----+-----+-----+---------+
- * minimal size (cells):  | 2^5 | 2^6 | ... | 2^(x+5) |
- *                        +-----+-----+-----+---------+
- * (5 is QCGC_LARGE_FREE_LIST_FIRST_EXP)
- *
- * where x is chosen such that 2^(x + 5) = 2^QCGC_LARGE_ALLOC_THRESHOLD_EXP
- * (i.e. such that the last bin contains all blocks that are larger or equal
- * than the threshold for huge blocks. These blocks can be returned to the
- * bump allocator)
- */
-#define QCGC_LARGE_FREE_LISTS (QCGC_LARGE_ALLOC_THRESHOLD_EXP - QCGC_LARGE_FREE_LIST_FIRST_EXP - 4 + 1)
-// -4 because of turning bytes into cells, +1 because we start to count at 0
-
-#define QCGC_SMALL_FREE_LISTS ((1<<QCGC_LARGE_FREE_LIST_FIRST_EXP) - 1)
-
-struct qcgc_allocator_state {
-	arena_bag_t *arenas;
-	arena_bag_t *free_arenas;
-	struct bump_state {
-		cell_t *bump_ptr;
-		size_t remaining_cells;
-	} bump_state;
-	struct fit_state {
-		linear_free_list_t *small_free_list[QCGC_SMALL_FREE_LISTS];
-		exp_free_list_t *large_free_list[QCGC_LARGE_FREE_LISTS];
-	} fit_state;
-	bool use_bump_allocator;
-} qcgc_allocator_state;
-
-/**
- * Initialize allocator
- */
-void qcgc_allocator_initialize(void);
-
-/**
- * Destroy allocator
- */
-void qcgc_allocator_destroy(void);
-
-/**
- * Allocate new memory region using fit allocator
- *
- * @param	bytes	Desired size of the memory region in bytes
- * @return	Pointer to memory large enough to hold size bytes, NULL in case of
- *			errors or if there is no block sufficently large block, already zero
- *			initialized if QCGC_INIT_ZERO is set
- */
-object_t *qcgc_fit_allocate(size_t bytes);
-
-/**
- * Allocate new memory region using bump allocator
- *
- * @param	bytes	Desired size of the memory region in bytes
- * @return	Pointer to memory large enough to hold size bytes, NULL in case of
- *			errors, already zero initialized if QCGC_INIT_ZERO is set
- */
-object_t *qcgc_bump_allocate(size_t bytes);
-
-/**
- * Allocate new memory region using huge block allocator
- *
- * @param	bytes	Desired size of the memory region in bytes
- * @return	Pointer to memory large enough to hold size bytes, NULL in case of
- *			errors, already zero initialized if QCGC_INIT_ZERO is set
- */
-object_t *qcgc_large_allocate(size_t bytes);
-
-/**
- * Empty all free lists (used before sweep)
- */
-void qcgc_fit_allocator_empty_lists(void);
-
-
-/**
- * Add memory to free lists
- *
- * @param	ptr		Pointer to memory region
- * @param	cells	Size of memory region in cells
- */
-void qcgc_fit_allocator_add(cell_t *ptr, size_t cells);
diff --git a/rpython/translator/c/src/qcgc/arena.c b/rpython/translator/c/src/qcgc/arena.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/arena.c
+++ /dev/null
@@ -1,398 +0,0 @@
-#include "arena.h"
-
-#include <assert.h>
-#include <stdlib.h>
-#include <sys/mman.h>
-#include <unistd.h>
-
-#if DEBUG_ZERO_ON_SWEEP
-#include <string.h>
-#endif
-
-#include "allocator.h"
-#include "event_logger.h"
-#include "gc_state.h"
-
-/**
- * Internal functions
- */
-QCGC_STATIC blocktype_t get_blocktype(arena_t *arena, size_t index);
-QCGC_STATIC void set_blocktype(arena_t *arena, size_t index, blocktype_t type);
-
-arena_t *qcgc_arena_create(void) {
-	qcgc_event_logger_log(EVENT_NEW_ARENA, 0, NULL);
-
-	arena_t *result;
-	// Linux: MAP_ANONYMOUS is initialized to zero
-	cell_t *mem = (cell_t *) mmap(0, 2 * QCGC_ARENA_SIZE,
-			PROT_READ | PROT_WRITE,
-			MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
-	if (mem == MAP_FAILED) {
-		// ERROR: OUT OF MEMORY
-		return NULL;
-	}
-	if (mem != qcgc_arena_addr(mem)->cells) {
-		// Not aligned -> align
-		cell_t *aligned_mem = (cell_t *)(
-				(intptr_t) qcgc_arena_addr(mem) + QCGC_ARENA_SIZE);
-		size_t size_before = (size_t)((intptr_t) aligned_mem - (intptr_t) mem);
-		size_t size_after = QCGC_ARENA_SIZE - size_before;
-
-		munmap((void *) mem, size_before);
-		munmap((void *)((intptr_t) aligned_mem + QCGC_ARENA_SIZE), size_after);
-		result = (arena_t *) aligned_mem;
-	} else {
-		// free second half
-		munmap((void *)((intptr_t) mem + QCGC_ARENA_SIZE), QCGC_ARENA_SIZE);
-		result = (arena_t *) mem;
-	}
-
-	// Init bitmaps: One large free block
-	qcgc_arena_set_bitmap_entry(result->mark_bitmap, QCGC_ARENA_FIRST_CELL_INDEX, true);
-
-	// Create gray stack
-	result->gray_stack = qcgc_gray_stack_create(QCGC_GRAY_STACK_INIT_SIZE);
-	return result;
-}
-
-void qcgc_arena_destroy(arena_t *arena) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	free(arena->gray_stack);
-	munmap((void *) arena, QCGC_ARENA_SIZE);
-}
-
-arena_t *qcgc_arena_addr(cell_t *ptr) {
-	return (arena_t *)((intptr_t) ptr & ~(QCGC_ARENA_SIZE - 1));
-}
-
-size_t qcgc_arena_cell_index(cell_t *ptr) {
-	return (size_t)((intptr_t) ptr & (QCGC_ARENA_SIZE - 1)) >> 4;
-}
-
-bool qcgc_arena_get_bitmap_entry(uint8_t *bitmap, size_t index) {
-#if CHECKED
-	assert(bitmap != NULL);
-#endif
-	return (((bitmap[index / 8] >> (index % 8)) & 0x1) == 0x01);
-}
-
-void qcgc_arena_set_bitmap_entry(uint8_t *bitmap, size_t index, bool value) {
-#if CHECKED
-	assert(bitmap != NULL);
-#endif
-	if (value) {
-		bitmap[index / 8] |= 1<<(index % 8);
-	} else {
-		bitmap[index / 8] &= ~(1<<(index % 8));
-	}
-}
-
-QCGC_STATIC blocktype_t get_blocktype(arena_t *arena, size_t index) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	uint8_t block_bit = qcgc_arena_get_bitmap_entry(arena->block_bitmap, index);
-	uint8_t mark_bit = qcgc_arena_get_bitmap_entry(arena->mark_bitmap, index);
-
-	if (block_bit) {
-		if (mark_bit) {
-			return BLOCK_BLACK;
-		} else {
-			return BLOCK_WHITE;
-		}
-	} else {
-		if (mark_bit) {
-			return BLOCK_FREE;
-		} else {
-			return BLOCK_EXTENT;
-		}
-	}
-}
-
-blocktype_t qcgc_arena_get_blocktype(cell_t *ptr) {
-	size_t index = qcgc_arena_cell_index(ptr);
-	arena_t *arena = qcgc_arena_addr(ptr);
-
-	return get_blocktype(arena, index);
-}
-
-QCGC_STATIC void set_blocktype(arena_t *arena, size_t index, blocktype_t type) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	switch(type) {
-		case BLOCK_EXTENT:
-			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, false);
-			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, false);
-			break;
-		case BLOCK_FREE:
-			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, false);
-			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, true);
-			break;
-		case BLOCK_WHITE:
-			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, true);
-			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, false);
-			break;
-		case BLOCK_BLACK:
-			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, true);
-			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, true);
-			break;
-	}
-}
-
-void qcgc_arena_set_blocktype(cell_t *ptr, blocktype_t type) {
-	size_t index = qcgc_arena_cell_index(ptr);
-	arena_t *arena = qcgc_arena_addr(ptr);
-	set_blocktype(arena, index, type);
-}
-
-void qcgc_arena_mark_allocated(cell_t *ptr, size_t cells) {
-	size_t index = qcgc_arena_cell_index(ptr);
-	arena_t *arena = qcgc_arena_addr(ptr);
-#if CHECKED
-	assert(get_blocktype(arena, index) == BLOCK_FREE);
-	for (size_t i = 1; i < cells; i++) {
-		assert(get_blocktype(arena, index + i) == BLOCK_EXTENT);
-	}
-#endif
-	set_blocktype(arena, index, BLOCK_WHITE);
-	size_t index_of_next_block = index + cells;
-	if (index_of_next_block < QCGC_ARENA_CELLS_COUNT &&
-			get_blocktype(arena, index_of_next_block) == BLOCK_EXTENT) {
-		set_blocktype(arena, index_of_next_block, BLOCK_FREE);
-	}
-#if CHECKED
-	assert(get_blocktype(arena, index) == BLOCK_WHITE);
-	for (size_t i = 1; i < cells; i++) {
-		assert(get_blocktype(arena, index + i) == BLOCK_EXTENT);
-	}
-	if (index_of_next_block < QCGC_ARENA_CELLS_COUNT) {
-		assert(get_blocktype(arena, index + cells) != BLOCK_EXTENT);
-	}
-#endif
-}
-
-void qcgc_arena_mark_free(cell_t *ptr) {
-	qcgc_arena_set_blocktype(ptr, BLOCK_FREE);
-	// No coalescing, collector will do this
-}
-
-bool qcgc_arena_pseudo_sweep(arena_t *arena) {
-#if CHECKED
-	assert(arena != NULL);
-	assert(qcgc_arena_is_coalesced(arena));
-	assert(qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) == arena);
-#endif
-	// Ignore free cell / largest block counting here, as blocks are not
-	// registerd in free lists as well
-	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
-			cell < QCGC_ARENA_CELLS_COUNT;
-			cell++) {
-		switch (get_blocktype(arena, cell)) {
-			case BLOCK_BLACK:
-				set_blocktype(arena, cell, BLOCK_WHITE);
-			case BLOCK_FREE: // Fall through
-			case BLOCK_EXTENT: // Fall through
-			case BLOCK_WHITE: // Fall through
-				break;
-		}
-	}
-#if CHECKED
-	assert(qcgc_arena_is_coalesced(arena));
-#endif
-	return false;
-}
-
-bool qcgc_arena_sweep(arena_t *arena) {
-#if CHECKED
-	assert(arena != NULL);
-	assert(qcgc_arena_is_coalesced(arena));
-	//assert(qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) != arena);
-#endif
-	if (qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) == arena) {
-		return qcgc_arena_pseudo_sweep(arena);
-	}
-
-	size_t last_free_cell = 0;
-	bool free = true;
-	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
-			cell < QCGC_ARENA_CELLS_COUNT;
-			cell++) {
-		switch (get_blocktype(arena, cell)) {
-			case BLOCK_EXTENT:
-				break;
-			case BLOCK_FREE:
-				if (last_free_cell != 0) {
-					// Coalesce
-					set_blocktype(arena, cell, BLOCK_EXTENT);
-				} else {
-					last_free_cell = cell;
-				}
-				// ==> last_free_cell != 0
-				break;
-			case BLOCK_WHITE:
-				if (last_free_cell != 0) {
-					// Coalesce
-					set_blocktype(arena, cell, BLOCK_EXTENT);
-				} else {
-					set_blocktype(arena, cell, BLOCK_FREE);
-					last_free_cell = cell;
-				}
-				// ==> last_free_cell != 0
-				break;
-			case BLOCK_BLACK:
-				set_blocktype(arena, cell, BLOCK_WHITE);
-				if (last_free_cell != 0) {
-					qcgc_fit_allocator_add(arena->cells + last_free_cell,
-							cell - last_free_cell);
-#if DEBUG_ZERO_ON_SWEEP
-					memset(arena->cells + last_free_cell, 0,
-							sizeof(cell_t) * (cell - last_free_cell));
-#endif
-					qcgc_state.free_cells += cell - last_free_cell;
-					qcgc_state.largest_free_block = MAX(
-							qcgc_state.largest_free_block,
-							cell - last_free_cell);
-					last_free_cell = 0;
-				}
-				free = false;
-				// ==> last_free_cell == 0
-				break;
-		}
-	}
-	if (last_free_cell != 0 && !free) {
-		qcgc_fit_allocator_add(arena->cells + last_free_cell,
-				QCGC_ARENA_CELLS_COUNT - last_free_cell);
-#if DEBUG_ZERO_ON_SWEEP
-		memset(arena->cells + last_free_cell, 0,
-				sizeof(cell_t) * (QCGC_ARENA_CELLS_COUNT - last_free_cell));
-#endif
-		qcgc_state.free_cells += QCGC_ARENA_CELLS_COUNT - last_free_cell;
-		qcgc_state.largest_free_block = MAX(
-				qcgc_state.largest_free_block,
-				QCGC_ARENA_CELLS_COUNT - last_free_cell);
-		last_free_cell = 0;
-	}
-#if CHECKED
-	assert(qcgc_arena_is_coalesced(arena));
-	assert(free == qcgc_arena_is_empty(arena));
-#endif
-	return free;
-}
-
-bool qcgc_arena_is_empty(arena_t *arena) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
-			cell < QCGC_ARENA_CELLS_COUNT;
-			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
-			case BLOCK_WHITE: // Fall through
-			case BLOCK_BLACK:
-				return false;
-
-			default:
-				break;
-		}
-	}
-	return true;
-}
-
-
-bool qcgc_arena_is_coalesced(arena_t *arena) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	bool prev_was_free = false;
-	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
-			cell < QCGC_ARENA_CELLS_COUNT;
-			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
-			case BLOCK_WHITE: // Fall through
-			case BLOCK_BLACK:
-				prev_was_free = false;
-				break;
-
-			case BLOCK_FREE:
-				if (prev_was_free) {
-					return false;
-				} else {
-					prev_was_free = true;
-				}
-				break;
-
-			case BLOCK_EXTENT:
-				break;
-		}
-	}
-	return true;
-}
-
-size_t qcgc_arena_free_blocks(arena_t *arena) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	size_t result = 0;
-	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
-			cell < QCGC_ARENA_CELLS_COUNT;
-			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
-			case BLOCK_WHITE: // Fall through
-			case BLOCK_BLACK:
-			case BLOCK_EXTENT:
-				break;
-
-			case BLOCK_FREE:
-				result++;
-				break;
-		}
-	}
-	return result;
-}
-
-size_t qcgc_arena_white_blocks(arena_t *arena) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	size_t result = 0;
-	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
-			cell < QCGC_ARENA_CELLS_COUNT;
-			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
-			case BLOCK_BLACK:	// Fall through
-			case BLOCK_EXTENT:
-			case BLOCK_FREE:
-				break;
-
-			case BLOCK_WHITE:
-				result++;
-				break;
-		}
-	}
-	return result;
-}
-
-size_t qcgc_arena_black_blocks(arena_t *arena) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	size_t result = 0;
-	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
-			cell < QCGC_ARENA_CELLS_COUNT;
-			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
-			case BLOCK_WHITE: // Fall through
-			case BLOCK_FREE:
-			case BLOCK_EXTENT:
-				break;
-
-			case BLOCK_BLACK:
-				result++;
-				break;
-		}
-	}
-	return result;
-}
diff --git a/rpython/translator/c/src/qcgc/arena.h b/rpython/translator/c/src/qcgc/arena.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/arena.h
+++ /dev/null
@@ -1,197 +0,0 @@
-/**
- * @file	arena.h
- */
-
-#pragma once
-
-#include "config.h"
-
-#include <stdbool.h>
-#include <stdint.h>
-#include <sys/types.h>
-
-#include "gray_stack.h"
-
-#define QCGC_ARENA_SIZE (1<<QCGC_ARENA_SIZE_EXP)
-
-#define QCGC_ARENA_BITMAP_SIZE (1<<(QCGC_ARENA_SIZE_EXP - 7)) // 1 / 128
-#define QCGC_ARENA_CELLS_COUNT (1<<(QCGC_ARENA_SIZE_EXP - 4))
-
-#define QCGC_ARENA_FIRST_CELL_INDEX (1<<(QCGC_ARENA_SIZE_EXP - 10))
-
-/**
- * @typedef cell_t
- * The smallest unit of memory that can be addressed and allocated in arenas.
- */
-typedef uint8_t cell_t[16];
-
-/**
- * @typedef arena_t
- * Arena object
- */
-typedef union {
-	struct {
-		union {
-			gray_stack_t *gray_stack;
-			uint8_t block_bitmap[QCGC_ARENA_BITMAP_SIZE];
-		};
-		uint8_t mark_bitmap[QCGC_ARENA_BITMAP_SIZE];
-	};
-	cell_t cells[QCGC_ARENA_CELLS_COUNT];
-} arena_t;
-
-/**
- * @typedef blocktype_t
- * Blocktypes:
- * - BLOCK_EXTENT	Extension of previous block
- * - BLOCK_FREE		Free block
- * - BLOCK_WHITE	Allocated block, marked white
- * - BLOCK_BLACK	Allocated block, marked black
- */
-typedef enum blocktype {
-	BLOCK_EXTENT,
-	BLOCK_FREE,
-	BLOCK_WHITE,
-	BLOCK_BLACK,
-} blocktype_t;
-
-/**
- * Create a new arena.
- *
- * @return Pointer to new arena, NULL in case of errors
- */
-arena_t *qcgc_arena_create(void);
-
-/**
- * Destroys an arena (return to OS).
- *
- * @param	arena	The arena to destroy
- */
-void qcgc_arena_destroy(arena_t *arena);
-
-/**
- * Arena pointer for given cell.
- *
- * @param	ptr		Pointer to cell for which you want to know the corresponding
- *					arena
- * @return	The arena the pointer belongs to
- */
-arena_t *qcgc_arena_addr(cell_t *ptr);
-
-/**
- * Index of cell in arena.
- *
- * @param	ptr		Pointer to cell for which you want to know the cell index
- * @return	Index of the cell to which ptr points to
- */
-size_t qcgc_arena_cell_index(cell_t *ptr);
-
-/**
- * Get bitmap value for given bitmap and cell index.
- *
- * @param	bitmap	Bitmap
- * @param	index	Index of cell
- * @return	true if bitmap entry is set, false otherwise
- */
-bool qcgc_arena_get_bitmap_entry(uint8_t *bitmap, size_t index);
-
-/**
- * Set bitmap value for given bitmap and cell index.
- *
- * @param	bitmap	Bitmap
- * @param	index	Index of cell
- * @param	value	true to set entry, false to reset entry
- */
-void qcgc_arena_set_bitmap_entry(uint8_t *bitmap, size_t index, bool value);
-
-/**
- * Get blocktype.
- *
- * @param	ptr		Pointer to cell for which you want to know the blocktype
- * @return	Blocktype
- */
-blocktype_t qcgc_arena_get_blocktype(cell_t *ptr);
-
-/**
- * Set blocktype.
- *
- * @param	ptr		Pointer to cell for which you want to set the blocktype
- * @param	type	Blocktype that should be set
- */
-void qcgc_arena_set_blocktype(cell_t *ptr, blocktype_t type);
-
-/**
- * Mark ptr as allocated area with given size.
- *
- * @param	ptr		Pointer to first cell of area
- * @param	cells	Size in cells
- */
-void qcgc_arena_mark_allocated(cell_t *ptr, size_t cells);
-
-/**
- * Mark cell ptr point to as free (no coalescing).
- *
- * @param	ptr		Pointer to cell that should be marked as free
- */
-void qcgc_arena_mark_free(cell_t *ptr);
-
-/**
- * Sweep given arena.
- *
- * @param	arena	Arena
- * @return	Whether arena is empty after sweeping
- */
-bool qcgc_arena_sweep(arena_t *arena);
-
-/**
- * Sweep given arena, but only reset black to white, no white to free
- *
- * @param	arena	Arena
- * @return	Whether arena is empty after sweeping, always false
- */
-bool qcgc_arena_pseudo_sweep(arena_t *arena);
-
-
-/*******************************************************************************
- * Debug functions                                                             *
- ******************************************************************************/
-
-/**
- * Check whether arena is empty.
- *
- * @param	arena	Arena
- * @return	true iff given arena is empty
- */
-bool qcgc_arena_is_empty(arena_t *arena);
-
-/**
- * Check whether arena is coalesced (no consecutive free blocks).
- *
- * @param	arena	Arena
- * @return	true iff given arena is coalesced
- */
-bool qcgc_arena_is_coalesced(arena_t *arena);
-
-/**
- * Count free blocks.
- *
- * @param	arena	Arena
- * @return	Number of free blocks
- */
-size_t qcgc_arena_free_blocks(arena_t *arena);
-
-/**
- * Count white blocks.
- *
- * @param	arena	Arena
- * @return	Number of white blocks
- */
-size_t qcgc_arena_white_blocks(arena_t *arena);
-
-/**
- * Count black blocks.
- *
- * @param	arena	Arena
- * @return	Number of black blocks
- */
-size_t qcgc_arena_black_blocks(arena_t *arena);
diff --git a/rpython/translator/c/src/qcgc/bag.c b/rpython/translator/c/src/qcgc/bag.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/bag.c
+++ /dev/null
@@ -1,9 +0,0 @@
-#include "bag.h"
-
-#include <assert.h>
-
-DEFINE_BAG(arena_bag, arena_t *);
-DEFINE_BAG(linear_free_list, cell_t *);
-DEFINE_BAG(exp_free_list, struct exp_free_list_item_s);
-DEFINE_BAG(hbbucket, struct hbtable_entry_s);
-DEFINE_BAG(weakref_bag, struct weakref_bag_item_s);
diff --git a/rpython/translator/c/src/qcgc/bag.h b/rpython/translator/c/src/qcgc/bag.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/bag.h
+++ /dev/null
@@ -1,104 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#include <stddef.h>
-#include <stdlib.h>
-
-#include "arena.h"
-
-#define DECLARE_BAG(name, type)												\
-typedef struct name##_s {													\
-	size_t size;															\
-	size_t count;															\
-	type items[];															\
-} name##_t;																	\
-																			\
-__attribute__ ((warn_unused_result))										\
-name##_t *qcgc_##name##_create(size_t size);								\
-																			\
-__attribute__ ((warn_unused_result))										\
-name##_t *qcgc_##name##_add(name##_t *self, type item);						\
-																			\
-__attribute__ ((warn_unused_result))										\
-name##_t *qcgc_##name##_remove_index(name##_t *self, size_t index);
-
-#define DEFINE_BAG(name, type)												\
-																			\
-QCGC_STATIC size_t name##_size(size_t size);								\
-																			\
-__attribute__ ((warn_unused_result))										\
-QCGC_STATIC name##_t *name##_grow(name##_t *self);							\
-																			\
-__attribute__ ((warn_unused_result))										\
-QCGC_STATIC name##_t *name##_shrink(name##_t *self);						\
-																			\
-name##_t *qcgc_##name##_create(size_t size) {								\
-	name##_t *result = (name##_t *) malloc(name##_size(size));				\
-	result->size = size;													\
-	result->count = 0;														\
-	return result;															\
-}																			\
-																			\
-name##_t *qcgc_##name##_add(name##_t *self, type item) {					\
-	if (self->count >= self->size) {										\
-		self = name##_grow(self);											\
-	}																		\
-	self->items[self->count++] = item;										\
-	return self;															\
-}																			\
-																			\
-name##_t *qcgc_##name##_remove_index(name##_t *self, size_t index) {		\
-	if (index + 1 < self->count) {											\
-		self->items[index] = self->items[self->count - 1];					\
-	}																		\
-	self->count--;															\
-																			\
-	if (self->count < self->size / 4) {										\
-		self = name##_shrink(self);											\
-	}																		\
-	return self;															\
-}																			\
-																			\
-QCGC_STATIC name##_t *name##_grow(name##_t *self) {								\
-	name##_t *new_self = (name##_t *) realloc(self,							\
-			name##_size(self->size * 2));									\
-	assert(new_self != NULL);												\
-	self = new_self;														\
-	self->size *= 2;														\
-	return self;															\
-}																			\
-																			\
-QCGC_STATIC name##_t *name##_shrink(name##_t *self) {							\
-	name##_t *new_self = (name##_t *) realloc(self,							\
-			name##_size(self->size / 2));									\
-	assert(new_self != NULL);												\
-	self = new_self;														\
-	self->size /= 2;														\
-	return self;															\
-}																			\
-																			\
-QCGC_STATIC size_t name##_size(size_t size) {									\
-	return sizeof(name##_t) + size * sizeof(type);							\
-}
-
-struct exp_free_list_item_s {
-	cell_t *ptr;
-	size_t size;
-};
-
-struct hbtable_entry_s {
-	object_t *object;
-	bool mark_flag;
-};
-
-struct weakref_bag_item_s {
-	object_t *weakrefobj;
-	object_t **target;
-};
-
-DECLARE_BAG(arena_bag, arena_t *);
-DECLARE_BAG(linear_free_list, cell_t *);
-DECLARE_BAG(exp_free_list, struct exp_free_list_item_s);
-DECLARE_BAG(hbbucket, struct hbtable_entry_s);
-DECLARE_BAG(weakref_bag, struct weakref_bag_item_s);
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/config.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#pragma once
-
-#define CHECKED 0							// Enable runtime sanity checks
-#define DEBUG_ZERO_ON_SWEEP 0				// Zero memory on sweep (debug only)
-
-#define QCGC_INIT_ZERO 1					// Init new objects with zero bytes
-
-/**
- * Event logger
- */
-#define EVENT_LOG 1							// Enable event log
-#define LOGFILE "./qcgc_events.log"			// Default logfile
-#define LOG_ALLOCATION 0					// Enable allocation log (warning:
-											// significant performance impact)
-
-#define QCGC_SHADOWSTACK_SIZE 163840		// Total shadowstack size
-#define QCGC_ARENA_BAG_INIT_SIZE 16			// Initial size of the arena bag
-#define QCGC_ARENA_SIZE_EXP 20				// Between 16 (64kB) and 20 (1MB)
-#define QCGC_LARGE_ALLOC_THRESHOLD_EXP 14	// Less than QCGC_ARENA_SIZE_EXP
-#define QCGC_MARK_LIST_SEGMENT_SIZE 64		// TODO: Tune for performance
-#define QCGC_GRAY_STACK_INIT_SIZE 128		// TODO: Tune for performance
-#define QCGC_INC_MARK_MIN 64				// TODO: Tune for performance
-
-/**
- * Fit allocator
- */
-#define QCGC_LARGE_FREE_LIST_FIRST_EXP 5	// First exponent of large free list
-#define QCGC_LARGE_FREE_LIST_INIT_SIZE 4	// Initial size for large free lists
-#define QCGC_SMALL_FREE_LIST_INIT_SIZE 16	// Initial size for small free lists
-
-/**
- * Auto Mark/Collect
- */
-#define QCGC_MAJOR_COLLECTION_THRESHOLD (5 * (1<<QCGC_ARENA_SIZE_EXP))
-#define QCGC_INCMARK_THRESHOLD (1<<QCGC_ARENA_SIZE_EXP)
-
-/**
- * DO NOT MODIFY BELOW HERE
- */
-
-#if QCGC_LARGE_ALLOC_THRESHOLD_EXP >= QCGC_ARENA_SIZE_EXP
-#error	"Inconsistent configuration. Huge block threshold must be smaller " \
-		"than the arena size."
-#endif
-
-#ifdef TESTING
-#define QCGC_STATIC
-#else
-#define QCGC_STATIC static
-#endif
-
-#define MAX(a,b) (((a)>(b))?(a):(b))
-#define MIN(a,b) (((a)<(b))?(a):(b))
-#define UNUSED(x) (void)(x)
diff --git a/rpython/translator/c/src/qcgc/event_logger.c b/rpython/translator/c/src/qcgc/event_logger.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/event_logger.c
+++ /dev/null
@@ -1,81 +0,0 @@
-#include "event_logger.h"
-
-#include <assert.h>
-#include <time.h>
-#include <stdio.h>
-
-#if EVENT_LOG
-static struct {
-	FILE *logfile;
-} event_logger_state;
-
-#endif
-
-void qcgc_event_logger_initialize(void) {
-#if EVENT_LOG
-	event_logger_state.logfile = fopen(LOGFILE, "w");
-	qcgc_event_logger_log(EVENT_LOG_START, 0, NULL);
-
-	if (event_logger_state.logfile == NULL)  {
-		fprintf(stderr, "%s\n", "Failed to create logfile.");
-	}
-#endif
-}
-
-void qcgc_event_logger_destroy(void) {
-#if EVENT_LOG
-	qcgc_event_logger_log(EVENT_LOG_STOP, 0, NULL);
-
-	if (event_logger_state.logfile != NULL) {
-		fflush(event_logger_state.logfile);
-		fclose(event_logger_state.logfile);
-		event_logger_state.logfile = NULL;
-	}
-#endif
-}
-
-void qcgc_event_logger_log(enum event_e event, uint32_t additional_data_size,
-		uint8_t *additional_data) {
-#if EVENT_LOG
-#if CHECKED
-	assert((additional_data_size == 0) == (additional_data == NULL));
-#endif // CHECKED
-	struct {
-		uint32_t sec;
-		uint32_t nsec;
-		uint8_t event_id;
-		uint32_t additional_data_size;
-	} __attribute__ ((packed)) log_entry;
-
-	if (event_logger_state.logfile != NULL) {
-		struct timespec t;
-		clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &t);
-
-		log_entry.sec = (uint32_t) t.tv_sec;
-		log_entry.nsec = (uint32_t) t.tv_nsec;
-		log_entry.event_id = (uint8_t) event;
-		log_entry.additional_data_size = additional_data_size;
-
-		// The size and nmemb fields are flipped intentionally
-		int result = 0;
-		result = fwrite(&log_entry, sizeof(log_entry), 1,
-				event_logger_state.logfile);
-		if (result != 1) {
-			fprintf(stderr, "%s\n", "Failed to write log entry.");
-			event_logger_state.logfile = NULL;
-			return;
-		}
-		if (additional_data_size > 0) {
-			result = fwrite(additional_data, additional_data_size, 1,
-					event_logger_state.logfile);
-
-			if (result != 1) {
-				fprintf(stderr, "%s\n", "Failed to write additional data.");
-				event_logger_state.logfile = NULL;
-				return;
-			}
-		}
-	}
-
-#endif // EVENT_LOG
-}
diff --git a/rpython/translator/c/src/qcgc/event_logger.h b/rpython/translator/c/src/qcgc/event_logger.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/event_logger.h
+++ /dev/null
@@ -1,41 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#include <stddef.h>
-#include <stdint.h>
-
-/**
- * All events
- */
-enum event_e {
-	EVENT_LOG_START,			// = 0
-	EVENT_LOG_STOP,
-
-	EVENT_SWEEP_START,
-	EVENT_SWEEP_DONE,
-
-	EVENT_ALLOCATE_START,
-	EVENT_ALLOCATE_DONE,		// = 5
-
-	EVENT_NEW_ARENA,
-
-	EVENT_MARK_START,
-	EVENT_MARK_DONE,
-};
-
-/**
- * Initialize logger
- */
-void qcgc_event_logger_initialize(void);
-
-/**
- * Destroy logger
- */
-void qcgc_event_logger_destroy(void);
-
-/**
- * Log event
- */
-void qcgc_event_logger_log(enum event_e event, uint32_t additional_data_size,
-		uint8_t *additional_data);
diff --git a/rpython/translator/c/src/qcgc/gc_state.h b/rpython/translator/c/src/qcgc/gc_state.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/gc_state.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#pragma once
-
-#include <stddef.h>
-
-#include "bag.h"
-#include "gray_stack.h"
-#include "shadow_stack.h"
-
-/**
- * @typedef gc_state_t
- * Garbage collection states.
- * - GC_PAUSE	No gc in progress
- * - GC_MARK	Currently marking
- * - GC_COLLECT	Currently collecting
- */
-typedef enum gc_phase {
-	GC_PAUSE,
-	GC_MARK,
-	GC_COLLECT,
-} gc_phase_t;
-
-/**
- * @var qcgc_state
- *
- * Global state of the garbage collector
- */
-struct qcgc_state {
-	object_t **shadow_stack;
-	object_t **shadow_stack_base;
-	shadow_stack_t *prebuilt_objects;
-	weakref_bag_t *weakrefs;
-	gray_stack_t *gp_gray_stack;
-	size_t gray_stack_size;
-	gc_phase_t phase;
-	size_t bytes_since_collection;
-	size_t bytes_since_incmark;
-	size_t free_cells;			// Overall amount of free cells without huge
-								// blocks and free areans. Valid right after sweep
-	size_t largest_free_block;	// Size of the largest free block.
-								// (Free arenas don't count as free blocks)
-								// Valid right after sweep
-} qcgc_state;
diff --git a/rpython/translator/c/src/qcgc/gray_stack.c b/rpython/translator/c/src/qcgc/gray_stack.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/gray_stack.c
+++ /dev/null
@@ -1,60 +0,0 @@
-#include "gray_stack.h"
-
-#include <assert.h>
-#include <stdlib.h>
-
-#include "gc_state.h"
-
-QCGC_STATIC size_t gray_stack_size(size_t size);
-QCGC_STATIC gray_stack_t *gray_stack_grow(gray_stack_t *stack);
-QCGC_STATIC gray_stack_t *gray_stack_shrink(gray_stack_t *stack);
-
-gray_stack_t *qcgc_gray_stack_create(size_t size) {
-	gray_stack_t *result = (gray_stack_t *) malloc(gray_stack_size(size));
-	result->size = size;
-	result->index = 0;
-	return result;
-}
-
-gray_stack_t *qcgc_gray_stack_push(gray_stack_t *stack, object_t *item) {
-	if (stack->size == stack->index) {
-		stack = gray_stack_grow(stack);
-	}
-	stack->items[stack->index] = item;
-	stack->index++;
-	qcgc_state.gray_stack_size++;
-	return stack;
-}
-
-object_t *qcgc_gray_stack_top(gray_stack_t *stack) {
-#if CHECKED
-	assert(stack->index != 0);
-#endif
-	return stack->items[stack->index - 1];
-}
-
-gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack) {
-	// TODO: Add lower bound for size (config.h)
-	if (stack->index < stack->size / 4) {
-		stack = gray_stack_shrink(stack);
-	}
-	stack->index--;
-	qcgc_state.gray_stack_size--;
-	return stack;
-}
-
-QCGC_STATIC size_t gray_stack_size(size_t size) {
-	return (sizeof(gray_stack_t) + size * sizeof(object_t *));
-}
-
-QCGC_STATIC gray_stack_t *gray_stack_grow(gray_stack_t *stack) {
-	stack = (gray_stack_t *) realloc(stack, gray_stack_size(stack->size * 2));
-	stack->size *= 2;
-	return stack;
-}
-
-QCGC_STATIC gray_stack_t *gray_stack_shrink(gray_stack_t *stack) {
-	stack = (gray_stack_t *) realloc(stack, gray_stack_size(stack->size / 2));
-	stack->size /= 2;
-	return stack;
-}
diff --git a/rpython/translator/c/src/qcgc/gray_stack.h b/rpython/translator/c/src/qcgc/gray_stack.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/gray_stack.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#include <stddef.h>
-
-#include "object.h"
-
-typedef struct gray_stack_s {
-	size_t index;
-	size_t size;
-	object_t *items[];
-} gray_stack_t;
-
-__attribute__ ((warn_unused_result))
-gray_stack_t *qcgc_gray_stack_create(size_t size);
-
-__attribute__ ((warn_unused_result))
-gray_stack_t *qcgc_gray_stack_push(gray_stack_t *stack, object_t *item);
-
-object_t *qcgc_gray_stack_top(gray_stack_t *stack);
-
-__attribute__ ((warn_unused_result))
-gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack);
diff --git a/rpython/translator/c/src/qcgc/hugeblocktable.c b/rpython/translator/c/src/qcgc/hugeblocktable.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/hugeblocktable.c
+++ /dev/null
@@ -1,91 +0,0 @@
-#include "hugeblocktable.h"
-
-#include <assert.h>
-
-#include "gc_state.h"
-
-QCGC_STATIC size_t bucket(object_t *object);
-
-void qcgc_hbtable_initialize(void) {
-	qcgc_hbtable.mark_flag_ref = false;
-	for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
-		qcgc_hbtable.bucket[i] = qcgc_hbbucket_create(4);
-	}
-}
-
-void qcgc_hbtable_destroy(void) {
-	for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
-		free(qcgc_hbtable.bucket[i]);
-	}
-}
-
-void qcgc_hbtable_insert(object_t *object) {
-	size_t i = bucket(object);
-	qcgc_hbtable.bucket[i] = qcgc_hbbucket_add(qcgc_hbtable.bucket[i],
-			(struct hbtable_entry_s) {
-				.object = object,
-				.mark_flag = !qcgc_hbtable.mark_flag_ref});
-}
-
-bool qcgc_hbtable_mark(object_t *object) {
-	hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
-	size_t count = b->count;
-	for (size_t i = 0; i < count; i++) {
-		if (b->items[i].object == object) {
-			if (b->items[i].mark_flag != qcgc_hbtable.mark_flag_ref) {
-				b->items[i].mark_flag = qcgc_hbtable.mark_flag_ref;
-				return true;
-			}
-			return false;
-		}
-	}
-#if CHECKED
-	assert(false);
-#endif
-	return false;
-}
-
-bool qcgc_hbtable_has(object_t *object) {
-	hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
-	size_t count = b->count;
-	for (size_t i = 0; i < count; i++) {
-		if (b->items[i].object == object) {
-			return true;
-		}
-	}
-	return false;
-}
-
-bool qcgc_hbtable_is_marked(object_t *object) {
-	hbbucket_t *b = qcgc_hbtable.bucket[bucket(object)];
-	size_t count = b->count;
-	for (size_t i = 0; i < count; i++) {
-		if (b->items[i].object == object) {
-			return b->items[i].mark_flag == qcgc_hbtable.mark_flag_ref;
-		}
-	}
-	return false;
-}
-
-void qcgc_hbtable_sweep(void) {
-	for (size_t i = 0; i < QCGC_HBTABLE_BUCKETS; i++) {
-		hbbucket_t *b = qcgc_hbtable.bucket[i];
-		size_t j = 0;
-		while(j < b->count) {
-			if (b->items[j].mark_flag != qcgc_hbtable.mark_flag_ref) {
-				// White object
-				free(b->items[j].object);
-				b = qcgc_hbbucket_remove_index(b, j);
-			} else {
-				// Black object
-				j++;
-			}
-		}
-		qcgc_hbtable.bucket[i] = b;
-	}
-	qcgc_hbtable.mark_flag_ref = !qcgc_hbtable.mark_flag_ref;
-}
-
-QCGC_STATIC size_t bucket(object_t *object) {
-	return ((uintptr_t) object >> (QCGC_ARENA_SIZE_EXP)) % QCGC_HBTABLE_BUCKETS;
-}
diff --git a/rpython/translator/c/src/qcgc/hugeblocktable.h b/rpython/translator/c/src/qcgc/hugeblocktable.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/hugeblocktable.h
+++ /dev/null
@@ -1,25 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#include <stdbool.h>
-
-#include "bag.h"
-#include "object.h"
-#include "gray_stack.h"
-
-// Choosing a prime number, hoping for good results
-#define QCGC_HBTABLE_BUCKETS 61
-
-struct hbtable_s {
-	bool mark_flag_ref;
-	hbbucket_t *bucket[QCGC_HBTABLE_BUCKETS];
-} qcgc_hbtable;
-
-void qcgc_hbtable_initialize(void);
-void qcgc_hbtable_destroy(void);
-void qcgc_hbtable_insert(object_t *object);
-bool qcgc_hbtable_mark(object_t *object);
-bool qcgc_hbtable_has(object_t *object);
-bool qcgc_hbtable_is_marked(object_t *object);
-void qcgc_hbtable_sweep(void);
diff --git a/rpython/translator/c/src/qcgc/object.h b/rpython/translator/c/src/qcgc/object.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/object.h
+++ /dev/null
@@ -1,17 +0,0 @@
-#pragma once
-
-
-#include "config.h"
-#include <stdint.h>
-
-/**
- * The lower half of flags is reserved for the library, the upper half for
- * clients
- */
-#define QCGC_GRAY_FLAG (1<<0)
-#define QCGC_PREBUILT_OBJECT (1<<1)
-#define QCGC_PREBUILT_REGISTERED (1<<2)
-
-typedef struct object_s {
-	uint32_t flags;
-} object_t;
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -1,507 +1,26 @@
-#include "qcgc.h"
+#include "src/config.h"
+#include "src/allocator.h"
+#include "src/arena.h"
+#include "src/bag.h"
+#include "src/collector.h"
+#include "src/core.h"
+#include "src/event_logger.h"
+#include "src/gc_state.h"
+#include "src/gray_stack.h"
+#include "src/hugeblocktable.h"
+#include "src/object.h"
+#include "src/shadow_stack.h"
+#include "src/signal_handler.h"
+#include "src/weakref.h"
 
-#include <assert.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/mman.h>
-
-#include "allocator.h"
-#include "event_logger.h"
-#include "hugeblocktable.h"
-#include "signal_handler.h"
-
-#define env_or_fallback(var, env_name, fallback) while(0) {		\
-	char *env_val = getenv(env_name);							\
-	if (env_val != NULL) {										\
-		if (1 != sscanf(env_val, "%zu", &var)) {				\
-			var = fallback;										\
-		}														\
-	}															\
-}
-
-void qcgc_mark(bool incremental);
-void qcgc_pop_object(object_t *object);
-void qcgc_push_object(object_t *object);
-void qcgc_sweep(void);
-
-static size_t major_collection_threshold = QCGC_MAJOR_COLLECTION_THRESHOLD;
-static size_t incmark_threshold = QCGC_INCMARK_THRESHOLD;
-
-QCGC_STATIC void update_weakrefs(void);
-QCGC_STATIC void initialize_shadowstack(void);
-QCGC_STATIC void destroy_shadowstack(void);
-
-void qcgc_initialize(void) {
-	initialize_shadowstack();
-	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); // XXX
-	qcgc_state.weakrefs = qcgc_weakref_bag_create(16); // XXX
-	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
-	qcgc_state.gray_stack_size = 0;
-	qcgc_state.phase = GC_PAUSE;
-	qcgc_state.bytes_since_collection = 0;
-	qcgc_state.bytes_since_incmark = 0;
-	qcgc_state.free_cells = 0;
-	qcgc_state.largest_free_block = 0;
-	qcgc_allocator_initialize();
-	qcgc_hbtable_initialize();
-	qcgc_event_logger_initialize();
-
-	env_or_fallback(major_collection_threshold, "QCGC_MAJOR_COLLECTION",
-			QCGC_MAJOR_COLLECTION_THRESHOLD);
-	env_or_fallback(incmark_threshold, "QCGC_INCMARK", QCGC_INCMARK_THRESHOLD);
-
-	setup_signal_handler();
-}
-
-void qcgc_destroy(void) {
-	qcgc_event_logger_destroy();
-	qcgc_hbtable_destroy();
-	qcgc_allocator_destroy();
-	destroy_shadowstack();
-	free(qcgc_state.prebuilt_objects);
-	free(qcgc_state.weakrefs);
-	free(qcgc_state.gp_gray_stack);
-}
-
-/**
- * Shadow stack
- */
-void qcgc_shadowstack_push(object_t *object) {
-	*qcgc_state.shadow_stack = object;
-	qcgc_state.shadow_stack++;
-}
-
-object_t *qcgc_shadowstack_pop(void) {
-	qcgc_state.shadow_stack--;
-	return *qcgc_state.shadow_stack;
-}
-
-/*******************************************************************************
- * Write barrier                                                               *
- ******************************************************************************/
-void qcgc_write(object_t *object) {
-#if CHECKED
-	assert(object != NULL);
-#endif
-	if ((object->flags & QCGC_GRAY_FLAG) != 0) {
-		// Already gray, skip
-		return;
-	}
-	object->flags |= QCGC_GRAY_FLAG;
-
-	// Register prebuilt object if necessary
-	if (((object->flags & QCGC_PREBUILT_OBJECT) != 0) &&
-			((object->flags & QCGC_PREBUILT_REGISTERED) == 0)) {
-		object->flags |= QCGC_PREBUILT_REGISTERED;
-		qcgc_state.prebuilt_objects = qcgc_shadow_stack_push(
-				qcgc_state.prebuilt_objects, object);
-	}
-
-	if (qcgc_state.phase == GC_PAUSE) {
-		return; // We are done
-	}
-
-	// Triggered barrier, we must not collect now
-	qcgc_state.phase = GC_MARK;
-
-	// Test reachability of object and push if neccessary
-	if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
-		// NOTE: No mark test here, as prebuilt objects are always reachable
-		// Push prebuilt object to general purpose gray stack
-		qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
-				qcgc_state.gp_gray_stack, object);
-	} else if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
-		if (qcgc_hbtable_is_marked(object)) {
-			// Push huge block to general purpose gray stack
-			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
-					qcgc_state.gp_gray_stack, object);
-		}
-	} else {
-		if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK) {
-			// This was black before, push it to gray stack again
-			arena_t *arena = qcgc_arena_addr((cell_t *) object);
-			arena->gray_stack = qcgc_gray_stack_push(
-					arena->gray_stack, object);
-		}
-	}
-}
-
-/*******************************************************************************
- * Allocation                                                                  *
- ******************************************************************************/
-
-object_t *qcgc_allocate(size_t size) {
-#if LOG_ALLOCATION
-	qcgc_event_logger_log(EVENT_ALLOCATE_START, sizeof(size_t),
-			(uint8_t *) &size);
-#endif
-	object_t *result;
-
-	if (qcgc_state.bytes_since_collection > major_collection_threshold) {
-		qcgc_collect();
-	}
-	if (qcgc_state.bytes_since_incmark > incmark_threshold) {
-		qcgc_mark(true);
-	}
-
-	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
-		// Use bump / fit allocator
-		if (qcgc_allocator_state.use_bump_allocator) {
-			result = qcgc_bump_allocate(size);
-		} else {
-			result = qcgc_fit_allocate(size);
-
-			// Fallback to bump allocator
-			if (result == NULL) {
-				result = qcgc_bump_allocate(size);
-			}
-		}
-	} else {
-		// Use huge block allocator
-		result = qcgc_large_allocate(size);
-	}
-
-	// XXX: Should we use cells instead of bytes?
-	qcgc_state.bytes_since_collection += size;
-	qcgc_state.bytes_since_incmark += size;
-
-
-#if LOG_ALLOCATION
-	qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
-			(uint8_t *) &result);
-#endif
-	return result;
-}
-
-/*******************************************************************************
- * Collection                                                                  *
- ******************************************************************************/
-
-mark_color_t qcgc_get_mark_color(object_t *object) {
-#if CHECKED
-	assert(object != NULL);
-#endif
-	blocktype_t blocktype = qcgc_arena_get_blocktype((cell_t *) object);
-	bool gray = (object->flags & QCGC_GRAY_FLAG) == QCGC_GRAY_FLAG;
-	if (blocktype == BLOCK_WHITE) {
-		if (gray) {
-			return MARK_COLOR_LIGHT_GRAY;
-		} else {
-			return MARK_COLOR_WHITE;
-		}
-	} else if(blocktype == BLOCK_BLACK) {
-		if (gray) {
-			return MARK_COLOR_DARK_GRAY;

From pypy.commits at gmail.com  Tue Sep 13 03:31:26 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 13 Sep 2016 00:31:26 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase (bugfix)
Message-ID: <57d7ab4e.a699c20a.ae8ce.22fe@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87058:253023b456c9
Date: 2016-09-13 09:30 +0200
http://bitbucket.org/pypy/pypy/changeset/253023b456c9/

Log:	Update qcgc codebase (bugfix)

diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -12,7 +12,7 @@
     _alloc_flavor_ = "raw"
     moving_gc = False
     needs_write_barrier = True
-    malloc_zero_filled = True
+    malloc_zero_filled = False
     prebuilt_gc_objects_are_static_roots = True # XXX: ?
     can_usually_pin_objects = False
     object_minimal_size = 0
@@ -32,7 +32,7 @@
         hdr.tid = llop.combine_ushort(lltype.Signed, typeid, 0)
         hdr.hash = rffi.cast(lltype.Signed, 0)
 
-    def malloc_fixedsize_clear(self, typeid, size,
+    def malloc_fixedsize(self, typeid, size,
                                needs_finalizer=False,
                                is_finalizer_light=False,
                                contains_weakptr=False):
@@ -43,8 +43,7 @@
         self.init_gc_object(obj, typeid)
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
-    def malloc_varsize_clear(self, typeid, length, size, itemsize,
-                             offset_to_length):
+    def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length):
         if length < 0:
             raise MemoryError
         #
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -46,7 +46,7 @@
         if needs_hash:
             hdr.hash = lltype.identityhash_nocache(obj._as_ptr())
         else:
-            assert hdr.hash == 0
+            hdr.hash = 0
         return hdr
 
     def push_roots(self, hop, keep_current_args=False):
diff --git a/rpython/translator/c/src/qcgc/src/allocator.c b/rpython/translator/c/src/qcgc/src/allocator.c
--- a/rpython/translator/c/src/qcgc/src/allocator.c
+++ b/rpython/translator/c/src/qcgc/src/allocator.c
@@ -137,7 +137,7 @@
 	memset(result, 0, cells * sizeof(cell_t));
 #endif
 
-	result->flags |= QCGC_GRAY_FLAG;
+	result->flags = QCGC_GRAY_FLAG;
 #if CHECKED
 	assert(qcgc_arena_is_coalesced(qcgc_arena_addr((cell_t *)result)));
 	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
@@ -243,7 +243,7 @@
 	memset(result, 0, cells * sizeof(cell_t));
 #endif
 
-	result->flags |= QCGC_GRAY_FLAG;
+	result->flags = QCGC_GRAY_FLAG;
 	return result;
 }
 
@@ -260,6 +260,7 @@
 	memset(result, 0, bytes);
 #endif
 	qcgc_hbtable_insert(result);
+	result->flags = QCGC_GRAY_FLAG;
 	return result;
 }
 
diff --git a/rpython/translator/c/src/qcgc/src/config.h b/rpython/translator/c/src/qcgc/src/config.h
--- a/rpython/translator/c/src/qcgc/src/config.h
+++ b/rpython/translator/c/src/qcgc/src/config.h
@@ -3,7 +3,7 @@
 #define CHECKED 0							// Enable runtime sanity checks
 #define DEBUG_ZERO_ON_SWEEP 0				// Zero memory on sweep (debug only)
 
-#define QCGC_INIT_ZERO 1					// Init new objects with zero bytes
+#define QCGC_INIT_ZERO 0					// Init new objects with zero bytes
 
 /**
  * Event logger
diff --git a/rpython/translator/c/src/qcgc/src/core.c b/rpython/translator/c/src/qcgc/src/core.c
--- a/rpython/translator/c/src/qcgc/src/core.c
+++ b/rpython/translator/c/src/qcgc/src/core.c
@@ -85,8 +85,7 @@
 
 	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
 		// Use bump / fit allocator
-		//if (qcgc_allocator_state.use_bump_allocator) {
-		if (true) {
+		if (qcgc_allocator_state.use_bump_allocator) {
 			result = qcgc_bump_allocate(size);
 		} else {
 			result = qcgc_fit_allocate(size);

From pypy.commits at gmail.com  Tue Sep 13 04:22:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 13 Sep 2016 01:22:58 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: resolve some test issues
Message-ID: <57d7b762.68adc20a.aeaa2.2898@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87059:7384b2be2e75
Date: 2016-09-13 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/7384b2be2e75/

Log:	resolve some test issues

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -16,7 +16,7 @@
 from rpython.jit.backend.detect_cpu import getcpuclass
 
 CPU = getcpuclass()
-if not CPU.vector_extension:
+if not CPU.vector_ext:
     py.test.skip("this cpu %s has no implemented vector backend" % CPU)
 
 def get_profiler():
@@ -29,7 +29,7 @@
     interp = None
 
     def setup_method(self, method):
-        if not self.CPUClass.vector_extension:
+        if not self.CPUClass.vector_ext:
             py.test.skip("needs vector extension to run (for now)")
 
     def assert_float_equal(self, f1, f2, delta=0.0001):
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -553,14 +553,15 @@
     """
     def __init__(self, loop):
         self.loop = loop
-        self.label = Node(loop.label, 0)
+        label = loop.prefix_label or loop.label
+        self.label = Node(label, 0)
         self.nodes = [ Node(op,0) for op in loop.operations if not rop.is_jit_debug(op.opnum) ]
         for i,node in enumerate(self.nodes):
             node.opidx = i+1
         self.inodes = [] # imaginary nodes
         self.jump = Node(loop.jump, len(self.nodes)+1)
         self.invariant_vars = {}
-        self.update_invariant_vars()
+        self.update_invariant_vars(label)
         self.memory_refs = {}
         self.schedulable_nodes = []
         self.index_vars = {}
@@ -576,8 +577,9 @@
         self.inodes.append(node)
         return node
 
-    def update_invariant_vars(self):
-        label_op = self.label.getoperation()
+    def update_invariant_vars(self, label_op=None):
+        if not label_op:
+            label_op = self.label.getoperation()
         jump_op = self.jump.getoperation()
         assert label_op.numargs() == jump_op.numargs()
         for i in range(label_op.numargs()):
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -84,33 +84,14 @@
         self.renamer.rename(op)
         self.oplist.append(op)
 
-
     def post_schedule(self):
         loop = self.graph.loop
-        #
         if self.delayed:
             # some operations can be delayed until the jump instruction,
             # handle them here
             self.resolve_delayed({}, self.delayed, loop.jump)
-
-        #
+        loop.operations = self.oplist
         self.renamer.rename(loop.jump)
-        self.ensure_args_unpacked(loop.jump)
-        loop.operations = self.oplist
-        loop.prefix = self.invariant_oplist
-        if len(self.invariant_vector_vars) + len(self.invariant_oplist) > 0:
-            # label
-            args = loop.label.getarglist_copy() + self.invariant_vector_vars
-            opnum = loop.label.getopnum()
-            op = loop.label.copy_and_change(opnum, args)
-            self.renamer.rename(op)
-            loop.prefix_label = op
-            # jump
-            args = loop.jump.getarglist_copy() + self.invariant_vector_vars
-            opnum = loop.jump.getopnum()
-            op = loop.jump.copy_and_change(opnum, args)
-            self.renamer.rename(op)
-            loop.jump = op
 
     def profitable(self):
         return True
@@ -755,6 +736,25 @@
                 break
             self.setvector_of_box(arg, i, box)
 
+    def post_schedule(self):
+        SchedulerState.post_schedule(self)
+        loop = self.graph.loop
+        self.ensure_args_unpacked(loop.jump)
+        loop.prefix = self.invariant_oplist
+        if len(self.invariant_vector_vars) + len(self.invariant_oplist) > 0:
+            # label
+            args = loop.label.getarglist_copy() + self.invariant_vector_vars
+            opnum = loop.label.getopnum()
+            op = loop.label.copy_and_change(opnum, args)
+            self.renamer.rename(op)
+            loop.prefix_label = op
+            # jump
+            args = loop.jump.getarglist_copy() + self.invariant_vector_vars
+            opnum = loop.jump.getopnum()
+            op = loop.jump.copy_and_change(opnum, args)
+            self.renamer.rename(op)
+            loop.jump = op
+
 class Pack(object):
     """ A pack is a set of n statements that are:
         * isomorphic
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -205,6 +205,7 @@
         state.post_schedule()
         #
         oplist = loop.operations
+
         loop.operations = loop.prefix[:]
         if loop.prefix_label:
             loop.operations += [loop.prefix_label]
@@ -1074,23 +1075,14 @@
         guard_true(i3) [p0,i0]
         jump(p0,i2)
         """
-        dead_code =  '\n        '.join([
-          "i{t1} = int_add(i0,{i})\n        i{s} = int_lt(i{t1}, 102)".format(
-              i=i+2, t1=i+201, t=i+200, s=i+20)
-          for i in range(0,14)])
         opt="""
         [p0,i0]
-        i200 = int_add(i0, 1)
-        i400 = int_lt(i200, 102)
         i2 = int_add(i0, 16)
         i3 = int_lt(i2, 102)
         guard_true(i3) [p0,i0]
-        {dead_code}
-        i500 = int_add(i0, 16)
-        i501 = int_lt(i500, 102)
         v10[16xi8] = vec_load_i(p0, i0, 1, 0, descr=chararraydescr)
         jump(p0,i2)
-        """.format(dead_code=dead_code)
+        """
         loop = self.parse_loop(ops)
         vopt = self.schedule(loop,15,with_guard_opt=True)
         self.assert_equal(loop, self.parse_loop(opt))
@@ -1127,13 +1119,9 @@
         [p0,i0]
         v3[2xf64] = vec_expand_i(42)
         label(p0,i0,v3[2xf64])
-        i20 = int_add(i0, 1)
-        i30 = int_lt(i20, 10)
         i2 = int_add(i0, 2)
         i3 = int_lt(i2, 10)
         guard_true(i3) [p0,i0]
-        i4 = int_add(i0, 2)
-        i5 = int_lt(i4, 10)
         v1[2xf64] = vec_load_i(p0, i0, 8, 0, descr=arraydescr)
         v2[2xf64] = vec_int_sub(v1[2xf64], v3[2xf64])
         vec_store(p0, i0, v2[2xf64], 8, 0, descr=arraydescr)
@@ -1158,13 +1146,9 @@
         [p0,i0,f3]
         v3[2xf64] = vec_expand_f(f3)
         label(p0,i0,f3,v3[2xf64])
-        i20 = int_add(i0, 1)
-        i30 = int_lt(i20, 10)
         i2 = int_add(i0, 2)
         i3 = int_lt(i2, 10)
         guard_true(i3) [p0,i0,f3]
-        i4 = int_add(i0, 2)
-        i5 = int_lt(i4, 10)
         v1[2xf64] = vec_load_f(p0, i0, 8, 0, descr=floatarraydescr)
         v2[2xf64] = vec_float_add(v1[2xf64], v3[2xf64])
         vec_store(p0, i0, v2[2xf64], 8, 0, descr=floatarraydescr)
@@ -1190,13 +1174,9 @@
         v7[2xf64] = vec_float_xor(v6[0xf64], v6[0xf64])
         v2[2xf64] = vec_pack_f(v7[2xf64], f0, 0, 1)
         label(p0, i0, v2[2xf64])
-        i100 = int_add(i0, 8)
-        i200 = int_lt(i100, 100)
         i1 = int_add(i0, 16)
         i2 = int_lt(i1, 100)
         guard_true(i2) [p0, i0, v2[2xf64]]
-        i10 = int_add(i0, 16)
-        i20 = int_lt(i10, 100)
         v1[2xf64] = vec_load_f(p0, i0, 1, 0, descr=floatarraydescr)
         v3[2xf64] = vec_float_add(v2[2xf64], v1[2xf64])
         jump(p0, i1, v3[2xf64])
@@ -1224,20 +1204,16 @@
         trace_opt = self.parse_loop("""
         [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18]
         guard_not_invalidated() [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18]
-        i50 = int_add(i28, 1) 
-        i20 = int_ge(i50, i18)
         i54 = int_add(i28, 2) 
         i638 = int_ge(i54, i18)
         guard_false(i638) [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18]
-        i47 = int_add(i28, 2) 
-        i52 = int_ge(i47, i18) 
         v61[2xf64] = vec_load_f(i21, i44, 1, 0, descr=floatarraydescr) 
         v62[2xf64] = vec_load_f(i4, i41, 1, 0, descr=floatarraydescr) 
         v63[2xf64] = vec_float_add(v61, v62) 
         vec_store(i0, i37, v63, 1, 0, descr=floatarraydescr) 
+        i637 = int_add(i37, 16)
+        i629 = int_add(i41, 16)
         i55 = int_add(i44, 16) 
-        i629 = int_add(i41, 16)
-        i637 = int_add(i37, 16)
         f100 = vec_unpack_f(v61, 1, 1)
         f101 = vec_unpack_f(v62, 1, 1)
         jump(p36, i637, p9, i629, p14, f100, p12, p38, f101, p39, i40, i54, p42, i43, i55, i21, i4, i0, i18)
@@ -1258,20 +1234,13 @@
         """
         opt = """
         [p0, p1, i1]
-        i3 = int_add(i1, 1)
-        i4 = int_ge(i3, 36)
         i50 = int_add(i1, 4)
         i51 = int_ge(i50, 36)
         guard_false(i51) [p0, p1, i1]
+        v17 = vec_load_f(p0, i1, 8, 0, descr=floatarraydescr)
         i5 = int_add(i1, 2)
-        i8 = int_ge(i5, 36)
-        i6 = int_add(i1, 3)
-        i11 = int_ge(i6, 36)
-        i7 = int_add(i1, 4)
-        i14 = int_ge(i7, 36)
-        v17 = vec_load_f(p0, i1, 8, 0, descr=floatarraydescr)
+        v18 = vec_load_f(p0, i5, 8, 0, descr=floatarraydescr)
         v19 = vec_cast_float_to_singlefloat(v17)
-        v18 = vec_load_f(p0, i5, 8, 0, descr=floatarraydescr)
         v20 = vec_cast_float_to_singlefloat(v18)
         v21 = vec_pack_i(v19, v20, 2, 2)
         vec_store(p1, i1, v21, 4, 0, descr=float32arraydescr)
@@ -1299,26 +1268,18 @@
         """)
         trace_opt = self.parse_loop("""
         [p0, p1, p2, i0, i4]
-        i5 = int_add(i4, 4)
-        i186 = int_lt(i5, 100)
         i500 = int_add(i4, 16)
         i501 = int_lt(i500, 100)
         guard_true(i501) [p0, p1, p2, i0, i4]
-        i187 = int_add(i4, 8)
-        i188 = int_lt(i187, 100)
-        i196 = int_add(i4, 12)
-        i197 = int_lt(i196, 100)
-        i400 = int_add(i4, 16)
-        i401= int_lt(i400, 100)
         v228[4xi32] = vec_load_i(p0, i0, 1, 0, descr=float32arraydescr)
-        v229[2xf64] = vec_cast_singlefloat_to_float(v228)
-        v230 = vec_unpack_i(v228, 2, 2)
-        v231 = vec_cast_singlefloat_to_float(v230)
         i189 = int_add(i0, 4)
         v232 = vec_load_i(p1, i189, 1, 0, descr=float32arraydescr)
+        v229[2xf64] = vec_cast_singlefloat_to_float(v228)
         v233 = vec_cast_singlefloat_to_float(v232)
         v236 = vec_float_add(v229, v233)
         v238 = vec_cast_float_to_singlefloat(v236)
+        v230 = vec_unpack_i(v228, 2, 2)
+        v231 = vec_cast_singlefloat_to_float(v230)
         v234 = vec_unpack_i(v232, 2, 2)
         v235 = vec_cast_singlefloat_to_float(v234)
         v237 = vec_float_add(v231, v235)
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -58,7 +58,6 @@
             op.set_forwarded(None)
 
     def finaloplist(self, jitcell_token=None, reset_label_token=True, label=False):
-        oplist = []
         if jitcell_token:
             if reset_label_token:
                 token = TargetToken(jitcell_token)
@@ -76,6 +75,7 @@
                 self.jump.setdescr(token)
             if reset_label_token:
                 self.jump.setdescr(token)
+        oplist = []
         if self.prefix_label:
             oplist = self.prefix + [self.prefix_label]
         elif self.prefix:
@@ -133,7 +133,7 @@
         #
         start = time.clock()
         opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, warmstate.vec_cost)
-        opt.run_optimization(metainterp_sd, info, loop)
+        oplist = opt.run_optimization(metainterp_sd, info, loop, jitcell_token, user_code)
         end = time.clock()
         #
         metainterp_sd.profiler.count(Counters.OPT_VECTORIZED)
@@ -144,7 +144,7 @@
         debug_stop("vec-opt-loop")
         #
         info.label_op = loop.label
-        return info, loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
+        return info, oplist
     except NotAVectorizeableLoop:
         debug_stop("vec-opt-loop")
         # vectorization is not possible
@@ -221,7 +221,7 @@
         self.smallest_type_bytes = 0
         self.orig_label_args = None
 
-    def run_optimization(self, metainterp_sd, info, loop):
+    def run_optimization(self, metainterp_sd, info, loop, jitcell_token, user_code):
         self.orig_label_args = loop.label.getarglist_copy()
         self.linear_find_smallest_type(loop)
         byte_count = self.smallest_type_bytes
@@ -260,15 +260,16 @@
         gso = GuardStrengthenOpt(graph.index_vars)
         gso.propagate_all_forward(info, loop, user_code)
 
-        # re-schedule the trace -> removes index operations
+        # re-schedule the trace -> removes many pure operations
         graph = DependencyGraph(loop)
         costmodel = GenericCostModel(self.cpu, self.cost_threshold)
-        state = ScheduleState(self.cpu, graph)
+        state = SchedulerState(self.cpu, graph)
         state.prepare()
-        scheduler = Scheduler()
-        scheduler.walk_and_emit(state)
+        Scheduler().walk_and_emit(state)
         state.post_schedule()
 
+        return loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
+
     def unroll_loop_iterations(self, loop, unroll_count):
         """ Unroll the loop X times. unroll_count + 1 = unroll_factor """
         numops = len(loop.operations)

From pypy.commits at gmail.com  Tue Sep 13 04:32:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 01:32:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57d7b993.02c51c0a.cdea0.3141@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87060:6a82949682c3
Date: 2016-09-13 09:25 +0100
http://bitbucket.org/pypy/pypy/changeset/6a82949682c3/

Log:	hg merge py3k

diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -2,7 +2,7 @@
 from rpython.rlib import buffer
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, Py_DecRef
 
 class CBuffer(buffer.Buffer):
 

From pypy.commits at gmail.com  Tue Sep 13 04:32:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 01:32:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: Warning
Message-ID: <57d7b994.28eac20a.666bd.355e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87061:a8a4ffe56b74
Date: 2016-09-13 09:37 +0100
http://bitbucket.org/pypy/pypy/changeset/a8a4ffe56b74/

Log:	Warning

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -250,6 +250,8 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
+                if hasattr(graph.func, '_call_shortcut_'):
+                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -259,6 +259,8 @@
 
 def call_shortcut(func):
     """A decorator to ensure that a function has a fast-path.
+    DOES NOT RELIABLY WORK ON METHODS, USE ONLY ON FUNCTIONS!
+
     Only useful on functions that the JIT doesn't normally look inside.
     It still replaces residual calls to that function with inline code
     that checks for a fast path, and only does the call if not.  For

From pypy.commits at gmail.com  Tue Sep 13 04:32:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 01:32:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57d7b996.4756c20a.dc392.0ac1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87062:5f216ce9b438
Date: 2016-09-13 10:31 +0200
http://bitbucket.org/pypy/pypy/changeset/5f216ce9b438/

Log:	hg merge default

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -28,10 +28,7 @@
     else:
         n = len(fmt)
     for i in range(n):
-        if ord(fmt[i]) > 255:
-            view.c_format[i] = '*'
-        else:
-            view.c_format[i] = fmt[i]
+        view.c_format[i] = fmt[i]
     view.c_format[n] = '\x00'
     shape = buf.getshape()
     strides = buf.getstrides()
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,9 +4,7 @@
 PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
 {
     PyObject* newType;
-    /*printf("in foo3type_tp_new, preprocessing...\n"); */
     newType = PyType_Type.tp_new(metatype, args, kwds);
-    /*printf("in foo3type_tp_new, postprocessing...\n"); */
     return newType;
 }
 
@@ -93,5 +91,5 @@
         return NULL;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return NULL;
-    return mod;
+    Py_INCREF(&footype);
 }
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -944,7 +944,6 @@
 
     def test_tp_new_in_subclass_of_type(self):
         module = self.import_module(name='foo3')
-        #print('calling module.footype()...')
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -22,7 +22,7 @@
             del self.__class__.appleveldefs['__stderr__']
 
         super(Module, self).__init__(space, w_name)
-        self.recursionlimit = 100
+        self.recursionlimit = 1000
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -250,6 +250,8 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
+                if hasattr(graph.func, '_call_shortcut_'):
+                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -259,6 +259,8 @@
 
 def call_shortcut(func):
     """A decorator to ensure that a function has a fast-path.
+    DOES NOT RELIABLY WORK ON METHODS, USE ONLY ON FUNCTIONS!
+
     Only useful on functions that the JIT doesn't normally look inside.
     It still replaces residual calls to that function with inline code
     that checks for a fast path, and only does the call if not.  For

From pypy.commits at gmail.com  Tue Sep 13 04:32:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 01:32:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57d7b998.854bc20a.232de.7212@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87063:29fc1b0602a8
Date: 2016-09-13 10:31 +0200
http://bitbucket.org/pypy/pypy/changeset/29fc1b0602a8/

Log:	hg merge py3k

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -28,10 +28,7 @@
     else:
         n = len(fmt)
     for i in range(n):
-        if ord(fmt[i]) > 255:
-            view.c_format[i] = '*'
-        else:
-            view.c_format[i] = fmt[i]
+        view.c_format[i] = fmt[i]
     view.c_format[n] = '\x00'
     shape = buf.getshape()
     strides = buf.getstrides()
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,9 +4,7 @@
 PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
 {
     PyObject* newType;
-    /*printf("in foo3type_tp_new, preprocessing...\n"); */
     newType = PyType_Type.tp_new(metatype, args, kwds);
-    /*printf("in foo3type_tp_new, postprocessing...\n"); */
     return newType;
 }
 
@@ -93,5 +91,5 @@
         return NULL;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return NULL;
-    return mod;
+    Py_INCREF(&footype);
 }
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -944,7 +944,6 @@
 
     def test_tp_new_in_subclass_of_type(self):
         module = self.import_module(name='foo3')
-        #print('calling module.footype()...')
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -22,7 +22,7 @@
             del self.__class__.appleveldefs['__stderr__']
 
         super(Module, self).__init__(space, w_name)
-        self.recursionlimit = 100
+        self.recursionlimit = 1000
         self.defaultencoding = "utf-8"
         self.filesystemencoding = None
         self.debug = True
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -250,6 +250,8 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
+                if hasattr(graph.func, '_call_shortcut_'):
+                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -259,6 +259,8 @@
 
 def call_shortcut(func):
     """A decorator to ensure that a function has a fast-path.
+    DOES NOT RELIABLY WORK ON METHODS, USE ONLY ON FUNCTIONS!
+
     Only useful on functions that the JIT doesn't normally look inside.
     It still replaces residual calls to that function with inline code
     that checks for a fast path, and only does the call if not.  For

From pypy.commits at gmail.com  Tue Sep 13 04:35:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 01:35:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Move this method as a global function
Message-ID: <57d7ba44.010c1c0a.8043.57d4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87064:464f767392fb
Date: 2016-09-13 10:34 +0200
http://bitbucket.org/pypy/pypy/changeset/464f767392fb/

Log:	Move this method as a global function

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -77,21 +77,8 @@
     def unicode_w(self, space):
         return self._value
 
-    @jit.elidable
-    @jit.call_shortcut
     def identifier_w(self, space):
-        identifier = self._utf8
-        if identifier is not None:
-            return identifier
-        u = self._value
-        eh = unicodehelper.rpy_encode_error_handler()
-        try:
-            identifier = unicode_encode_utf_8(u, len(u), None,
-                                              errorhandler=eh)
-        except unicodehelper.RUnicodeEncodeError as ue:
-            raise wrap_encode_error(space, ue)
-        self._utf8 = identifier
-        return identifier
+        return g_identifier_w(self, space)
 
     def listview_unicode(self):
         return _create_list_from_unicode(self._value)
@@ -1307,6 +1294,22 @@
     return unicodehelper.encode_utf8(space, u''.join(result),
                                      allow_surrogates=allow_surrogates)
 
+ at jit.elidable
+ at jit.call_shortcut
+def g_identifier_w(self, space):
+    """This is a global function because of @jit.call_shortcut"""
+    identifier = self._utf8
+    if identifier is not None:
+        return identifier
+    u = self._value
+    eh = unicodehelper.rpy_encode_error_handler()
+    try:
+        identifier = unicode_encode_utf_8(u, len(u), None,
+                                          errorhandler=eh)
+    except unicodehelper.RUnicodeEncodeError as ue:
+        raise wrap_encode_error(space, ue)
+    self._utf8 = identifier
+    return identifier
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=True, unicode_output=True, quotes=True, prefix='')

From pypy.commits at gmail.com  Tue Sep 13 04:46:26 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 13 Sep 2016 01:46:26 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: translation issue,
 no common base class
Message-ID: <57d7bce2.c70a1c0a.2484a.329d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87065:eebdc539c56a
Date: 2016-09-13 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/eebdc539c56a/

Log:	translation issue, no common base class

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -195,7 +195,8 @@
                 op = node.getoperation()
                 self.resolve_delayed({}, delayed, op)
             for node in delayed:
-                if node in self.seen:
+                op = node.getoperation()
+                if op in self.seen:
                     continue
                 if node is not None:
                     provides = node.provides()

From pypy.commits at gmail.com  Tue Sep 13 04:47:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 13 Sep 2016 01:47:54 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: start new branch for zarch
 simd
Message-ID: <57d7bd3a.2858c20a.19c40.4465@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87066:33adc424fba3
Date: 2016-09-13 10:47 +0200
http://bitbucket.org/pypy/pypy/changeset/33adc424fba3/

Log:	start new branch for zarch simd

diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -0,0 +1,2 @@
+
+

From pypy.commits at gmail.com  Tue Sep 13 05:44:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 02:44:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Back out ce4d4b3c159f,
 i.e. redo 26dfa2f67afd, in the py3.5 branch where
Message-ID: <57d7ca6f.a7bec20a.d01f4.50f2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87067:438fce751fce
Date: 2016-09-13 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/438fce751fce/

Log:	Back out ce4d4b3c159f, i.e. redo 26dfa2f67afd, in the py3.5 branch
	where imports are done differently. I'll see what exactly this
	breaks in py3.5 before deciding something, e.g., a space option we'd
	need to set in the tests that do break. I'm not doing it in py3k
	this time.

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -139,10 +139,16 @@
         space = self.space
 
         if not space.config.translating:
-            from pypy.module.sys.interp_encoding import _getfilesystemencoding
-            self.filesystemencoding = _getfilesystemencoding(space)
+            ##from pypy.module.sys.interp_encoding import _getfilesystemencoding
+            ##self.filesystemencoding = _getfilesystemencoding(space)
             # XXX the two lines above take a few seconds to run whenever
-            # we initialize the space
+            # we initialize the space; for tests, use a simpler version.
+            # Check what exactly breaks, if anything, in py3.5.  This is
+            # not strictly necessary but is an extremely nice-to-have
+            # feature: running just one test for example take 3.5
+            # seconds instead of 11.
+            from pypy.module.sys.interp_encoding import base_encoding
+            self.filesystemencoding = base_encoding
 
     def flush_std_files(self, space):
         w_stdout = space.sys.getdictvalue(space, 'stdout')

From pypy.commits at gmail.com  Tue Sep 13 05:45:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 02:45:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Tweak tweak, now it seems to translate
Message-ID: <57d7cab8.081dc20a.434ed.5592@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87068:52f22f2a9d90
Date: 2016-09-13 10:54 +0100
http://bitbucket.org/pypy/pypy/changeset/52f22f2a9d90/

Log:	Tweak tweak, now it seems to translate

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -78,7 +78,21 @@
         return self._value
 
     def identifier_w(self, space):
-        return g_identifier_w(self, space)
+        try:
+            # call the elidable function, with a jit.call_shortcut in case
+            # self._utf8 is already computed
+            identifier = g_identifier_w(self, space)
+        except UnicodeEncodeError:
+            # bah, this is just to get an official app-level
+            # UnicodeEncodeError
+            u = self._value
+            eh = unicodehelper.rpy_encode_error_handler()
+            try:
+                identifier = unicode_encode_utf_8(u, len(u), None,
+                                                  errorhandler=eh)
+            except unicodehelper.RUnicodeEncodeError as ue:
+                raise wrap_encode_error(space, ue)
+        return identifier
 
     def listview_unicode(self):
         return _create_list_from_unicode(self._value)
@@ -1301,13 +1315,7 @@
     identifier = self._utf8
     if identifier is not None:
         return identifier
-    u = self._value
-    eh = unicodehelper.rpy_encode_error_handler()
-    try:
-        identifier = unicode_encode_utf_8(u, len(u), None,
-                                          errorhandler=eh)
-    except unicodehelper.RUnicodeEncodeError as ue:
-        raise wrap_encode_error(space, ue)
+    identifier = self._value.encode('utf-8')
     self._utf8 = identifier
     return identifier
 

From pypy.commits at gmail.com  Tue Sep 13 10:09:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 13 Sep 2016 07:09:47 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: copied over vector_ext file,
 some minor modifications
Message-ID: <57d808ab.a468c20a.84054.b5f5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87069:29bd4e207e97
Date: 2016-09-13 10:59 +0200
http://bitbucket.org/pypy/pypy/changeset/29bd4e207e97/

Log:	copied over vector_ext file, some minor modifications

diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -1,2 +1,842 @@
+import py
+from rpython.jit.metainterp.compile import ResumeGuardDescr
+from rpython.jit.metainterp.history import (ConstInt, INT, REF,
+    FLOAT, VECTOR, TargetToken)
+from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
+    unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
+from rpython.jit.backend.llsupport.regalloc import get_scale
+from rpython.jit.metainterp.resoperation import (rop, ResOperation,
+        VectorOp, VectorGuardOp)
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem import lltype
+from rpython.jit.backend.ppc.locations import imm, RegisterLocation
+from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN
+from rpython.jit.backend.llsupport.vector_ext import VectorExt
+from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET
+import rpython.jit.backend.ppc.register as r
+import rpython.jit.backend.ppc.condition as c
+import rpython.jit.backend.ppc.locations as l
+from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.jit.codewriter import longlong
+from rpython.jit.backend.ppc.detect_feature import detect_vsx
+from rpython.rlib.objectmodel import always_inline
 
+def not_implemented(msg):
+    msg = '[zarch/vector_ext] %s\n' % msg
+    if we_are_translated():
+        llop.debug_print(lltype.Void, msg)
+    raise NotImplementedError(msg)
 
+def flush_vec_cc(asm, regalloc, condition, size, result_loc):
+    # After emitting an instruction that leaves a boolean result in
+    # a condition code (cc), call this.  In the common case, result_loc
+    # will be set to SPP by the regalloc, which in this case means
+    # "propagate it between this operation and the next guard by keeping
+    # it in the cc".  In the uncommon case, result_loc is another
+    # register, and we emit a load from the cc into this register.
+
+    # Possibly invert the bit in the CR
+    #bit, invert = c.encoding[condition]
+    #assert 24 <= bit <= 27
+    #if invert == 12:
+    #    pass
+    #elif invert == 4:
+    #    asm.mc.crnor(bit, bit, bit)
+    #else:
+    #    assert 0
+    #assert asm.guard_success_cc == c.cond_none
+    ##
+    #if result_loc is r.SPP:
+    #    asm.guard_success_cc = condition
+    #else:
+    #    resval = result_loc.value
+    #    # either doubleword integer 1 (2x) or word integer 1 (4x)
+    #    ones = regalloc.vrm.get_scratch_reg(type=INT).value
+    #    zeros = regalloc.vrm.get_scratch_reg(type=INT).value
+    #    asm.mc.vxor(zeros, zeros, zeros)
+    #    if size == 4:
+    #        asm.mc.vspltisw(ones, 1)
+    #    else:
+    #        assert size == 8
+    #        tloc = regalloc.rm.get_scratch_reg()
+    #        asm.mc.load_imm(tloc, asm.VEC_DOUBLE_WORD_ONES)
+    #        asm.mc.lvx(ones, 0, tloc.value)
+    #    asm.mc.vsel(resval, zeros, ones, resval)
+
+class ZSIMDVectorExt(VectorExt):
+    def setup_once(self, asm):
+        if detect_simd_z():
+            self.enable(16, accum=True)
+            asm.setup_once_vector()
+        self._setup = True
+
+class VectorAssembler(object):
+    _mixin_ = True
+
+    # TODO VEC_DOUBLE_WORD_ONES = 0
+
+    def setup_once_vector(self):
+        # TODO if IS_BIG_ENDIAN:
+        # TODO     # 2x 64 bit signed integer(1) BE
+        # TODO     data = (b'\x00' * 7 + b'\x01') * 2
+        # TODO else:
+        # TODO     # 2x 64 bit signed integer(1) LE
+        # TODO     data = (b'\x01' + b'\x00' * 7) * 2
+        # TODO datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, [])
+        # TODO mem = datablockwrapper.malloc_aligned(len(data), alignment=16)
+        # TODO datablockwrapper.done()
+        # TODO addr = rffi.cast(rffi.CArrayPtr(lltype.Char), mem)
+        # TODO for i in range(len(data)):
+        # TODO     addr[i] = data[i]
+        # TODO self.VEC_DOUBLE_WORD_ONES = mem
+
+    def emit_vec_load_f(self, op, arglocs, regalloc):
+        resloc, baseloc, indexloc, size_loc, ofs, integer_loc = arglocs
+        indexloc = self._apply_offset(indexloc, ofs)
+        itemsize = size_loc.value
+        if integer_loc.value:
+            self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+        elif itemsize == 4:
+            self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
+        elif itemsize == 8:
+            self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+        else:
+            not_implemented("vec_load_f itemsize %d" % itemsize)
+
+    emit_vec_load_i = emit_vec_load_f
+
+    def emit_vec_store(self, op, arglocs, regalloc):
+        baseloc, indexloc, valueloc, sizeloc, baseofs, \
+            integer_loc = arglocs
+        indexloc = self._apply_offset(indexloc, baseofs)
+        assert baseofs.value == 0
+        if integer_loc.value:
+            self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
+        else:
+            itemsize = sizeloc.value
+            if itemsize == 4:
+                self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value)
+            elif itemsize == 8:
+                self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
+            else:
+                not_implemented("vec_store itemsize %d" % itemsize)
+
+    def emit_vec_int_add(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, size_loc = arglocs
+        size = size_loc.value
+        if size == 1:
+            self.mc.vaddubm(resloc.value, loc0.value, loc1.value)
+        elif size == 2:
+            self.mc.vadduhm(resloc.value, loc0.value, loc1.value)
+        elif size == 4:
+            self.mc.vadduwm(resloc.value, loc0.value, loc1.value)
+        elif size == 8:
+            self.mc.vaddudm(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_int_sub(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, size_loc = arglocs
+        size = size_loc.value
+        if size == 1:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsububm(resloc.value, loc0.value, loc1.value)
+        elif size == 2:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsubuhm(resloc.value, loc0.value, loc1.value)
+        elif size == 4:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsubuwm(resloc.value, loc0.value, loc1.value)
+        elif size == 8:
+            self.mc.vsubudm(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_add(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvaddsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvadddp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_sub(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvsubsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvsubdp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_mul(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvmulsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvmuldp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_truediv(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvdivsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvdivdp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_int_mul(self, op, arglocs, regalloc):
+        raise NotImplementedError
+        pass # TODO
+
+    def emit_vec_int_and(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, sizeloc = arglocs
+        self.mc.vand(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_int_or(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, sizeloc = arglocs
+        self.mc.vor(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_int_xor(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, sizeloc = arglocs
+        self.mc.vxor(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_int_signext(self, op, arglocs, regalloc):
+        resloc, loc0 = arglocs
+        # TODO
+        self.regalloc_mov(loc0, resloc)
+
+    def emit_vec_float_abs(self, op, arglocs, regalloc):
+        resloc, argloc, sizeloc = arglocs
+        size = sizeloc.value
+        if size == 4:
+            self.mc.xvabssp(resloc.value, argloc.value)
+        elif size == 8:
+            self.mc.xvabsdp(resloc.value, argloc.value)
+        else:
+            not_implemented("float abs for size %d" % size)
+
+    def emit_vec_float_neg(self, op, arglocs, regalloc):
+        resloc, argloc, sizeloc = arglocs
+        size = sizeloc.value
+        if size == 4:
+            self.mc.xvnegsp(resloc.value, argloc.value)
+        elif size == 8:
+            self.mc.xvnegdp(resloc.value, argloc.value)
+        else:
+            not_implemented("float neg for size %d" % size)
+
+    def emit_vec_guard_true(self, guard_op, arglocs, regalloc):
+        self._emit_guard(guard_op, arglocs)
+
+    def emit_vec_guard_false(self, guard_op, arglocs, regalloc):
+        self.guard_success_cc = c.negate(self.guard_success_cc)
+        self._emit_guard(guard_op, arglocs)
+
+    def _update_at_exit(self, fail_locs, fail_args, faildescr, regalloc):
+        """ If accumulation is done in this loop, at the guard exit
+            some vector registers must be adjusted to yield the correct value
+        """
+        if not isinstance(faildescr, ResumeGuardDescr):
+            return
+        accum_info = faildescr.rd_vector_info
+        while accum_info:
+            pos = accum_info.getpos_in_failargs()
+            scalar_loc = fail_locs[pos]
+            vector_loc = accum_info.location
+            # the upper elements will be lost if saved to the stack!
+            scalar_arg = accum_info.getoriginal()
+            if not scalar_loc.is_reg():
+                scalar_loc = regalloc.force_allocate_reg(scalar_arg)
+            assert scalar_arg is not None
+            op = accum_info.accum_operation
+            self._accum_reduce(op, scalar_arg, vector_loc, scalar_loc)
+            accum_info = accum_info.next()
+
+    def _accum_reduce(self, op, arg, accumloc, targetloc):
+        # Currently the accumulator can ONLY be the biggest
+        # 64 bit float/int
+        tgt = targetloc.value
+        acc = accumloc.value
+        if arg.type == FLOAT:
+            # r = (r[0]+r[1],r[0]+r[1])
+            if IS_BIG_ENDIAN:
+                self.mc.xxpermdi(tgt, acc, acc, 0b00)
+            else:
+                self.mc.xxpermdi(tgt, acc, acc, 0b10)
+            if op == '+':
+                self.mc.xsadddp(tgt, tgt, acc)
+            elif op == '*':
+                self.mc.xsmuldp(tgt, tgt, acc)
+            else:
+                not_implemented("sum not implemented")
+            return
+        else:
+            assert arg.type == INT
+            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
+            self.mc.stvx(acc, r.SCRATCH2.value, r.SP.value)
+            self.mc.load(tgt, r.SP.value, PARAM_SAVE_AREA_OFFSET)
+            self.mc.load(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
+            if op == '+':
+                self.mc.add(tgt, tgt, acc)
+            elif op == '*':
+                self.mc.mulld(tgt, tgt, acc)
+            else:
+                not_implemented("sum not implemented")
+            return
+
+        not_implemented("reduce sum for %s not impl." % arg)
+
+    def emit_vec_int_is_true(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resloc, argloc, sizeloc = arglocs
+        size = sizeloc.value
+        tmp = regalloc.vrm.get_scratch_reg(type=INT).value
+        self.mc.vxor(tmp, tmp, tmp)
+        # argloc[i] > 0:
+        # For an unsigned integer that is equivalent to argloc[i] != 0
+        if size == 1:
+            self.mc.vcmpgtubx(resloc.value, argloc.value, tmp)
+        elif size == 2:
+            self.mc.vcmpgtuhx(resloc.value, argloc.value, tmp)
+        elif size == 4:
+            self.mc.vcmpgtuwx(resloc.value, argloc.value, tmp)
+        elif size == 8:
+            self.mc.vcmpgtudx(resloc.value, argloc.value, tmp)
+        flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc)
+
+    def emit_vec_float_eq(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resloc, loc1, loc2, sizeloc = arglocs
+        size = sizeloc.value
+        tmp = regalloc.vrm.get_scratch_reg().value
+        offloc = regalloc.rm.get_scratch_reg()
+        off = offloc.value
+        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
+        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
+        if size == 4:
+            self.mc.xvcmpeqspx(tmp, loc1.value, loc2.value)
+            self.mc.stxvw4x(tmp, off, r.SP.value)
+        elif size == 8:
+            self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value)
+            self.mc.stxvd2x(tmp, off, r.SP.value)
+        else:
+            not_implemented("[ppc/assembler] float == for size %d" % size)
+        self.mc.lvx(resloc.value, off, r.SP.value)
+        flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
+
+    def emit_vec_float_xor(self, op, arglocs, regalloc):
+        resloc, l0, l1, sizeloc = arglocs
+        res = resloc.value
+        r0 = l0.value
+        r1 = l1.value
+        self.mc.xxlxor(res, r0, r1)
+
+    def emit_vec_float_ne(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resloc, loc1, loc2, sizeloc = arglocs
+        size = sizeloc.value
+        tmp = regalloc.vrm.get_scratch_reg().value
+        offloc = regalloc.rm.get_scratch_reg()
+        off = offloc.value
+        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
+        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
+        if size == 4:
+            self.mc.xvcmpeqspx(tmp, loc1.value, loc2.value)
+            self.mc.stxvw4x(tmp, off, r.SP.value)
+        elif size == 8:
+            self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value)
+            self.mc.stxvd2x(tmp, off, r.SP.value)
+        else:
+            not_implemented("float == for size %d" % size)
+        res = resloc.value
+        self.mc.lvx(res, off, r.SP.value)
+        self.mc.vnor(res, res, res) # complement
+        flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc)
+
+    def emit_vec_cast_int_to_float(self, op, arglocs, regalloc):
+        res, l0 = arglocs
+        offloc = regalloc.rm.get_scratch_reg()
+        off = offloc.value
+        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
+        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
+        self.mc.stvx(l0.value, off, r.SP.value)
+        self.mc.lxvd2x(res.value, off, r.SP.value)
+        self.mc.xvcvsxddp(res.value, res.value)
+
+    def emit_vec_int_eq(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        res, l0, l1, sizeloc = arglocs
+        size = sizeloc.value
+        if size == 1:
+            self.mc.vcmpequbx(res.value, l0.value, l1.value)
+        elif size == 2:
+            self.mc.vcmpequhx(res.value, l0.value, l1.value)
+        elif size == 4:
+            self.mc.vcmpequwx(res.value, l0.value, l1.value)
+        elif size == 8:
+            self.mc.vcmpequdx(res.value, l0.value, l1.value)
+        flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res)
+
+    def emit_vec_int_ne(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        res, l0, l1, sizeloc = arglocs
+        size = sizeloc.value
+        tmp = regalloc.vrm.get_scratch_reg(type=INT).value
+        self.mc.vxor(tmp, tmp, tmp)
+        if size == 1:
+            self.mc.vcmpequbx(res.value, res.value, tmp)
+        elif size == 2:
+            self.mc.vcmpequhx(res.value, res.value, tmp)
+        elif size == 4:
+            self.mc.vcmpequwx(res.value, res.value, tmp)
+        elif size == 8:
+            self.mc.vcmpequdx(res.value, res.value, tmp)
+        self.mc.vnor(res.value, res.value, res.value)
+        flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res)
+
+    def emit_vec_expand_f(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resloc, srcloc = arglocs
+        size = op.bytesize
+        res = resloc.value
+        if isinstance(srcloc, l.ConstFloatLoc):
+            # they are aligned!
+            assert size == 8
+            tloc = regalloc.rm.get_scratch_reg()
+            self.mc.load_imm(tloc, srcloc.value)
+            self.mc.lxvd2x(res, 0, tloc.value)
+        elif size == 8:
+            # splat the low of src to both slots in res
+            src = srcloc.value
+            self.mc.xxspltdl(res, src, src)
+        else:
+            not_implemented("vec expand in this combination not supported")
+
+    def emit_vec_expand_i(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        res, l0, off = arglocs
+        size = op.bytesize
+
+        self.mc.load_imm(r.SCRATCH2, off.value)
+        self.mc.lvx(res.value, r.SCRATCH2.value, r.SP.value)
+        if size == 1:
+            if IS_BIG_ENDIAN:
+                self.mc.vspltb(res.value, res.value, 0b0000)
+            else:
+                self.mc.vspltb(res.value, res.value, 0b1111)
+        elif size == 2:
+            if IS_BIG_ENDIAN:
+                self.mc.vsplth(res.value, res.value, 0b000)
+            else:
+                self.mc.vsplth(res.value, res.value, 0b111)
+        elif size == 4:
+            if IS_BIG_ENDIAN:
+                self.mc.vspltw(res.value, res.value, 0b00)
+            else:
+                self.mc.vspltw(res.value, res.value, 0b11)
+        elif size == 8:
+            pass
+        else:
+            not_implemented("expand int size not impl")
+
+    def emit_vec_pack_i(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs
+        srcidx = srcidxloc.value
+        residx = residxloc.value
+        count = countloc.value
+        res = resultloc.value
+        vector = vloc.value
+        src = sourceloc.value
+        size = op.bytesize
+        assert resultloc.is_vector_reg() # vector <- reg
+        self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
+        self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value)
+        idx = residx
+        if size == 8:
+            if not IS_BIG_ENDIAN:
+                idx = (16 // size) - 1 - idx
+            self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx)
+        elif size == 4:
+            for j in range(count):
+                idx = j + residx
+                if not IS_BIG_ENDIAN:
+                    idx = (16 // size) - 1 - idx
+                self.mc.stw(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+4*idx)
+        elif size == 2:
+            for j in range(count):
+                idx = j + residx
+                if not IS_BIG_ENDIAN:
+                    idx = (16 // size) - 1 - idx
+                self.mc.sth(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+2*idx)
+        elif size == 1:
+            for j in range(count):
+                idx = j + residx
+                if not IS_BIG_ENDIAN:
+                    idx = (16 // size) - 1 - idx
+                self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+idx)
+        self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+
+    def emit_vec_unpack_i(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resloc, srcloc, idxloc, countloc, sizeloc = arglocs
+        idx = idxloc.value
+        res = resloc.value
+        src = srcloc.value
+        size = sizeloc.value
+        count = countloc.value
+        if count == 1:
+            assert srcloc.is_vector_reg()
+            assert not resloc.is_vector_reg()
+            off = PARAM_SAVE_AREA_OFFSET
+            self.mc.load_imm(r.SCRATCH2, off)
+            self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
+            if not IS_BIG_ENDIAN:
+                idx = (16 // size) - 1 - idx
+            off += size * idx
+            if size == 8:
+                self.mc.load(res, r.SP.value, off)
+                return
+            elif size == 4:
+                self.mc.lwa(res, r.SP.value, off)
+                return
+            elif size == 2:
+                self.mc.lha(res, r.SP.value, off)
+                return
+            elif size == 1:
+                self.mc.lbz(res, r.SP.value, off)
+                self.mc.extsb(res, res)
+                return
+        else:
+            # count is not 1, but only 2 is supported for i32
+            # 4 for i16 and 8 for i8.
+            src = srcloc.value
+            res = resloc.value
+
+            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
+            self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
+            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
+            self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
+            if count * size == 8:
+                if not IS_BIG_ENDIAN:
+                    endian_off = 8
+                off = PARAM_SAVE_AREA_OFFSET
+                off = off + endian_off - (idx * size)
+                assert idx * size + 8 <= 16
+                self.mc.load(r.SCRATCH.value, r.SP.value, off)
+                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
+                self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+                return
+
+        not_implemented("%d bit integer, count %d" % \
+                       (size*8, count))
+
+    def emit_vec_pack_f(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resloc, vloc, srcloc, residxloc, srcidxloc, countloc = arglocs
+        vec = vloc.value
+        res = resloc.value
+        src = srcloc.value
+        count = countloc.value
+        residx = residxloc.value
+        srcidx = srcidxloc.value
+        size = op.bytesize
+        # srcloc is always a floating point register f, this means it is
+        # vsr[0] == valueof(f)
+        if srcidx == 0:
+            if residx == 0:
+                # r = (s[0], v[1])
+                self.mc.xxpermdi(res, src, vec, permi(0,1))
+            else:
+                assert residx == 1
+                # r = (v[0], s[0])
+                self.mc.xxpermdi(res, vec, src, permi(1,1))
+        else:
+            assert srcidx == 1
+            if residx == 0:
+                # r = (s[1], v[1])
+                self.mc.xxpermdi(res, src, vec, permi(1,1))
+            else:
+                assert residx == 1
+                # r = (v[0], s[1])
+                self.mc.xxpermdi(res, vec, src, permi(0,1))
+
+    def emit_vec_unpack_f(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resloc, srcloc, srcidxloc, countloc = arglocs
+        res = resloc.value
+        src = srcloc.value
+        srcidx = srcidxloc.value
+        size = op.bytesize
+        # srcloc is always a floating point register f, this means it is
+        # vsr[0] == valueof(f)
+        if srcidx == 0:
+            # r = (s[0], s[1])
+            self.mc.xxpermdi(res, src, src, permi(0,1))
+            return
+        else:
+            # r = (s[1], s[0])
+            self.mc.xxpermdi(res, src, src, permi(1,0))
+            return
+        not_implemented("unpack for combination src %d -> res %d" % (srcidx, residx))
+
+    def emit_vec_cast_float_to_int(self, op, arglocs, regalloc):
+        res, l0 = arglocs
+        offloc = regalloc.rm.get_scratch_reg()
+        v0 = regalloc.vrm.get_scratch_reg(type=INT)
+        off = offloc.value
+        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
+        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
+        self.mc.xvcvdpsxds(res.value, l0.value)
+
+    # needed as soon as PPC's support_singlefloat is implemented!
+    #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, regalloc):
+    #    self.mc.CVTPS2PD(resloc, arglocs[0])
+
+    def emit_vec_f(self, op, arglocs, regalloc):
+        pass
+    emit_vec_i = emit_vec_f
+
+class VectorRegalloc(object):
+    _mixin_ = True
+
+    def force_allocate_vector_reg(self, op):
+        forbidden_vars = self.vrm.temp_boxes
+        return self.vrm.force_allocate_reg(op, forbidden_vars)
+
+    def force_allocate_vector_reg_or_cc(self, op):
+        assert op.type == INT
+        if self.next_op_can_accept_cc(self.operations, self.rm.position):
+            # hack: return the SPP location to mean "lives in CC".  This
+            # SPP will not actually be used, and the location will be freed
+            # after the next op as usual.
+            self.rm.force_allocate_frame_reg(op)
+            return r.SPP
+        else:
+            return self.force_allocate_vector_reg(op)
+
+    def ensure_vector_reg(self, box):
+        return self.vrm.make_sure_var_in_reg(box,
+                           forbidden_vars=self.vrm.temp_boxes)
+
+    def _prepare_load(self, op):
+        descr = op.getdescr()
+        assert isinstance(descr, ArrayDescr)
+        assert not descr.is_array_of_pointers() and \
+               not descr.is_array_of_structs()
+        itemsize, ofs, _ = unpack_arraydescr(descr)
+        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
+        args = op.getarglist()
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        base_loc = self.ensure_reg(a0)
+        ofs_loc = self.ensure_reg(a1)
+        result_loc = self.force_allocate_vector_reg(op)
+        return [result_loc, base_loc, ofs_loc, imm(itemsize), imm(ofs),
+                imm(integer)]
+
+    prepare_vec_load_i = _prepare_load
+    prepare_vec_load_f = _prepare_load
+
+    def prepare_vec_arith(self, op):
+        assert isinstance(op, VectorOp)
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        size = op.bytesize
+        args = op.getarglist()
+        loc0 = self.ensure_vector_reg(a0)
+        loc1 = self.ensure_vector_reg(a1)
+        resloc = self.force_allocate_vector_reg(op)
+        return [resloc, loc0, loc1, imm(size)]
+
+    prepare_vec_int_add = prepare_vec_arith
+    prepare_vec_int_sub = prepare_vec_arith
+    prepare_vec_int_mul = prepare_vec_arith
+    prepare_vec_float_add = prepare_vec_arith
+    prepare_vec_float_sub = prepare_vec_arith
+    prepare_vec_float_mul = prepare_vec_arith
+    prepare_vec_float_truediv = prepare_vec_arith
+
+    # logic functions
+    prepare_vec_int_and = prepare_vec_arith
+    prepare_vec_int_or = prepare_vec_arith
+    prepare_vec_int_xor = prepare_vec_arith
+    prepare_vec_float_xor = prepare_vec_arith
+    del prepare_vec_arith
+
+    def prepare_vec_bool(self, op):
+        assert isinstance(op, VectorOp)
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        size = op.bytesize
+        args = op.getarglist()
+        loc0 = self.ensure_vector_reg(a0)
+        loc1 = self.ensure_vector_reg(a1)
+        resloc = self.force_allocate_vector_reg_or_cc(op)
+        return [resloc, loc0, loc1, imm(size)]
+
+    prepare_vec_float_eq = prepare_vec_bool
+    prepare_vec_float_ne = prepare_vec_bool
+    prepare_vec_int_eq = prepare_vec_bool
+    prepare_vec_int_ne = prepare_vec_bool
+    del prepare_vec_bool
+
+    def prepare_vec_store(self, op):
+        descr = op.getdescr()
+        assert isinstance(descr, ArrayDescr)
+        assert not descr.is_array_of_pointers() and \
+               not descr.is_array_of_structs()
+        itemsize, ofs, _ = unpack_arraydescr(descr)
+        a0 = op.getarg(0)
+        a1 = op.getarg(1)
+        a2 = op.getarg(2)
+        baseloc = self.ensure_reg(a0)
+        ofsloc = self.ensure_reg(a1)
+        valueloc = self.ensure_vector_reg(a2)
+
+        integer = not (descr.is_array_of_floats() or descr.getconcrete_type() == FLOAT)
+        return [baseloc, ofsloc, valueloc,
+                imm(itemsize), imm(ofs), imm(integer)]
+
+    def prepare_vec_int_signext(self, op):
+        assert isinstance(op, VectorOp)
+        a0 = op.getarg(0)
+        loc0 = self.ensure_vector_reg(a0)
+        resloc = self.force_allocate_vector_reg(op)
+        return [resloc, loc0]
+
+    def prepare_vec_arith_unary(self, op):
+        assert isinstance(op, VectorOp)
+        a0 = op.getarg(0)
+        loc0 = self.ensure_vector_reg(a0)
+        resloc = self.force_allocate_vector_reg(op)
+        sizeloc = imm(op.bytesize)
+        return [resloc, loc0, sizeloc]
+
+    prepare_vec_float_neg = prepare_vec_arith_unary
+    prepare_vec_float_abs = prepare_vec_arith_unary
+    del prepare_vec_arith_unary
+
+    def prepare_vec_pack_i(self, op):
+        # new_res = vec_pack_i(res, src, index, count)
+        assert isinstance(op, VectorOp)
+        arg = op.getarg(1)
+        index = op.getarg(2)
+        count = op.getarg(3)
+        assert isinstance(index, ConstInt)
+        assert isinstance(count, ConstInt)
+        vloc = self.ensure_vector_reg(op.getarg(0))
+        srcloc = self.ensure_reg(arg)
+        resloc = self.force_allocate_vector_reg(op)
+        residx = index.value # where to put it in result?
+        srcidx = 0
+        return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value)]
+
+    def prepare_vec_pack_f(self, op):
+        # new_res = vec_pack_f(res, src, index, count)
+        assert isinstance(op, VectorOp)
+        arg = op.getarg(1)
+        index = op.getarg(2)
+        count = op.getarg(3)
+        assert isinstance(index, ConstInt)
+        assert isinstance(count, ConstInt)
+        assert not arg.is_vector()
+        srcloc = self.ensure_reg(arg)
+        vloc = self.ensure_vector_reg(op.getarg(0))
+        if op.is_vector():
+            resloc = self.force_allocate_vector_reg(op)
+        else:
+            resloc = self.force_allocate_reg(op)
+        residx = index.value # where to put it in result?
+        srcidx = 0
+        return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value)]
+
+    def prepare_vec_unpack_f(self, op):
+        index = op.getarg(1)
+        count = op.getarg(2)
+        assert isinstance(index, ConstInt)
+        assert isinstance(count, ConstInt)
+        srcloc = self.ensure_vector_reg(op.getarg(0))
+        resloc = self.force_allocate_reg(op)
+        return [resloc, srcloc, imm(index.value), imm(count.value)]
+
+    def prepare_vec_unpack_i(self, op):
+        assert isinstance(op, VectorOp)
+        index = op.getarg(1)
+        count = op.getarg(2)
+        assert isinstance(index, ConstInt)
+        assert isinstance(count, ConstInt)
+        arg = op.getarg(0)
+        if arg.is_vector():
+            srcloc = self.ensure_vector_reg(arg)
+        else:
+            # unpack
+            srcloc = self.ensure_reg(arg0)
+        size = arg.bytesize
+        if op.is_vector():
+            resloc = self.force_allocate_vector_reg(op)
+        else:
+            resloc = self.force_allocate_reg(op)
+        return [resloc, srcloc, imm(index.value), imm(count.value), imm(size)]
+
+    def expand_float(self, size, box):
+        adr = self.assembler.datablockwrapper.malloc_aligned(16, 16)
+        fs = box.getfloatstorage()
+        rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = fs
+        rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[1] = fs
+        return l.ConstFloatLoc(adr)
+
+    def prepare_vec_expand_f(self, op):
+        assert isinstance(op, VectorOp)
+        arg = op.getarg(0)
+        if arg.is_constant():
+            l0 = self.expand_float(op.bytesize, arg)
+            res = self.force_allocate_vector_reg(op)
+        else:
+            l0 = self.ensure_reg(arg)
+            res = self.force_allocate_vector_reg(op)
+        return [res, l0]
+
+    def prepare_vec_expand_i(self, op):
+        assert isinstance(op, VectorOp)
+        arg = op.getarg(0)
+        mc = self.assembler.mc
+        if arg.is_constant():
+            assert isinstance(arg, ConstInt)
+            l0 = self.rm.get_scratch_reg()
+            mc.load_imm(l0, arg.value)
+        else:
+            l0 = self.ensure_reg(arg)
+        mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET)
+        size = op.bytesize
+        if size == 8:
+            mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
+        res = self.force_allocate_vector_reg(op)
+        return [res, l0, imm(PARAM_SAVE_AREA_OFFSET)]
+
+    def prepare_vec_int_is_true(self, op):
+        assert isinstance(op, VectorOp)
+        arg = op.getarg(0)
+        assert isinstance(arg, VectorOp)
+        argloc = self.ensure_vector_reg(arg)
+        resloc = self.force_allocate_vector_reg_or_cc(op)
+        return [resloc, argloc, imm(arg.bytesize)]
+
+    def _prepare_vec(self, op):
+        # pseudo instruction, needed to allocate a register for a new variable
+        return [self.force_allocate_vector_reg(op)]
+
+    prepare_vec_i = _prepare_vec
+    prepare_vec_f = _prepare_vec
+
+    def prepare_vec_cast_float_to_int(self, op):
+        l0 = self.ensure_vector_reg(op.getarg(0))
+        res = self.force_allocate_vector_reg(op)
+        return [res, l0]
+
+    prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int
+
+    def prepare_vec_guard_true(self, op):
+        self.assembler.guard_success_cc = c.VEQ
+        return self._prepare_guard(op)
+
+    def prepare_vec_guard_false(self, op):
+        self.assembler.guard_success_cc = c.VNE

From pypy.commits at gmail.com  Tue Sep 13 10:09:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 13 Sep 2016 07:09:51 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: more changes to get first
 tests to fail at assembly
Message-ID: <57d808af.4756c20a.dc392.903b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87071:15d0e9941274
Date: 2016-09-13 12:40 +0200
http://bitbucket.org/pypy/pypy/changeset/15d0e9941274/

Log:	more changes to get first tests to fail at assembly

diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py
--- a/rpython/jit/backend/zarch/assembler.py
+++ b/rpython/jit/backend/zarch/assembler.py
@@ -33,11 +33,13 @@
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref
 from rpython.rlib.jit import AsmInfo
 from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.jit.backend.zarch import vector_ext
 
 class JitFrameTooDeep(Exception):
     pass
 
-class AssemblerZARCH(BaseAssembler, OpAssembler):
+class AssemblerZARCH(BaseAssembler, OpAssembler,
+                     vector_ext.VectorAssembler):
 
     def __init__(self, cpu, translate_support_code=False):
         BaseAssembler.__init__(self, cpu, translate_support_code)
diff --git a/rpython/jit/backend/zarch/runner.py b/rpython/jit/backend/zarch/runner.py
--- a/rpython/jit/backend/zarch/runner.py
+++ b/rpython/jit/backend/zarch/runner.py
@@ -2,6 +2,7 @@
 from rpython.jit.backend.zarch import registers as r
 from rpython.jit.backend.zarch.assembler import AssemblerZARCH
 from rpython.jit.backend.zarch.codebuilder import InstrBuilder
+from rpython.jit.backend.zarch import vector_ext
 from rpython.rlib import rgc
 from rpython.rtyper.lltypesystem import lltype, llmemory
 
@@ -16,7 +17,7 @@
     supports_floats = True
     from rpython.jit.backend.zarch.registers import JITFRAME_FIXED_SIZE
 
-    vector_ext = SIMDZVectorExt()
+    vector_ext = vector_ext.ZSIMDVectorExt()
 
     backend_name = 'zarch'
 
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -10,17 +10,14 @@
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype
-from rpython.jit.backend.ppc.locations import imm, RegisterLocation
-from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN
 from rpython.jit.backend.llsupport.vector_ext import VectorExt
-from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET
-import rpython.jit.backend.ppc.register as r
-import rpython.jit.backend.ppc.condition as c
-import rpython.jit.backend.ppc.locations as l
+from rpython.jit.backend.zarch.detect_feature import detect_simd_z
+import rpython.jit.backend.zarch.registers as r
+import rpython.jit.backend.zarch.conditions as c
+import rpython.jit.backend.zarch.locations as l
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.jit.codewriter import longlong
-from rpython.jit.backend.ppc.detect_feature import detect_vsx
 from rpython.rlib.objectmodel import always_inline
 
 def not_implemented(msg):
@@ -64,6 +61,7 @@
     #        asm.mc.load_imm(tloc, asm.VEC_DOUBLE_WORD_ONES)
     #        asm.mc.lvx(ones, 0, tloc.value)
     #    asm.mc.vsel(resval, zeros, ones, resval)
+    pass
 
 class ZSIMDVectorExt(VectorExt):
     def setup_once(self, asm):
@@ -91,6 +89,7 @@
         # TODO for i in range(len(data)):
         # TODO     addr[i] = data[i]
         # TODO self.VEC_DOUBLE_WORD_ONES = mem
+        pass
 
     def emit_vec_load_f(self, op, arglocs, regalloc):
         resloc, baseloc, indexloc, size_loc, ofs, integer_loc = arglocs
@@ -314,7 +313,7 @@
             self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value)
             self.mc.stxvd2x(tmp, off, r.SP.value)
         else:
-            not_implemented("[ppc/assembler] float == for size %d" % size)
+            not_implemented("[zarch/assembler] float == for size %d" % size)
         self.mc.lvx(resloc.value, off, r.SP.value)
         flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
 
@@ -583,10 +582,6 @@
         self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
         self.mc.xvcvdpsxds(res.value, l0.value)
 
-    # needed as soon as PPC's support_singlefloat is implemented!
-    #def genop_vec_cast_singlefloat_to_float(self, op, arglocs, regalloc):
-    #    self.mc.CVTPS2PD(resloc, arglocs[0])
-
     def emit_vec_f(self, op, arglocs, regalloc):
         pass
     emit_vec_i = emit_vec_f

From pypy.commits at gmail.com  Tue Sep 13 10:09:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 13 Sep 2016 07:09:49 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: add runtime check to detect
 vector facility
Message-ID: <57d808ad.262ec20a.37090.c62a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87070:13d4d6bd1c2f
Date: 2016-09-13 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/13d4d6bd1c2f/

Log:	add runtime check to detect vector facility

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -20,7 +20,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.jit.codewriter import longlong
-from rpython.jit.backend.ppc.detect_feature import detect_vsx
+from rpython.jit.backend.zarch.detect_feature import detect_simd_z
 from rpython.rlib.objectmodel import always_inline
 
 def not_implemented(msg):
@@ -85,7 +85,7 @@
 
 class AltiVectorExt(VectorExt):
     def setup_once(self, asm):
-        if detect_vsx():
+        if detect_simd_z():
             self.enable(16, accum=True)
             asm.setup_once_vector()
         self._setup = True
diff --git a/rpython/jit/backend/zarch/runner.py b/rpython/jit/backend/zarch/runner.py
--- a/rpython/jit/backend/zarch/runner.py
+++ b/rpython/jit/backend/zarch/runner.py
@@ -16,6 +16,8 @@
     supports_floats = True
     from rpython.jit.backend.zarch.registers import JITFRAME_FIXED_SIZE
 
+    vector_ext = SIMDZVectorExt()
+
     backend_name = 'zarch'
 
     IS_64_BIT = True
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -182,10 +182,6 @@
         elif itemsize == 8:
             self.mc.xvdivdp(resloc.value, loc0.value, loc1.value)
 
-    def emit_vec_int_mul(self, op, arglocs, regalloc):
-        raise NotImplementedError
-        pass # TODO
-
     def emit_vec_int_and(self, op, arglocs, regalloc):
         resloc, loc0, loc1, sizeloc = arglocs
         self.mc.vand(resloc.value, loc0.value, loc1.value)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -553,7 +553,7 @@
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function',
     'vec': 'turn on the vectorization optimization (vecopt). ' \
-           'Supports powerpc (SVX), x86 (SSE 4.1)',
+           'Supports x86 (SSE 4.1), powerpc (SVX), s390x',
     'vec_all': 'try to vectorize trace loops that occur outside of the numpy library.',
     'vec_cost': 'threshold for which traces to bail. 0 means the costs.',
     'vec_length': 'the amount of instructions allowed in "all" traces.',
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -1,7 +1,6 @@
 import re
 
 def extract_s390x_cpu_ids(lines):
-    """ NOT_RPYTHON """
     ids = []
 
     re_number = re.compile("processor (\d+):")
@@ -41,7 +40,6 @@
 
 
 def s390x_cpu_revision():
-    """ NOT_RPYTHON """
     # linux kernel does the same classification
     # http://lists.llvm.org/pipermail/llvm-commits/Week-of-Mon-20131028/193311.html
 

From pypy.commits at gmail.com  Tue Sep 13 10:09:55 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 13 Sep 2016 07:09:55 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: add new files to detect
 features on zarch + basic vector test
Message-ID: <57d808b3.e17cc20a.217c1.bdba@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87073:c0a1a9742edd
Date: 2016-09-13 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/c0a1a9742edd/

Log:	add new files to detect features on zarch + basic vector test

diff --git a/rpython/jit/backend/zarch/detect_feature.py b/rpython/jit/backend/zarch/detect_feature.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/zarch/detect_feature.py
@@ -0,0 +1,25 @@
+import sys
+import struct
+import platform
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
+from rpython.rlib.rmmap import alloc, free
+from rpython.rlib.rstruct.runpack import runpack
+from rpython.translator.platform.arch.s390x import s390x_cpu_revision
+
+SYSTEM = platform.system()
+
+def detect_simd_z_linux():
+    return False
+
+def detect_simd_z():
+    if SYSTEM == 'Linux':
+        machine = s390x_cpu_revision()
+        if machine == "z13":
+            return True
+    return False
+
+if __name__ == '__main__':
+    print 'The following extensions are supported:'
+    if detect_simd_z():
+        print '  - SIMD Z'
diff --git a/rpython/jit/backend/zarch/test/test_zarchvector.py b/rpython/jit/backend/zarch/test/test_zarchvector.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/zarch/test/test_zarchvector.py
@@ -0,0 +1,26 @@
+import py
+from rpython.jit.backend.zarch.test import test_basic
+from rpython.jit.metainterp.test import test_vector
+from rpython.jit.backend.zarch.detect_feature import detect_simd_z
+
+
+class TestBasic(test_basic.JitZARCHMixin, test_vector.VectorizeTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_basic.py
+    def setup_method(self, method):
+        clazz = self.CPUClass
+        def init(*args, **kwargs):
+            cpu = clazz(*args, **kwargs)
+            # > 95% can be executed, thus let's cheat here a little
+            cpu.supports_guard_gc_type = True
+            return cpu
+        self.CPUClass = init
+
+    def supports_vector_ext(self):
+        return detect_simd_z()
+
+    def test_list_vectorize(self):
+        pass # needs support_guard_gc_type, disable for now
+
+    enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
+

From pypy.commits at gmail.com  Tue Sep 13 10:09:53 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 13 Sep 2016 07:09:53 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: do not generate a new
 operation for index calc if it can be expressed as a single operation
Message-ID: <57d808b1.89d11c0a.f7fea.dfee@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87072:1ea1c6504c10
Date: 2016-09-13 15:56 +0200
http://bitbucket.org/pypy/pypy/changeset/1ea1c6504c10/

Log:	do not generate a new operation for index calc if it can be
	expressed as a single operation

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -993,6 +993,23 @@
         self.next_nonconst = None
         self.current_end = None
 
+    def calculated_by(self, op):
+        # quick check to indicate if this operation is not directly expressable using
+        # the operation in the op parameter.
+        if op.getopnum() == rop.INT_ADD:
+            a0 = op.getarg(0)
+            a1 = op.getarg(1)
+            if a0 is self.var and a1.is_constant() and a1.getint() == self.constant:
+                return True
+            if a1 is self.var and a0.is_constant() and a0.getint() == self.constant:
+                return True
+        if op.getopnum() == rop.INT_SUB:
+            a0 = op.getarg(0)
+            a1 = op.getarg(1)
+            if a0 is self.var and a1.is_constant() and a1.getint() == self.constant:
+                return True
+        return False
+
     def stride_const(self):
         return self.next_nonconst is None
 
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -49,7 +49,8 @@
             if arg.is_constant() or arg.is_inputarg():
                 continue
             if arg not in self.seen:
-                needs_resolving[arg] = None
+                box = self.renamer.rename_box(arg)
+                needs_resolving[box] = None
 
         indexvars = self.graph.index_vars
         i = len(delayed)-1
@@ -60,18 +61,32 @@
                 # either it is a normal operation, or we know that there is a linear combination
                 del needs_resolving[op]
                 if op in indexvars:
-                    indexvar = indexvars[op]
-                    last = None
-                    for operation in indexvar.get_operations():
-                        self.append_to_oplist(operation)
-                        last = operation
-                    if last:
-                        indexvars[last] = indexvar
-                        self.renamer.start_renaming(op, last)
+                    opindexvar = indexvars[op]
+                    # there might be a variable already, that
+                    # calculated the index variable, thus just reuse it
+                    for var, indexvar in indexvars.items(): 
+                        if indexvar == opindexvar and var in self.seen:
+                            self.renamer.start_renaming(op, var)
+                            break
+                    else:
+                        if opindexvar.calculated_by(op):
+                            # just append this operation
+                            self.seen[op] = None
+                            self.append_to_oplist(op)
+                        else:
+                            # here is an easier way to calculate just this operation
+                            last = op
+                            for operation in opindexvar.get_operations():
+                                self.append_to_oplist(operation)
+                                last = operation
+                            indexvars[last] = opindexvar
+                            self.renamer.start_renaming(op, last)
+                            self.seen[op] = None
+                            self.seen[last] = None
                 else: 
                     self.resolve_delayed(needs_resolving, delayed, op)
                     self.append_to_oplist(op)
-                self.seen[op] = None
+                    self.seen[op] = None
                 if len(delayed) > i:
                     del delayed[i]
             i -= 1
@@ -84,14 +99,20 @@
         self.renamer.rename(op)
         self.oplist.append(op)
 
+    def schedule(self):
+        self.prepare()
+        Scheduler().walk_and_emit(self)
+        self.post_schedule()
+
     def post_schedule(self):
         loop = self.graph.loop
+        jump = loop.jump
         if self.delayed:
             # some operations can be delayed until the jump instruction,
             # handle them here
-            self.resolve_delayed({}, self.delayed, loop.jump)
+            self.resolve_delayed({}, self.delayed, jump)
+        self.renamer.rename(jump)
         loop.operations = self.oplist
-        self.renamer.rename(loop.jump)
 
     def profitable(self):
         return True
@@ -120,8 +141,10 @@
         delayed.append(node)
         node.delayed = None
         provides = node.provides()
+        op = node.getoperation()
         if len(provides) == 0:
-            self.delayed.append(node)
+            for n in delayed:
+                self.delayed.append(n)
         else:
             for to in node.provides():
                 tnode = to.target_node()
@@ -187,13 +210,14 @@
     def post_emit(self, node):
         pass
 
-    def pre_emit(self, node, pack_first=True):
-        delayed = node.delayed
+    def pre_emit(self, orignode, pack_first=True):
+        delayed = orignode.delayed
         if delayed:
             # there are some nodes that have been delayed just for this operation
             if pack_first:
-                op = node.getoperation()
+                op = orignode.getoperation()
                 self.resolve_delayed({}, delayed, op)
+
             for node in delayed:
                 op = node.getoperation()
                 if op in self.seen:
@@ -208,7 +232,7 @@
                         for to in node.provides():
                             tnode = to.target_node()
                             self.delegate_delay(tnode, [node])
-                        node.delayed = None
+            orignode.delayed = None
 
 class Scheduler(object):
     """ Create an instance of this class to (re)schedule a vector trace. """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -5,7 +5,7 @@
 from rpython.jit.metainterp.optimizeopt.renamer import Renamer
 from rpython.jit.metainterp.optimizeopt.vector import (VecScheduleState,
         Pack, Pair, NotAProfitableLoop, VectorizingOptimizer, GenericCostModel,
-        PackSet)
+        PackSet, SchedulerState)
 from rpython.jit.backend.llsupport.vector_ext import VectorExt
 from rpython.jit.metainterp.optimizeopt.dependency import Node, DependencyGraph
 from rpython.jit.metainterp.optimizeopt.schedule import Scheduler
@@ -460,5 +460,18 @@
         state.expand(['d','d','c'], 'ddc')
         assert state.find_expanded(['d','d','c']) == 'ddc'
 
-
-
+    def test_delayed_schedule(self):
+        loop = self.parse("""
+        [i0]
+        i1 = int_add(i0,1)
+        i2 = int_add(i0,1)
+        jump(i2)
+        """)
+        loop.prefix_label = None
+        loop.label = ResOperation(rop.LABEL, loop.inputargs)
+        ops = loop.operations
+        loop.operations = ops[:-1]
+        loop.jump = ops[-1]
+        state = SchedulerState(self.cpu, DependencyGraph(loop))
+        state.schedule()
+        assert len(loop.operations) == 1
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -262,11 +262,8 @@
 
         # re-schedule the trace -> removes many pure operations
         graph = DependencyGraph(loop)
-        costmodel = GenericCostModel(self.cpu, self.cost_threshold)
         state = SchedulerState(self.cpu, graph)
-        state.prepare()
-        Scheduler().walk_and_emit(state)
-        state.post_schedule()
+        state.schedule()
 
         return loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
 

From pypy.commits at gmail.com  Tue Sep 13 10:19:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 07:19:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: Write a note
Message-ID: <57d80b07.07c41c0a.297ec.c372@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87074:b2836ccb22d8
Date: 2016-09-13 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/b2836ccb22d8/

Log:	Write a note

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -332,6 +332,9 @@
         expected_class = llmemory.cast_adr_to_ptr(
             llmemory.cast_int_to_adr(vtable2),
             rclass.CLASSTYPE)
+        # note: the test is for a range including 'max', but 'max'
+        # should never be used for actual classes.  Including it makes
+        # it easier to pass artificial tests.
         if (expected_class.subclassrange_min
                 <= known_class.subclassrange_min
                 <= expected_class.subclassrange_max):

From pypy.commits at gmail.com  Tue Sep 13 10:29:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 07:29:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: We can't really do anything without the
 '_struct' module
Message-ID: <57d80d3d.861b1c0a.fd47c.fdfc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87075:82928531a68e
Date: 2016-09-13 16:21 +0200
http://bitbucket.org/pypy/pypy/changeset/82928531a68e/

Log:	We can't really do anything without the '_struct' module

diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -81,6 +81,7 @@
         error.RECORD_INTERPLEVEL_TRACEBACK = True
     # --allworkingmodules takes really long to start up, but can be forced on
     config.objspace.suggest(allworkingmodules=False)
+    config.objspace.usemodules.struct = True
     if config.objspace.allworkingmodules:
         pypyoption.enable_allworkingmodules(config)
     if config.objspace.usemodules._continuation:

From pypy.commits at gmail.com  Tue Sep 13 11:16:43 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 13 Sep 2016 08:16:43 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Ensure make_inputargs fails
 properly when given arguments with type information
Message-ID: <57d8185b.121a1c0a.ef65f.11cd@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r87076:3c1c1904097a
Date: 2016-09-13 11:15 -0400
http://bitbucket.org/pypy/pypy/changeset/3c1c1904097a/

Log:	Ensure make_inputargs fails properly when given arguments with type
	information which conflicts with the VirtualState

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
 from rpython.jit.metainterp.executor import execute
 from rpython.jit.metainterp.optimize import InvalidLoop
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -11,6 +11,8 @@
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.debug import debug_print
+from rpython.rtyper import rclass
+from rpython.rtyper.lltypesystem import llmemory
 from rpython.jit.metainterp.optimize import SpeculativeError
 
 
@@ -799,6 +801,21 @@
         if not (0 <= index < arraylength):
             raise SpeculativeError
 
+    @staticmethod
+    def _check_subclass(vtable1, vtable2): # checks that vtable1 is a subclass of vtable2
+        known_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable1),
+            rclass.CLASSTYPE)
+        expected_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable2),
+            rclass.CLASSTYPE)
+        # note: the test is for a range including 'max', but 'max'
+        # should never be used for actual classes.  Including it makes
+        # it easier to pass artificial tests.
+        return (expected_class.subclassrange_min
+                <= known_class.subclassrange_min
+                <= expected_class.subclassrange_max)
+
     def is_virtual(self, op):
         if op.type == 'r':
             opinfo = self.getptrinfo(op)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -324,37 +324,24 @@
             return
         self.emit_operation(op)
 
-    def _check_subclass(self, vtable1, vtable2):
-        # checks that vtable1 is a subclass of vtable2
-        known_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable1),
-            rclass.CLASSTYPE)
-        expected_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable2),
-            rclass.CLASSTYPE)
-        if (expected_class.subclassrange_min
-                <= known_class.subclassrange_min
-                <= expected_class.subclassrange_max):
-            return True
-        return False
-
     def optimize_GUARD_SUBCLASS(self, op):
         info = self.getptrinfo(op.getarg(0))
+        optimizer = self.optimizer
         if info and info.is_constant():
             c = self.get_box_replacement(op.getarg(0))
-            vtable = self.optimizer.cpu.ts.cls_of_box(c).getint()
-            if self._check_subclass(vtable, op.getarg(1).getint()):
+            vtable = optimizer.cpu.ts.cls_of_box(c).getint()
+            if optimizer._check_subclass(vtable, op.getarg(1).getint()):
                 return
             raise InvalidLoop("GUARD_SUBCLASS(const) proven to always fail")
         if info is not None and info.is_about_object():
-            known_class = info.get_known_class(self.optimizer.cpu)
+            known_class = info.get_known_class(optimizer.cpu)
             if known_class:
-                if self._check_subclass(known_class.getint(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(known_class.getint(),
+                                             op.getarg(1).getint()):
                     return
             elif info.get_descr() is not None:
-                if self._check_subclass(info.get_descr().get_vtable(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(info.get_descr().get_vtable(),
+                                             op.getarg(1).getint()):
                     return
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -17,12 +17,13 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
      VirtualStateInfo
-from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.optimizeopt import info, optimizer
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.tool import oparser
 
 class FakeOptimizer(object):
     optearlyforce = None
+    optimizer = optimizer.Optimizer
 
     class cpu:
         remove_gctypeptr = True
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -102,6 +102,8 @@
     node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2)
     node_vtable3 = lltype.malloc(OBJECT_VTABLE, immortal=True)
     node_vtable3.name = rclass.alloc_array_name('node3')
+    node_vtable3.subclassrange_min = 3
+    node_vtable3.subclassrange_max = 3
     node_vtable_adr3 = llmemory.cast_ptr_to_adr(node_vtable3)
     cpu = runner.LLGraphCPU(None)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -3,7 +3,8 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
      VArrayStateInfo, not_virtual, VirtualState,\
-     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
+     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo,\
+     VirtualStateConstructor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
      InputArgFloat
@@ -26,6 +27,7 @@
     def __init__(self, cpu):
         self.cpu = cpu
         self.optearlyforce = None
+        self.optimizer = Optimizer
 
 class BaseTestGenerateGuards(BaseTest):
     def setup_class(self):
@@ -87,6 +89,33 @@
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == []
 
+    def test_make_inputargs_2(self):
+        # Ensure that make_inputargs properly errors with VirtualStatesCantMatch
+        # when the type information for a virtual field conflicts. In practice the
+        # expected and given field always share a common subclass.
+        # This check is needed as not all paths to make_inputargs in unroll.py
+        # are guarded by a call to generate_guards.
+        optimizer = FakeOptimizer(self.cpu)
+        classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        innervalue1 = info.InstancePtrInfo(known_class=classbox1, is_virtual=True, descr=self.valuedescr.get_parent_descr())
+        for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
+            innervalue1.setfield(field, None, ConstInt(42))
+        classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3))
+        innervalue2 = info.InstancePtrInfo(known_class=classbox2, is_virtual=True, descr=self.valuedescr3.get_parent_descr())
+        for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
+            innervalue2.setfield(field, None, ConstInt(42))
+
+        nodebox1 = InputArgRef(self.nodeaddr)
+        nodebox2 = InputArgRef(self.myptr3)
+        nodebox1.set_forwarded(innervalue1)
+        nodebox2.set_forwarded(innervalue2)
+
+        constr = VirtualStateConstructor(optimizer)
+        vs = constr.get_virtual_state([nodebox1])
+
+        with py.test.raises(VirtualStatesCantMatch):
+            args = vs.make_inputargs([nodebox2], optimizer, force_boxes=True)
+
     def test_position_generalization(self):
         def postest(info1, info2):
             info1.position = 0
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -177,6 +177,14 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
+    @staticmethod
+    def descr_issubclass(descr1, descr2, optimizer):
+        if not descr1.is_object() or not descr2.is_object():
+            return True
+        vtable1 = descr1.get_vtable()
+        vtable2 = descr2.get_vtable()
+        return optimizer._check_subclass(vtable1, vtable2)
+
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
@@ -184,10 +192,14 @@
             raise VirtualStatesCantMatch()
         else:
             assert isinstance(info, AbstractStructPtrInfo)
+
         for i in range(len(self.fielddescrs)):
             state = self.fieldstate[i]
+            descr = self.fielddescrs[i].get_parent_descr()
             if not state:
                 continue
+            if not self.descr_issubclass(info.descr, descr, optimizer.optimizer):
+                raise VirtualStatesCantMatch()
             if state.position > self.position:
                 fieldbox = info._fields[i]
                 state.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)

From pypy.commits at gmail.com  Tue Sep 13 11:34:07 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 13 Sep 2016 08:34:07 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Extract make_source() function from
 .import_module()
Message-ID: <57d81c6f.d8941c0a.8dbea.1878@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87077:ac65b27f031b
Date: 2016-09-12 16:10 +0100
http://bitbucket.org/pypy/pypy/changeset/ac65b27f031b/

Log:	Extract make_source() function from .import_module()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,7 +18,7 @@
 
 from .support import c_compile
 
-only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
@@ -161,6 +161,32 @@
     """ % ('\n'.join(methods_table),)
     return body
 
+def make_source(name, init, body, PY_SSIZE_T_CLEAN):
+    code = """
+    %(PY_SSIZE_T_CLEAN)s
+    #include <Python.h>
+    /* fix for cpython 2.7 Python.h if running tests with -A
+        since pypy compiles with -fvisibility-hidden */
+    #undef PyMODINIT_FUNC
+    #ifdef __GNUC__
+    #  define RPY_EXPORTED extern __attribute__((visibility("default")))
+    #else
+    #  define RPY_EXPORTED extern __declspec(dllexport)
+    #endif
+    #define PyMODINIT_FUNC RPY_EXPORTED void
+
+    %(body)s
+
+    PyMODINIT_FUNC
+    init%(name)s(void) {
+    %(init)s
+    }
+    """ % dict(name=name, init=init, body=body,
+            PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
+                if PY_SSIZE_T_CLEAN else '')
+    return code
+
+
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
     return #ZZZ
@@ -364,7 +390,7 @@
             init specifies the overall template of the module.
 
             if init is None, the module source will be loaded from a file in this
-            test direcory, give a name given by the filename parameter.
+            test directory, give a name given by the filename parameter.
 
             if filename is None, the module name will be used to construct the
             filename.
@@ -374,28 +400,7 @@
             else:
                 include_dirs = [space.str_w(s) for s in space.listview(w_include_dirs)]
             if init is not None:
-                code = """
-                %(PY_SSIZE_T_CLEAN)s
-                #include <Python.h>
-                /* fix for cpython 2.7 Python.h if running tests with -A
-                   since pypy compiles with -fvisibility-hidden */
-                #undef PyMODINIT_FUNC
-                #ifdef __GNUC__
-                #  define RPY_EXPORTED extern __attribute__((visibility("default")))
-                #else
-                #  define RPY_EXPORTED extern __declspec(dllexport)
-                #endif
-                #define PyMODINIT_FUNC RPY_EXPORTED void
-
-                %(body)s
-
-                PyMODINIT_FUNC
-                init%(name)s(void) {
-                %(init)s
-                }
-                """ % dict(name=name, init=init, body=body,
-                           PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
-                                            if PY_SSIZE_T_CLEAN else '')
+                code = make_source(name, init, body, PY_SSIZE_T_CLEAN)
                 kwds = dict(source_strings=[code])
             else:
                 assert not PY_SSIZE_T_CLEAN

From pypy.commits at gmail.com  Tue Sep 13 11:34:09 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 13 Sep 2016 08:34:09 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Move implementation of load_module
 to SystemCompilationInfo subclasses
Message-ID: <57d81c71.0d8d1c0a.df2d3.32b6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87078:c88d32f9692f
Date: 2016-09-13 16:33 +0100
http://bitbucket.org/pypy/pypy/changeset/c88d32f9692f/

Log:	Move implementation of load_module to SystemCompilationInfo
	subclasses

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -82,6 +82,25 @@
         soname.rename(pydname)
         return str(pydname)
 
+class ExtensionCompiler(SystemCompilationInfo):
+    """Extension compiler for appdirect mode"""
+    def load_module(space, mod, name):
+        import imp
+        return imp.load_dynamic(name, mod)
+
+class SpaceCompiler(SystemCompilationInfo):
+    """Extension compiler for regular (untranslated PyPy) mode"""
+    def __init__(self, space, *args, **kwargs):
+        self.space = space
+        SystemCompilationInfo.__init__(self, *args, **kwargs)
+
+    def load_module(self, mod, name):
+        space = self.space
+        api.load_extension_module(space, mod, name)
+        return space.getitem(
+            space.sys.get('modules'), space.wrap(name))
+
+
 def get_cpyext_info(space):
     from pypy.module.imp.importing import get_so_extension
     state = space.fromcache(State)
@@ -102,7 +121,7 @@
             link_extra = ["-g"]
         else:
             compile_extra = link_extra = None
-    return SystemCompilationInfo(
+    return SpaceCompiler(space,
         include_extra=api.include_dirs,
         compile_extra=compile_extra,
         link_extra=link_extra,
@@ -132,7 +151,7 @@
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
         link_extra = None
     ext = get_so_suffix()
-    return SystemCompilationInfo(
+    return ExtensionCompiler(
         include_extra=[get_python_inc()],
         compile_extra=compile_extra,
         link_extra=link_extra,
@@ -411,7 +430,7 @@
                 kwds = dict(source_files=[filename])
             mod = self.sys_info.compile_extension_module(
                 name, include_dirs=include_dirs, **kwds)
-            w_result = load_module(space, mod, name)
+            w_result = self.sys_info.load_module(mod, name)
             if not self.runappdirect:
                 self.record_imported_module(name)
             return w_result
@@ -419,13 +438,7 @@
 
         @gateway.unwrap_spec(mod=str, name=str)
         def load_module(space, mod, name):
-            if self.runappdirect:
-                import imp
-                return imp.load_dynamic(name, mod)
-            else:
-                api.load_extension_module(space, mod, name)
-                return space.getitem(
-                    space.sys.get('modules'), space.wrap(name))
+            return self.sys_info.load_module(mod, name)
 
         @gateway.unwrap_spec(modname=str, prologue=str,
                              more_init=str, PY_SSIZE_T_CLEAN=bool)

From pypy.commits at gmail.com  Tue Sep 13 11:38:54 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 13 Sep 2016 08:38:54 -0700 (PDT)
Subject: [pypy-commit] pypy force-virtual-state: Merge with default
Message-ID: <57d81d8e.e17cc20a.217c1.e23c@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r87079:5d49fd764471
Date: 2016-09-13 11:38 -0400
http://bitbucket.org/pypy/pypy/changeset/5d49fd764471/

Log:	Merge with default

diff too long, truncating to 2000 out of 3169 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -31,3 +31,5 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.1
+Version: 1.8.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.1"
-__version_info__ = (1, 8, 1)
+__version__ = "1.8.2"
+__version_info__ = (1, 8, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.1"
+                               "\ncompiled with cffi version: 1.8.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
--- a/pypy/doc/release-pypy2.7-v5.4.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -5,26 +5,29 @@
 We have released a bugfix for PyPy2.7-v5.4.0, released last week,
 due to the following issues:
 
-  * Update list of contributors in documentation and LICENSE file, 
-    this was unfortunately left out of 5.4.0. My apoligies to the new
+  * Update list of contributors in documentation and LICENSE file,
+    this was unfortunately left out of 5.4.0. My apologies to the new
     contributors
 
-  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+  * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a
     dynamically loadable file
 
-  * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+  * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython
 
-  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+  * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
 
-  * Fix for an issue where os.access() accepted a float for mode
+  * Fix for an issue for translating the sandbox
 
-  * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+  * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
     the last byte of a unicode string sometimes
 
-  * Update built-in cffi_ to the soon-to-be-released 1.8.1 version
+  * Update built-in cffi_ to version 1.8.1
 
   * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
-    translation with a message asking for help porting it 
+    translation with a message asking for help porting it
+
+  * Fix a regression where a PyBytesObject was forced (converted to a RPython
+    object) when not required, reported as issue #2395
 
 Thanks to those who reported the issues.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,12 @@
 
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1428,6 +1428,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.1"
+VERSION = "1.8.2"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
                 return True
         return False
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        pass
-
     def convert_from_object(self, cdata, w_ob):
         if not self._copy_from_same(cdata, w_ob):
             self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
     )
     def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self.force_lazy_struct()
-        self._check_only_one_argument_for_union(w_ob)
 
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
-            if len(lst_w) > len(self._fields_list):
-                raise oefmt(space.w_ValueError,
-                            "too many initializers for '%s' (got %d)",
-                            self.name, len(lst_w))
-            for i in range(len(lst_w)):
-                optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+            j = 0
+            for w_obj in lst_w:
+                try:
+                    while (self._fields_list[j].flags &
+                               W_CField.BF_IGNORE_IN_CTOR):
+                        j += 1
+                except IndexError:
+                    raise oefmt(space.w_ValueError,
+                                "too many initializers for '%s' (got %d)",
+                                self.name, len(lst_w))
+                optvarsize = self._fields_list[j].write_v(cdata, w_obj,
                                                           optvarsize)
+                j += 1
             return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
 class W_CTypeUnion(W_CTypeStructOrUnion):
     kind = "union"
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        space = self.space
-        n = space.int_w(space.len(w_ob))
-        if n > 1:
-            raise oefmt(space.w_ValueError,
-                        "initializer for '%s': %d items given, but only one "
-                        "supported (use a dict if needed)", self.name, n)
-
 
 class W_CField(W_Root):
     _immutable_ = True
@@ -200,18 +194,21 @@
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
 
-    def __init__(self, ctype, offset, bitshift, bitsize):
+    BF_IGNORE_IN_CTOR = 0x01
+
+    def __init__(self, ctype, offset, bitshift, bitsize, flags):
         self.ctype = ctype
         self.offset = offset
         self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
         self.bitsize = bitsize
+        self.flags = flags       # BF_xxx
 
     def is_bitfield(self):
         return self.bitshift >= 0
 
-    def make_shifted(self, offset):
+    def make_shifted(self, offset, fflags):
         return W_CField(self.ctype, offset + self.offset,
-                        self.bitshift, self.bitsize)
+                        self.bitshift, self.bitsize, self.flags | fflags)
 
     def read(self, cdata):
         cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
     offset = interp_attrproperty('offset', W_CField),
     bitshift = interp_attrproperty('bitshift', W_CField),
     bitsize = interp_attrproperty('bitsize', W_CField),
+    flags = interp_attrproperty('flags', W_CField),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
         if alignment < falign and do_align:
             alignment = falign
         #
+        if is_union and i > 0:
+            fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+        else:
+            fflags = 0
+        #
         if fbitsize < 0:
             # not a bitfield: common case
 
@@ -372,7 +377,7 @@
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8)
+                    fld = srcfld.make_shifted(boffset // 8, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                                           fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
@@ -489,7 +495,7 @@
                     bitshift = 8 * ftype.size - fbitsize- bitshift
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
-                                           bitshift, fbitsize)
+                                           bitshift, fbitsize, fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -450,7 +450,12 @@
                 # For compatibility
                 assert exc.value.errno == _ssl.SSL_ERROR_WANT_READ
             finally:
-                c.shutdown()
+                try:
+                    c.shutdown()
+                except _ssl.SSLError:
+                    # If the expected exception was raised, the SSLContext
+                    # can't be shut down yet
+                    pass
         finally:
             s.close()
 
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -597,6 +597,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,10 +119,10 @@
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,6 +645,9 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,7 +1,6 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
@@ -13,33 +12,4 @@
         return 1
     return 0  
 
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
-
- at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
-    """Return 1 if the memory defined by the view is C-style (fortran is
-    'C') or Fortran-style (fortran is 'F') contiguous or either one
-    (fortran is 'A').  Return 0 otherwise."""
-    # PyPy only supports contiguous Py_buffers for now.
-    return 1
+    
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -14,45 +14,33 @@
 ## Implementation of PyBytesObject
 ## ================================
 ##
-## The problem
-## -----------
+## PyBytesObject has its own ob_sval buffer, so we have two copies of a string;
+## one in the PyBytesObject returned from various C-API functions and another
+## in the corresponding RPython object.
 ##
-## PyString_AsString() must return a (non-movable) pointer to the underlying
-## ob_sval, whereas pypy strings are movable.  C code may temporarily store
-## this address and use it, as long as it owns a reference to the PyObject.
-## There is no "release" function to specify that the pointer is not needed
-## any more.
+## The following calls can create a PyBytesObject without a correspoinding
+## RPython object:
 ##
-## Also, the pointer may be used to fill the initial value of string. This is
-## valid only when the string was just allocated, and is not used elsewhere.
+## PyBytes_FromStringAndSize(NULL, n) / PyString_FromStringAndSize(NULL, n)
 ##
-## Solution
-## --------
+## In the PyBytesObject returned, the ob_sval buffer may be modified as
+## long as the freshly allocated PyBytesObject is not "forced" via a call
+## to any of the more sophisticated C-API functions. 
 ##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
-##
-## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   NULL buffer.  The first time PyString_AsString() is called, memory is
-##   allocated (with flavor='raw') and content is copied.
-##
-## - A string allocated with PyString_FromStringAndSize(NULL, size) will
-##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, but the reference won't be stored in the global map; there is no
-##   corresponding object in pypy.  When from_ref() or Py_INCREF() is called,
-##   the pypy string is created, and added to the global map of tracked
-##   objects.  The buffer is then supposed to be immutable.
-##
-##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
-##   there is no corresponding pypy object for the string
-##
-## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
-##   similar object.
-##
-## - PyString_Size() doesn't need to force the object.
+## Care has been taken in implementing the functions below, so that
+## if they are called with a non-forced PyBytesObject, they will not 
+## unintentionally force the creation of a RPython object. As long as only these
+## are used, the ob_sval buffer is still modifiable:
+## 
+## PyBytes_AsString / PyString_AsString 
+## PyBytes_AS_STRING / PyString_AS_STRING
+## PyBytes_AsStringAndSize / PyString_AsStringAndSize
+## PyBytes_Size / PyString_Size
+## PyBytes_Resize / PyString_Resize
+## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string.
+##   corresponds to the pypy gc-managed string, 
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -156,9 +144,6 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not pyobj_has_w_obj(ref):
-        # XXX Force the ref?
-        bytes_realize(space, ref)
     return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
@@ -182,9 +167,6 @@
             raise oefmt(space.w_TypeError,
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
-    if not pyobj_has_w_obj(ref):
-        # force the ref
-        bytes_realize(space, ref)
     ref_str = rffi.cast(PyBytesObject, ref)
     data[0] = ref_str.c_ob_sval
     if length:
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -142,7 +142,9 @@
 typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
 typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -156,12 +158,14 @@
     char *format;
     Py_ssize_t *shape;
     Py_ssize_t *strides;
-    Py_ssize_t *suboffsets;
-
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format[Py_MAX_FMT];
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
        mono-dimensional buffers. */
     /* Py_ssize_t smalltable[2]; */
-    void *internal;
+    void *internal; /* always NULL for app-level objects */
 } Py_buffer;
 
 
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,10 +1,122 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    view.c_obj = make_ref(space, w_obj)
+    return fill_Py_buffer(space, buf, view)
+
+def fill_Py_buffer(space, buf, view):    
+    # c_buf, c_obj have been filled in
+    ndim = buf.getndim()
+    view.c_len = buf.getlength()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'        
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+ at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+    """Return 1 if the memory defined by the view is C-style (fortran is
+    'C') or Fortran-style (fortran is 'F') contiguous or either one
+    (fortran is 'A').  Return 0 otherwise."""
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
     return space.call_method(space.builtin, "memoryview", w_obj)
@@ -12,6 +124,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -20,21 +133,22 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-    # TODO - fill in fields
-    '''
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    ''' 
+    if not isinstance(w_obj, W_MemoryView):
+        return view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_obj)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        isstr = False
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        rffi.setintfield(view, 'c_readonly', 1)
+        isstr = True
+    fill_Py_buffer(space, w_obj.buf, view)     
     return view
 
-
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -508,10 +508,9 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Releases a Py_buffer obtained from getbuffer ParseTuple's s*.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
+    Release the buffer view. This should be called when the buffer is 
+    no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
     view.c_obj = lltype.nullptr(PyObject.TO)
+    # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
     def getshape(self):
         return self.shape
 
+    def getstrides(self):
+        return self.strides
+
     def getitemsize(self):
         return self.itemsize
 
+    def getndim(self):
+        return self.ndim
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
   PyMyArray* self = (PyMyArray*)obj;
-  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
-    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
-    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
@@ -188,7 +185,131 @@
     (initproc)PyMyArray_init,     /* tp_init */
 };
 
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+    Py_buffer* view = NULL;
+    PyObject* obj = PyTuple_GetItem(args, 0);
+    PyObject* memoryview = PyMemoryView_FromObject(obj);
+    if (memoryview == NULL)
+        return PyInt_FromLong(-1);
+    view = PyMemoryView_GET_BUFFER(memoryview);
+    Py_DECREF(memoryview);
+    return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG)                                        \
+    buf_flag = PyUnicode_FromString(#FLAG);                         \
+    flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ);  \
+    Py_DECREF(buf_flag);                                            \
+    if (flag_matches == 1) {                                        \
+        Py_DECREF(tmp);                                             \
+        flags |= PyBUF_##FLAG;                                      \
+        continue;                                                   \
+    }                                                               \
+    else if (flag_matches == -1) {                                  \
+        Py_DECREF(tmp);                                             \
+        return NULL;                                                \
+    }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+    PyObject *buffer_obj, *pyflags;
+    PyObject *tmp, *buf_flag;
+    Py_buffer buffer;
+    PyObject *shape, *strides;
+    Py_ssize_t i, n;
+    int flag_matches;
+    int flags = 0;
+
+    if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+        return NULL;
+    }
+
+    n = PySequence_Length(pyflags);
+    if (n < 0) {
+        return NULL;
+    }
+
+    for (i=0; i < n; i++) {
+        tmp = PySequence_GetItem(pyflags, i);
+        if (tmp == NULL) {
+            return NULL;
+        }
+
+        GET_PYBUF_FLAG(SIMPLE);
+        GET_PYBUF_FLAG(WRITABLE);
+        GET_PYBUF_FLAG(STRIDES);
+        GET_PYBUF_FLAG(ND);
+        GET_PYBUF_FLAG(C_CONTIGUOUS);
+        GET_PYBUF_FLAG(F_CONTIGUOUS);
+        GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+        GET_PYBUF_FLAG(INDIRECT);
+        GET_PYBUF_FLAG(FORMAT);
+        GET_PYBUF_FLAG(STRIDED);
+        GET_PYBUF_FLAG(STRIDED_RO);
+        GET_PYBUF_FLAG(RECORDS);
+        GET_PYBUF_FLAG(RECORDS_RO);
+        GET_PYBUF_FLAG(FULL);
+        GET_PYBUF_FLAG(FULL_RO);
+        GET_PYBUF_FLAG(CONTIG);
+        GET_PYBUF_FLAG(CONTIG_RO);
+
+        Py_DECREF(tmp);
+
+        /* One of the flags must match */
+        PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+        return NULL;
+    }
+
+    if (buffer.shape == NULL) {
+        Py_INCREF(Py_None);
+        shape = Py_None;
+    }
+    else {
+        shape = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+        }
+    }
+
+    if (buffer.strides == NULL) {
+        Py_INCREF(Py_None);
+        strides = Py_None;
+    }
+    else {
+        strides = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+        }
+    }
+
+    PyBuffer_Release(&buffer);
+    return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
 static PyMethodDef buffer_functions[] = {
+    {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
+    {"get_buffer_info",   (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,9 +4,7 @@
 PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
 {
     PyObject* newType;
-    /*printf("in foo3type_tp_new, preprocessing...\n"); */
     newType = PyType_Type.tp_new(metatype, args, kwds);
-    /*printf("in foo3type_tp_new, postprocessing...\n"); */
     return newType;
 }
 
@@ -81,4 +79,5 @@
         return;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return;
+    Py_INCREF(&footype);
 }
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
 import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.api import PyObject
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -183,8 +183,27 @@
                  Py_INCREF(Py_None);
                  return Py_None;
              """),
+            ("c_only", "METH_NOARGS",
+            """
+                int ret;
+                char * buf2;
+                PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+                if (!obj)
+                    return NULL;
+                buf2 = PyBytes_AsString(obj);
+                if (!buf2)
+                    return NULL;
+                /* buf should not have been forced, issue #2395 */
+                ret = _PyBytes_Resize(&obj, 512);
+                if (ret < 0)
+                    return NULL;
+                 Py_DECREF(obj);
+                 Py_INCREF(Py_None);
+                 return Py_None;
+            """),
             ])
         module.getbytes()
+        module.c_only()
 
     def test_py_string_as_string_Unicode(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
 
 from .support import c_compile
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0
@@ -53,7 +55,48 @@
         libraries=libraries)
     return soname
 
-def compile_extension_module(space, modname, include_dirs=[],
+class SystemCompilationInfo(object):
+    """Bundles all the generic information required to compile extensions.
+
+    Note: here, 'system' means OS + target interpreter + test config + ...
+    """
+    def __init__(self, include_extra=None, compile_extra=None, link_extra=None,
+            extra_libs=None, ext=None):
+        self.include_extra = include_extra or []
+        self.compile_extra = compile_extra
+        self.link_extra = link_extra
+        self.extra_libs = extra_libs
+        self.ext = ext
+
+def get_cpyext_info(space):
+    from pypy.module.imp.importing import get_so_extension
+    state = space.fromcache(State)
+    api_library = state.api_lib
+    if sys.platform == 'win32':
+        libraries = [api_library]
+        # '%s' undefined; assuming extern returning int
+        compile_extra = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+            (space.int_w(w_maj), space.int_w(w_min))]
+    else:
+        libraries = []
+        if sys.platform.startswith('linux'):
+            compile_extra = [
+                "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
+    return SystemCompilationInfo(
+        include_extra=api.include_dirs,
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        extra_libs=libraries,
+        ext=get_so_extension(space))
+
+
+def compile_extension_module(sys_info, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
@@ -65,35 +108,15 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).
     """
-    state = space.fromcache(State)
-    api_library = state.api_lib
-    if sys.platform == 'win32':
-        libraries = [api_library]
-        # '%s' undefined; assuming extern returning int
-        compile_extra = ["/we4013"]
-        # prevent linking with PythonXX.lib
-        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
-        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
-                              (space.int_w(w_maj), space.int_w(w_min))]
-    else:
-        libraries = []
-        if sys.platform.startswith('linux'):
-            compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
-            link_extra = ["-g"]
-        else:
-            compile_extra = link_extra = None
-
     modname = modname.split('.')[-1]
     soname = create_so(modname,
-            include_dirs=api.include_dirs + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra,
-            libraries=libraries)
-    from pypy.module.imp.importing import get_so_extension
-    ext = get_so_extension(space)
-    pydname = soname.new(purebasename=modname, ext=ext)
+        include_dirs=sys_info.include_extra + include_dirs,
+        source_files=source_files,
+        source_strings=source_strings,
+        compile_extra=sys_info.compile_extra,
+        link_extra=sys_info.link_extra,
+        libraries=sys_info.extra_libs)
+    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
     soname.rename(pydname)
     return str(pydname)
 
@@ -106,18 +129,8 @@
         raise RuntimeError("This interpreter does not define a filename "
             "suffix for C extensions!")
 
-def compile_extension_module_applevel(space, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
+def get_sys_info_app():
+    from distutils.sysconfig import get_python_inc
     if sys.platform == 'win32':
         compile_extra = ["/we4013"]
         link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
@@ -128,18 +141,13 @@
         compile_extra = [
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
         link_extra = None
+    ext = get_so_suffix()
+    return SystemCompilationInfo(
+        include_extra=[get_python_inc()],
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        ext=get_so_suffix())
 
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-            include_dirs=[space.include_dir] + include_dirs,
-            source_files=source_files,
-            source_strings=source_strings,
-            compile_extra=compile_extra,
-            link_extra=link_extra)
-    ext = get_so_suffix()
-    pydname = soname.new(purebasename=modname, ext=ext)
-    soname.rename(pydname)
-    return str(pydname)
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -154,9 +162,7 @@
 class FakeSpace(object):
     """Like TinyObjSpace, but different"""
     def __init__(self, config):
-        from distutils.sysconfig import get_python_inc
         self.config = config
-        self.include_dir = get_python_inc()
 
     def passthrough(self, arg):
         return arg
@@ -271,11 +277,11 @@
             "the test actually passed in the first place; if it failed "
             "it is likely to reach this place.")
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_only_import(self):
         import cpyext
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_load_error(self):
         import cpyext
         raises(ImportError, cpyext.load_module, "missing.file", "foo")
@@ -303,27 +309,23 @@
     def setup_method(self, func):
         @gateway.unwrap_spec(name=str)
         def compile_module(space, name,
-                           w_separate_module_files=None,
-                           w_separate_module_sources=None):
+                           w_source_files=None,
+                           w_source_strings=None):
             """
             Build an extension module linked against the cpyext api library.
             """
-            if not space.is_none(w_separate_module_files):
-                separate_module_files = space.listview_bytes(
-                    w_separate_module_files)
-                assert separate_module_files is not None
+            if not space.is_none(w_source_files):
+                source_files = space.listview_bytes(w_source_files)
             else:
-                separate_module_files = []
-            if not space.is_none(w_separate_module_sources):
-                separate_module_sources = space.listview_bytes(
-                    w_separate_module_sources)
-                assert separate_module_sources is not None
+                source_files = None
+            if not space.is_none(w_source_strings):
+                source_strings = space.listview_bytes(w_source_strings)
             else:
-                separate_module_sources = []
-            pydname = self.compile_extension_module(
-                space, name,
-                source_files=separate_module_files,
-                source_strings=separate_module_sources)
+                source_strings = None
+            pydname = compile_extension_module(
+                self.sys_info, name,
+                source_files=source_files,
+                source_strings=source_strings)
             return space.wrap(pydname)
 
         @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -376,7 +378,7 @@
                 filename = py.path.local(pypydir) / 'module' \
                         / 'cpyext'/ 'test' / (filename + ".c")
                 kwds = dict(source_files=[filename])
-            mod = self.compile_extension_module(space, name,
+            mod = compile_extension_module(self.sys_info, name,
                     include_dirs=include_dirs, **kwds)
 
             if load_it:
@@ -468,11 +470,11 @@
                 return run
             def wrap(func):
                 return func
-            self.compile_extension_module = compile_extension_module_applevel
+            self.sys_info = get_sys_info_app()
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
-            self.compile_extension_module = compile_extension_module
+            self.sys_info = get_cpyext_info(self.space)
         self.w_compile_module = wrap(interp2app(compile_module))
         self.w_import_module = wrap(interp2app(import_module))
         self.w_reimport_module = wrap(interp2app(reimport_module))
@@ -634,10 +636,10 @@
             skip('record_imported_module not supported in runappdirect mode')
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", separate_module_files=[self.here + 'banana.c'])
+            "apple.banana", source_files=[self.here + 'banana.c'])
         self.record_imported_module("apple.banana")
         date = self.compile_module(
-            "cherry.date", separate_module_files=[self.here + 'date.c'])
+            "cherry.date", source_files=[self.here + 'date.c'])
         self.record_imported_module("cherry.date")
 
         # Set up some package state so that the extensions can actually be
@@ -894,7 +896,7 @@
             ])
         raises(SystemError, mod.newexc, "name", Exception, {})
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy specific test')
+    @pytest.mark.skipif(only_pypy, reason='pypy specific test')
     def test_hash_pointer(self):
         mod = self.import_extension('foo', [
             ('get_hash', 'METH_NOARGS',
@@ -945,7 +947,7 @@
         print p
         assert 'py' in p
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_version(self):
         mod = self.import_extension('foo', [
             ('get_version', 'METH_NOARGS',
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,7 @@
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
 
 class TestMemoryViewObject(BaseApiTest):
     def test_fromobject(self, space, api):
@@ -12,6 +13,16 @@
         w_bytes = space.call_method(w_view, "tobytes")
         assert space.unwrap(w_bytes) == "hello"
 
+    def test_frombuffer(self, space, api):
+        w_buf = space.newbuffer(StringBuffer("hello"))
+        w_memoryview = api.PyMemoryView_FromObject(w_buf)
+        w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+        assert w_view.c_ndim == 1
+        f = rffi.charp2str(w_view.c_format)
+        assert f == 'B'
+        assert w_view.c_shape[0] == 5
+        assert w_view.c_strides[0] == 1
+        assert w_view.c_len == 5
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):
@@ -21,6 +32,21 @@
         y = memoryview(arr)
         assert y.format == 'i'
         assert y.shape == (10,)
+        assert len(y) == 10
         s = y[3]
         assert len(s) == struct.calcsize('i')
         assert s == struct.pack('i', 3)
+        viewlen = module.test_buffer(arr)
+        assert viewlen == y.itemsize * len(y)
+
+    def test_buffer_info(self):
+        from _numpypy import multiarray as np
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        arr = np.zeros((1, 10), order='F')
+        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+        assert strides[0] == 8
+        arr = np.zeros((10, 1), order='C')
+        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+        assert strides[-1] == 8
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
         assert module.test_fast_sequence(s[0:-1])
         assert module.test_fast_sequence(s[::-1])
 
+    def test_fast_keyerror(self):
+        module = self.import_extension('foo', [
+            ("test_fast_sequence", "METH_VARARGS",
+             """
+                PyObject *foo;
+                PyObject * seq = PyTuple_GetItem(args, 0);
+                if (seq == NULL)
+                    Py_RETURN_NONE;
+                foo = PySequence_Fast(seq, "Could not convert object to sequence");
+                if (foo != NULL)
+                {
+                    return foo;
+                }
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                    return PyBool_FromLong(1);
+                }
+                return NULL;
+             """)])
+        class Map(object):
+            def __len__(self):
+                return 1
+
+            def __getitem__(self, index):
+                raise KeyError()
+
+        assert module.test_fast_sequence(Map()) is True
+
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -1,12 +1,13 @@
 import sys
 
-import py, pytest
+import pytest
 
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 class AppTestThread(AppTestCpythonExtensionBase):
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_get_thread_ident(self):
         module = self.import_extension('foo', [
             ("get_thread_ident", "METH_NOARGS",
@@ -33,7 +34,7 @@
 
         assert results[0][0] != results[1][0]
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_acquire_lock(self):
         module = self.import_extension('foo', [
             ("test_acquire_lock", "METH_NOARGS",
@@ -57,7 +58,7 @@
             ])
         module.test_acquire_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_release_lock(self):
         module = self.import_extension('foo', [
             ("test_release_lock", "METH_NOARGS",
@@ -79,7 +80,7 @@
             ])
         module.test_release_lock()
 
-    @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_tls(self):
         module = self.import_extension('foo', [
             ("create_key", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -961,7 +961,6 @@
 
     def test_tp_new_in_subclass_of_type(self):
         module = self.import_module(name='foo3')
-        #print('calling module.footype()...')
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,6 +3,7 @@
 import py, pytest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
 
 def test_pragma_version():
     from pypy.module.sys.version import CPYTHON_VERSION
@@ -32,11 +33,9 @@
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro
 
-    #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
     def test_pypy_versions(self):
         import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("pypy only test")
         init = """
         if (Py_IsInitialized()) {
             PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
                     STRUCT_TYPE = PySequenceMethods
+                elif slot_names[0] == 'c_tp_as_buffer':
+                    STRUCT_TYPE = PyBufferProcs
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -259,10 +259,8 @@
         raise oefmt(space.w_ValueError, "Empty module name")
     w = space.wrap
 
-    if w_fromlist is not None and space.is_true(w_fromlist):
-        fromlist_w = space.fixedview(w_fromlist)
-    else:
-        fromlist_w = None
+    if w_fromlist is not None and not space.is_true(w_fromlist):
+        w_fromlist = None
 
     rel_modulename = None
     if (level != 0 and w_globals is not None and
@@ -284,19 +282,19 @@
                     w_mod = None
                 else:
                     w_mod = absolute_import(space, rel_modulename, rel_level,
-                                            fromlist_w, tentative=True)
+                                            w_fromlist, tentative=True)
             else:
                 w_mod = absolute_import(space, rel_modulename, rel_level,
-                                        fromlist_w, tentative=False)
+                                        w_fromlist, tentative=False)
             if w_mod is not None:
                 return w_mod
 
-    w_mod = absolute_import(space, modulename, 0, fromlist_w, tentative=0)
+    w_mod = absolute_import(space, modulename, 0, w_fromlist, tentative=0)
     if rel_modulename is not None:
         space.setitem(space.sys.get('modules'), w(rel_modulename), space.w_None)
     return w_mod
 
-def absolute_import(space, modulename, baselevel, fromlist_w, tentative):
+def absolute_import(space, modulename, baselevel, w_fromlist, tentative):
     # Short path: check in sys.modules, but only if there is no conflict
     # on the import lock.  In the situation of 'import' statements
     # inside tight loops, this should be true, and absolute_import_try()
@@ -304,25 +302,25 @@
     # if the import lock is currently held by another thread, then we
     # have to wait, and so shouldn't use the fast path.
     if not getimportlock(space).lock_held_by_someone_else():
-        w_mod = absolute_import_try(space, modulename, baselevel, fromlist_w)
+        w_mod = absolute_import_try(space, modulename, baselevel, w_fromlist)
         if w_mod is not None and not space.is_w(w_mod, space.w_None):
             return w_mod
     return absolute_import_with_lock(space, modulename, baselevel,
-                                     fromlist_w, tentative)
+                                     w_fromlist, tentative)
 
 @jit.dont_look_inside
 def absolute_import_with_lock(space, modulename, baselevel,
-                              fromlist_w, tentative):
+                              w_fromlist, tentative):
     lock = getimportlock(space)
     lock.acquire_lock()
     try:
         return _absolute_import(space, modulename, baselevel,
-                                fromlist_w, tentative)
+                                w_fromlist, tentative)
     finally:
         lock.release_lock(silent_after_fork=True)
 
 @jit.unroll_safe
-def absolute_import_try(space, modulename, baselevel, fromlist_w):
+def absolute_import_try(space, modulename, baselevel, w_fromlist):
     """ Only look up sys.modules, not actually try to load anything
     """
     w_path = None
@@ -330,7 +328,7 @@
     if '.' not in modulename:
         w_mod = check_sys_modules_w(space, modulename)
         first = w_mod
-        if fromlist_w is not None and w_mod is not None:
+        if w_fromlist is not None and w_mod is not None:
             w_path = try_getattr(space, w_mod, space.wrap('__path__'))
     else:
         level = 0
@@ -345,28 +343,37 @@
                 return None
             if level == baselevel:
                 first = w_mod
-            if fromlist_w is not None:
+            if w_fromlist is not None:
                 w_path = try_getattr(space, w_mod, space.wrap('__path__'))
             level += 1
-    if fromlist_w is not None:
+    if w_fromlist is not None:
+        # bit artificial code but important to not just unwrap w_fromlist
+        # to get a better trace. if it is unwrapped, the immutability of the
+        # tuple is lost
         if w_path is not None:
-            if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],
-                                                   space.wrap('*')):
+            length = space.len_w(w_fromlist)
+            if length == 1 and space.eq_w(
+                    space.getitem(w_fromlist, space.wrap(0)),
+                    space.wrap('*')):
                 w_all = try_getattr(space, w_mod, space.wrap('__all__'))
                 if w_all is not None:
-                    fromlist_w = space.fixedview(w_all)
+                    w_fromlist = w_all
+                    length = space.len_w(w_fromlist)
                 else:
-                    fromlist_w = []
+                    w_fromlist = None
                     # "from x import *" with x already imported and no x.__all__
                     # always succeeds without doing more imports.  It will
                     # just copy everything from x.__dict__ as it is now.
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    return None
+
+            if w_fromlist is not None:
+                for i in range(length):
+                    w_name = space.getitem(w_fromlist, space.wrap(i))
+                    if try_getattr(space, w_mod, w_name) is None:
+                        return None
         return w_mod
     return first
 
-def _absolute_import(space, modulename, baselevel, fromlist_w, tentative):
+def _absolute_import(space, modulename, baselevel, w_fromlist, tentative):
     w = space.wrap
 
     if '/' in modulename or '\\' in modulename:
@@ -394,18 +401,24 @@
         w_path = try_getattr(space, w_mod, w('__path__'))
         level += 1
 
-    if fromlist_w is not None:
+    if w_fromlist is not None:
         if w_path is not None:
-            if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],w('*')):
+            length = space.len_w(w_fromlist)
+            if length == 1 and space.eq_w(
+                    space.getitem(w_fromlist, space.wrap(0)),
+                    space.wrap('*')):
                 w_all = try_getattr(space, w_mod, w('__all__'))
                 if w_all is not None:
-                    fromlist_w = space.fixedview(w_all)
+                    w_fromlist = w_all
+                    length = space.len_w(w_fromlist)
                 else:
-                    fromlist_w = []
-            for w_name in fromlist_w:
-                if try_getattr(space, w_mod, w_name) is None:
-                    load_part(space, w_path, prefix, space.str0_w(w_name),
-                              w_mod, tentative=1)
+                    w_fromlist = None
+            if w_fromlist is not None:
+                for i in range(length):
+                    w_name = space.getitem(w_fromlist, space.wrap(i))
+                    if try_getattr(space, w_mod, w_name) is None:
+                        load_part(space, w_path, prefix, space.str0_w(w_name),
+                                  w_mod, tentative=1)
         return w_mod
     else:
         return first
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -65,8 +65,9 @@
              )
     setuppkg("pkg.pkg2", a='', b='')
     setuppkg("pkg.withall",
-             __init__  = "__all__ = ['foobar']",
-             foobar    = "found = 123")
+             __init__  = "__all__ = ['foobar', 'barbaz']",
+             foobar    = "found = 123",
+             barbaz    = "other = 543")
     setuppkg("pkg.withoutall",
              __init__  = "",
              foobar    = "found = 123")
@@ -707,6 +708,7 @@
             d = {}
             exec "from pkg.withall import *" in d
             assert d["foobar"].found == 123
+            assert d["barbaz"].other == 543
 
     def test_import_star_does_not_find_submodules_without___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
     def getdictvalue(self, space, key):
         return self.items[key]
 
+    def descr_memoryview(self, space, buf):
+        raise oefmt(space.w_TypeError, "error")
+
 class IterDictObject(W_Root):
     def __init__(self, space, w_dict):
         self.space = space
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -377,7 +377,25 @@
     def __exit__(self, typ, value, traceback):
         keepalive_until_here(self)
 
-    def get_buffer(self, space, readonly):
+    def get_buffer(self, space, flags):
+        errtype = space.w_ValueError # should be BufferError, numpy does this instead
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+                not self.flags & NPY.ARRAY_F_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not Fortran contiguous")
+        if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                     self.flags & NPY.ARRAY_C_CONTIGUOUS)):
+           raise oefmt(errtype, "ndarray is not contiguous")
+        if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
+                not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+           raise oefmt(errtype, "ndarray is not C-contiguous")
+        if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and
+            not self.flags & NPY.ARRAY_WRITEABLE):
+           raise oefmt(errtype, "buffer source array is read-only")
+        readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
         return ArrayBuffer(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
@@ -695,6 +713,7 @@
                  index + self.impl.start)
 
     def setitem(self, index, v):
+        # XXX what if self.readonly?
         raw_storage_setitem(self.impl.storage, index + self.impl.start,
                             rffi.cast(lltype.Char, v))
 
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstring import strip_spaces
@@ -42,7 +43,7 @@
         raise oefmt(space.w_ValueError,
                     "object __array__ method not producing an array")
 
-def try_interface_method(space, w_object):
+def try_interface_method(space, w_object, copy):
     try:
         w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
         if w_interface is None:
@@ -81,17 +82,20 @@
             raise oefmt(space.w_ValueError,
                     "__array_interface__ could not decode dtype %R", w_dtype
                     )
-        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
+        if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
+                                   space.isinstance_w(w_data, space.w_list)):
             data_w = space.listview(w_data)
-            data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
-            read_only = True # XXX why not space.is_true(data_w[1])
+            w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+            read_only = space.is_true(data_w[1]) or copy
             offset = 0
-            return W_NDimArray.from_shape_and_storage(space, shape, data, 
-                                    dtype, strides=strides, start=offset), read_only
+            w_base = w_object
+            if read_only:
+                w_base = None
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+                                dtype, w_base=w_base, strides=strides,
+                                start=offset), read_only
         if w_data is None:
-            data = w_object
-        else:
-            data = w_data
+            w_data = w_object
         w_offset = space.finditem(w_interface, space.wrap('offset'))
         if w_offset is None:
             offset = 0
@@ -101,7 +105,7 @@
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
                    "__array_interface__ strides not fully supported yet") 
-        arr = frombuffer(space, data, dtype, support.product(shape), offset)
+        arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
         
@@ -110,6 +114,78 @@
             return None, False
         raise
 
+def _descriptor_from_pep3118_format(space, c_format):
+    descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+    if descr:
+        return descr
+    msg = "invalid PEP 3118 format string: '%s'" % c_format
+    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+    return None 
+
+def _array_from_buffer_3118(space, w_object, dtype):
+    try:
+        w_buf = space.call_method(space.builtin, "memoryview", w_object)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            # object does not have buffer interface
+            return w_object
+        raise
+    format = space.getattr(w_buf,space.newbytes('format'))
+    if format:
+        descr = _descriptor_from_pep3118_format(space, space.str_w(format))
+        if not descr:
+            return w_object
+        if dtype and descr:
+            raise oefmt(space.w_NotImplementedError,
+                "creating an array from a memoryview while specifying dtype "
+                "not supported")
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
+            msg = ("Item size computed from the PEP 3118 buffer format "
+                  "string does not match the actual item size.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+        dtype = descr 
+    elif not dtype:
+        dtype = descriptor.get_dtype_cache(space).w_stringdtype
+        dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
+    nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim')))
+    shape = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('shape')))]
+    strides = []
+    buflen = space.len_w(w_buf) * dtype.elsize
+    if shape:
+        strides = [space.int_w(d) for d in space.listview(
+                            space.getattr(w_buf, space.newbytes('strides')))]
+        if not strides:
+            d = buflen
+            strides = [0] * nd
+            for k in range(nd):
+                if shape[k] > 0:
+                    d /= shape[k]
+                    strides[k] = d
+    else:
+        if nd == 1:
+            shape = [buflen / dtype.elsize, ]
+            strides = [dtype.elsize, ]
+        elif nd > 1:
+            msg = ("ndim computed from the PEP 3118 buffer format "
+                   "is greater than 1, but shape is NULL.")
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            return w_object
+    try:
+        w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
+    except OperationError as e:
+        if e.match(space, space.w_ValueError):
+            return w_object
+        else:
+            raise e
+    writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
+    w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
+               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
+               writable=writable, strides=strides)
+    if w_ret:
+        return w_ret
+    return w_object
 
 @unwrap_spec(ndmin=int, copy=bool, subok=bool)
 def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -127,6 +203,7 @@
 
 def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
 
+    from pypy.module.micronumpy.boxes import W_GenericBox
     # numpy testing calls array(type(array([]))) and expects a ValueError
     if space.isinstance_w(w_object, space.w_type):
         raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
@@ -134,13 +211,19 @@
     dtype = descriptor.decode_w_dtype(space, w_dtype)
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
-        if w_array is not None:
+        if w_array is None:
+            if (    not space.isinstance_w(w_object, space.w_str) and 
+                    not space.isinstance_w(w_object, space.w_unicode) and
+                    not isinstance(w_object, W_GenericBox)):
+                # use buffer interface
+                w_object = _array_from_buffer_3118(space, w_object, dtype)
+        else:
             # continue with w_array, but do further operations in place
             w_object = w_array
             copy = False
             dtype = w_object.get_dtype()
     if not isinstance(w_object, W_NDimArray):
-        w_array, _copy = try_interface_method(space, w_object)
+        w_array, _copy = try_interface_method(space, w_object, copy)
         if w_array is not None:
             w_object = w_array
             copy = _copy
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -805,19 +805,19 @@
         return w_result
 
     def buffer_w(self, space, flags):
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, flags)
 
     def readbuf_w(self, space):
-        return self.implementation.get_buffer(space, True)
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO)
 
     def writebuf_w(self, space):
-        return self.implementation.get_buffer(space, False)
+        return self.implementation.get_buffer(space, space.BUF_FULL)
 
     def charbuf_w(self, space):
-        return self.implementation.get_buffer(space, True).as_str()
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
 
     def descr_get_data(self, space):
-        return space.newbuffer(self.implementation.get_buffer(space, False))
+        return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3215,7 +3215,9 @@
         raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)}))
 
         a = array([1, 2, 3])
-        b = array(Dummy(a.__array_interface__))
+        d = Dummy(a.__array_interface__)
+        b = array(d)
+        assert b.base is None
         b[1] = 200
         assert a[1] == 2 # upstream compatibility, is this a bug?
         interface_a = a.__array_interface__
@@ -3226,6 +3228,8 @@
         interface_b.pop('data')
         interface_a.pop('data')
         assert interface_a == interface_b
+        b = array(d, copy=False)
+        assert b.base is d
 
         b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
                          'data': 'a'*100}))
@@ -3594,6 +3598,7 @@
         cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
         cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
         cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+        cls.w_one = cls.space.wrap(struct.pack('i', 1))
 
     def test_frombuffer(self):
         import numpy as np
@@ -3645,8 +3650,6 @@
         else:
             EMPTY = None
         x = np.array([1, 2, 3, 4, 5], dtype='i')
-        y = memoryview('abc')
-        assert y.format == 'B'
         y = memoryview(x)
         assert y.format == 'i'
         assert y.shape == (5,)
@@ -3654,6 +3657,16 @@
         assert y.strides == (4,)
         assert y.suboffsets == EMPTY
         assert y.itemsize == 4
+        assert isinstance(y, memoryview)
+        assert y[0] == self.one
+        assert (np.array(y) == x).all()
+
+        x = np.array([0, 0, 0, 0], dtype='O')
+        y = memoryview(x)
+        # handles conversion of address to pinned object?
+        z = np.array(y)
+        assert z.dtype == 'O'
+        assert (z == x).all()
 
     def test_fromstring(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -702,3 +702,32 @@
         ret = obj.sum()
         print type(ret)
         assert ret.info == 'spam'
+
+    def test_ndarray_subclass_assigns_base(self):
+        import numpy as np
+        init_called = []
+        class _DummyArray(object):
+            """ Dummy object that just exists to hang __array_interface__ dictionaries
+            and possibly keep alive a reference to a base array.
+            """
+            def __init__(self, interface, base=None):
+                self.__array_interface__ = interface
+                init_called.append(1)
+                self.base = base
+
+        x = np.zeros(10)
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 1
+        assert y.base is d
+
+        x = np.zeros((0,), dtype='float32')
+        intf = x.__array_interface__.copy()
+        intf["strides"] = x.strides
+        x.__array_interface__["strides"] = x.strides
+        d = _DummyArray(x.__array_interface__, base=x)
+        y = np.array(d, copy=False)
+        assert sum(init_called) == 2
+        assert y.base is d
+
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1851,7 +1851,7 @@
                     arr.gcstruct)
 
     def read(self, arr, i, offset, dtype):
-        if arr.gcstruct is V_OBJECTSTORE:
+        if arr.gcstruct is V_OBJECTSTORE and not arr.base():
             raise oefmt(self.space.w_NotImplementedError,
                 "cannot read object from array with no gc hook")
         return self.box(self._read(arr.storage, i, offset))
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -392,31 +392,39 @@
         extobj_w = space.newlist([space.wrap(8192), space.wrap(0), space.w_None])
         return extobj_w
 
+
+_reflected_ops = {
+        'add': 'radd',
+        'subtract': 'rsub',
+        'multiply': 'rmul',
+        'divide': 'rdiv',
+        'true_divide': 'rtruediv',
+        'floor_divide': 'rfloordiv',
+        'remainder': 'rmod',
+        'power': 'rpow',
+        'left_shift': 'rlshift',
+        'right_shift': 'rrshift',
+        'bitwise_and': 'rand',
+        'bitwise_xor': 'rxor',
+        'bitwise_or': 'ror',
+        #/* Comparisons */

From pypy.commits at gmail.com  Tue Sep 13 11:40:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 08:40:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix some pickling tests
Message-ID: <57d81e0a.121a1c0a.ef65f.1b81@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87080:709d1b5a8b17
Date: 2016-09-13 16:45 +0200
http://bitbucket.org/pypy/pypy/changeset/709d1b5a8b17/

Log:	Fix some pickling tests

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -690,6 +690,9 @@
     def descr_function_repr(self):
         return self.space.wrap('<built-in function %s>' % (self.name,))
 
+    def descr__reduce__(self, space):
+        return space.wrap(self.qualname)
+
 def is_builtin_code(w_func):
     from pypy.interpreter.gateway import BuiltinCode
     if isinstance(w_func, Method):
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -762,6 +762,7 @@
     '__self__': GetSetProperty(always_none, cls=BuiltinFunction),
     '__repr__': interp2app(BuiltinFunction.descr_function_repr),
     '__doc__': getset_func_doc,
+    '__reduce__': interp2app(BuiltinFunction.descr__reduce__),
     })
 del BuiltinFunction.typedef.rawdict['__get__']
 BuiltinFunction.typedef.acceptable_as_base_class = False

From pypy.commits at gmail.com  Tue Sep 13 11:41:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 08:41:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Use ast.NameConstant instead of ast.Name
 for None/True/False
Message-ID: <57d81e0c.a7bec20a.d01f4.e13a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87081:19e8fc8dcfd0
Date: 2016-09-13 17:39 +0200
http://bitbucket.org/pypy/pypy/changeset/19e8fc8dcfd0/

Log:	Use ast.NameConstant instead of ast.Name for None/True/False (that
	class was not used so far)

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1194,9 +1194,19 @@
         first_child = atom_node.get_child(0)
         first_child_type = first_child.type
         if first_child_type == tokens.NAME:
-            name = self.new_identifier(first_child.get_value())
-            return ast.Name(name, ast.Load, first_child.get_lineno(),
-                            first_child.get_column())
+            name = first_child.get_value()
+            if name == "None":
+                w_singleton = self.space.w_None
+            elif name == "True":
+                w_singleton = self.space.w_True
+            elif name == "False":
+                w_singleton = self.space.w_False
+            else:
+                name = self.new_identifier(name)
+                return ast.Name(name, ast.Load, first_child.get_lineno(),
+                                first_child.get_column())
+            return ast.NameConstant(w_singleton, first_child.get_lineno(),
+                                first_child.get_column())
         elif first_child_type == tokens.STRING:
             space = self.space
             encoding = self.compile_info.encoding
diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py
--- a/pypy/interpreter/astcompiler/asthelpers.py
+++ b/pypy/interpreter/astcompiler/asthelpers.py
@@ -22,15 +22,19 @@
 class __extend__(ast.expr):
 
     constant = False
+    _description = None
 
     def as_node_list(self, space):
         return None
 
     def set_context(self, ctx):
+        d = self._description
+        if d is None:
+            d = "%r" % (self,)
         if ctx == ast.Del:
-            msg = "can't delete %s" % (self._description,)
+            msg = "can't delete %s" % (d,)
         else:
-            msg = "can't assign to %s" % (self._description,)
+            msg = "can't assign to %s" % (d,)
         raise UnacceptableExpressionContext(self, msg)
 
 
@@ -174,3 +178,9 @@
 
     _description = "Ellipsis"
     constant = True
+
+
+class __extend__(ast.NameConstant):
+
+    _description = "name constant"
+    constant = True
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1271,6 +1271,10 @@
         self.update_position(name.lineno)
         self.name_op(name.id, name.ctx)
 
+    def visit_NameConstant(self, node):
+        self.update_position(node.lineno)
+        self.load_const(node.single)
+
     def visit_keyword(self, keyword):
         if keyword.arg is not None:
             self.load_const(self.space.wrap(keyword.arg.decode('utf-8')))
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -67,6 +67,11 @@
     def as_constant(self):
         return self.obj
 
+class __extend__(ast.NameConstant):
+
+    def as_constant(self):
+        return self.single
+
 class __extend__(ast.Index):
     def as_constant(self):
         return self.value.as_constant()
diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -390,9 +390,21 @@
     def test_tuple(self):
         self._sequence(ast.Tuple)
 
+    def test_nameconstant(self):
+        node = ast.NameConstant("True", 0, 0)
+        self.expr(node, "singleton must be True, False, or None")
+
     def test_stdlib_validates(self):
         stdlib = os.path.join(os.path.dirname(ast.__file__), '../../../lib-python/3')
-        tests = ["os.py", "test/test_grammar.py", "test/test_unpack_ex.py"]
+        if 1:    # enable manually for a complete test
+            tests = [fn for fn in os.listdir(stdlib) if fn.endswith('.py')]
+            tests += ['test/'+fn for fn in os.listdir(stdlib+'/test')
+                                 if fn.endswith('.py')
+                                    and not fn.startswith('bad')]
+            tests.sort()
+        else:
+            tests = ["os.py", "test/test_grammar.py", "test/test_unpack_ex.py"]
+        #
         for module in tests:
             fn = os.path.join(stdlib, module)
             print 'compiling', fn
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -1,6 +1,6 @@
 """A visitor to validate an AST object."""
 
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import oefmt
 from pypy.interpreter.astcompiler import ast
 from rpython.tool.pairtype import pair, pairtype
 from pypy.interpreter.baseobjspace import W_Root
@@ -440,3 +440,10 @@
 
     def visit_Index(self, node):
         self._validate_expr(node.value)
+
+    def visit_NameConstant(self, node):
+        space = self.space
+        if (node.single is not space.w_None and
+            node.single is not space.w_True and
+            node.single is not space.w_False):
+            raise ValidationError("singleton must be True, False, or None")
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -689,10 +689,11 @@
             except NameError:
                 pass
         '''
-        code = self.compiler.compile(snippet, '<tmp>', 'exec', 0)
+        e = py.test.raises(OperationError, self.compiler.compile,
+                snippet, '<tmp>', 'exec', 0)
+        ex = e.value
         space = self.space
-        w_d = space.newdict()
-        space.exec_(code, w_d, w_d)
+        assert ex.match(space, space.w_SyntaxError)
 
     def test_from_future_import(self):
         source = """from __future__ import with_statement

From pypy.commits at gmail.com  Tue Sep 13 11:56:48 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 13 Sep 2016 08:56:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in force-virtual-state (pull
 request #478)
Message-ID: <57d821c0.8bd51c0a.e7f41.2289@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: 
Changeset: r87082:cfcbc07174e8
Date: 2016-09-13 11:56 -0400
http://bitbucket.org/pypy/pypy/changeset/cfcbc07174e8/

Log:	Merged in force-virtual-state (pull request #478)

	Ensure make_inputargs fails properly when given arguments with type
	information

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
 from rpython.jit.metainterp.executor import execute
 from rpython.jit.metainterp.optimize import InvalidLoop
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -11,6 +11,8 @@
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.debug import debug_print
+from rpython.rtyper import rclass
+from rpython.rtyper.lltypesystem import llmemory
 from rpython.jit.metainterp.optimize import SpeculativeError
 
 
@@ -799,6 +801,21 @@
         if not (0 <= index < arraylength):
             raise SpeculativeError
 
+    @staticmethod
+    def _check_subclass(vtable1, vtable2): # checks that vtable1 is a subclass of vtable2
+        known_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable1),
+            rclass.CLASSTYPE)
+        expected_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable2),
+            rclass.CLASSTYPE)
+        # note: the test is for a range including 'max', but 'max'
+        # should never be used for actual classes.  Including it makes
+        # it easier to pass artificial tests.
+        return (expected_class.subclassrange_min
+                <= known_class.subclassrange_min
+                <= expected_class.subclassrange_max)
+
     def is_virtual(self, op):
         if op.type == 'r':
             opinfo = self.getptrinfo(op)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -324,40 +324,24 @@
             return
         self.emit_operation(op)
 
-    def _check_subclass(self, vtable1, vtable2):
-        # checks that vtable1 is a subclass of vtable2
-        known_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable1),
-            rclass.CLASSTYPE)
-        expected_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable2),
-            rclass.CLASSTYPE)
-        # note: the test is for a range including 'max', but 'max'
-        # should never be used for actual classes.  Including it makes
-        # it easier to pass artificial tests.
-        if (expected_class.subclassrange_min
-                <= known_class.subclassrange_min
-                <= expected_class.subclassrange_max):
-            return True
-        return False
-
     def optimize_GUARD_SUBCLASS(self, op):
         info = self.getptrinfo(op.getarg(0))
+        optimizer = self.optimizer
         if info and info.is_constant():
             c = self.get_box_replacement(op.getarg(0))
-            vtable = self.optimizer.cpu.ts.cls_of_box(c).getint()
-            if self._check_subclass(vtable, op.getarg(1).getint()):
+            vtable = optimizer.cpu.ts.cls_of_box(c).getint()
+            if optimizer._check_subclass(vtable, op.getarg(1).getint()):
                 return
             raise InvalidLoop("GUARD_SUBCLASS(const) proven to always fail")
         if info is not None and info.is_about_object():
-            known_class = info.get_known_class(self.optimizer.cpu)
+            known_class = info.get_known_class(optimizer.cpu)
             if known_class:
-                if self._check_subclass(known_class.getint(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(known_class.getint(),
+                                             op.getarg(1).getint()):
                     return
             elif info.get_descr() is not None:
-                if self._check_subclass(info.get_descr().get_vtable(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(info.get_descr().get_vtable(),
+                                             op.getarg(1).getint()):
                     return
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -17,12 +17,13 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
      VirtualStateInfo
-from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.optimizeopt import info, optimizer
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.tool import oparser
 
 class FakeOptimizer(object):
     optearlyforce = None
+    optimizer = optimizer.Optimizer
 
     class cpu:
         remove_gctypeptr = True
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -102,6 +102,8 @@
     node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2)
     node_vtable3 = lltype.malloc(OBJECT_VTABLE, immortal=True)
     node_vtable3.name = rclass.alloc_array_name('node3')
+    node_vtable3.subclassrange_min = 3
+    node_vtable3.subclassrange_max = 3
     node_vtable_adr3 = llmemory.cast_ptr_to_adr(node_vtable3)
     cpu = runner.LLGraphCPU(None)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -3,7 +3,8 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
      VArrayStateInfo, not_virtual, VirtualState,\
-     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
+     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo,\
+     VirtualStateConstructor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
      InputArgFloat
@@ -26,6 +27,7 @@
     def __init__(self, cpu):
         self.cpu = cpu
         self.optearlyforce = None
+        self.optimizer = Optimizer
 
 class BaseTestGenerateGuards(BaseTest):
     def setup_class(self):
@@ -87,6 +89,33 @@
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == []
 
+    def test_make_inputargs_2(self):
+        # Ensure that make_inputargs properly errors with VirtualStatesCantMatch
+        # when the type information for a virtual field conflicts. In practice the
+        # expected and given field always share a common subclass.
+        # This check is needed as not all paths to make_inputargs in unroll.py
+        # are guarded by a call to generate_guards.
+        optimizer = FakeOptimizer(self.cpu)
+        classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        innervalue1 = info.InstancePtrInfo(known_class=classbox1, is_virtual=True, descr=self.valuedescr.get_parent_descr())
+        for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
+            innervalue1.setfield(field, None, ConstInt(42))
+        classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3))
+        innervalue2 = info.InstancePtrInfo(known_class=classbox2, is_virtual=True, descr=self.valuedescr3.get_parent_descr())
+        for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
+            innervalue2.setfield(field, None, ConstInt(42))
+
+        nodebox1 = InputArgRef(self.nodeaddr)
+        nodebox2 = InputArgRef(self.myptr3)
+        nodebox1.set_forwarded(innervalue1)
+        nodebox2.set_forwarded(innervalue2)
+
+        constr = VirtualStateConstructor(optimizer)
+        vs = constr.get_virtual_state([nodebox1])
+
+        with py.test.raises(VirtualStatesCantMatch):
+            args = vs.make_inputargs([nodebox2], optimizer, force_boxes=True)
+
     def test_position_generalization(self):
         def postest(info1, info2):
             info1.position = 0
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -177,6 +177,14 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
+    @staticmethod
+    def descr_issubclass(descr1, descr2, optimizer):
+        if not descr1.is_object() or not descr2.is_object():
+            return True
+        vtable1 = descr1.get_vtable()
+        vtable2 = descr2.get_vtable()
+        return optimizer._check_subclass(vtable1, vtable2)
+
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
@@ -184,10 +192,14 @@
             raise VirtualStatesCantMatch()
         else:
             assert isinstance(info, AbstractStructPtrInfo)
+
         for i in range(len(self.fielddescrs)):
             state = self.fieldstate[i]
+            descr = self.fielddescrs[i].get_parent_descr()
             if not state:
                 continue
+            if not self.descr_issubclass(info.descr, descr, optimizer.optimizer):
+                raise VirtualStatesCantMatch()
             if state.position > self.position:
                 fieldbox = info._fields[i]
                 state.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)

From pypy.commits at gmail.com  Tue Sep 13 12:07:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 09:07:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Meant to be turned off by default
Message-ID: <57d82458.87adc20a.db50d.f470@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87083:5c0791e6dc4c
Date: 2016-09-13 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/5c0791e6dc4c/

Log:	Meant to be turned off by default

diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -396,7 +396,7 @@
 
     def test_stdlib_validates(self):
         stdlib = os.path.join(os.path.dirname(ast.__file__), '../../../lib-python/3')
-        if 1:    # enable manually for a complete test
+        if 0:    # enable manually for a complete test
             tests = [fn for fn in os.listdir(stdlib) if fn.endswith('.py')]
             tests += ['test/'+fn for fn in os.listdir(stdlib+'/test')
                                  if fn.endswith('.py')

From pypy.commits at gmail.com  Tue Sep 13 13:09:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 10:09:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: rposix.sync()
Message-ID: <57d832da.c5cb1c0a.c1697.37b6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87084:f099ecbce222
Date: 2016-09-13 19:05 +0200
http://bitbucket.org/pypy/pypy/changeset/f099ecbce222/

Log:	rposix.sync()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -461,6 +461,8 @@
                    save_err=rffi.RFFI_SAVE_ERRNO)
 c_fdatasync = external('fdatasync', [rffi.INT], rffi.INT,
                        save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    c_sync = external('sync', [], lltype.Void)
 
 @replace_os_function('ftruncate')
 def ftruncate(fd, length):
@@ -477,6 +479,9 @@
     validate_fd(fd)
     handle_posix_error('fdatasync', c_fdatasync(fd))
 
+def sync():
+    c_sync()
+
 #___________________________________________________________________
 
 c_chdir = external('chdir', [rffi.CCHARP], rffi.INT,
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -604,3 +604,7 @@
     assert rposix.get_inheritable(fd1) == False
     os.close(fd1)
     os.close(fd2)
+
+def test_sync():
+    if sys.platform != 'win32':
+        rposix.sync()

From pypy.commits at gmail.com  Tue Sep 13 13:09:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 10:09:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57d832dc.436ec20a.555b6.02e9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87085:a65846c5dec3
Date: 2016-09-13 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/a65846c5dec3/

Log:	hg merge default

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
 from rpython.jit.metainterp.executor import execute
 from rpython.jit.metainterp.optimize import InvalidLoop
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -11,6 +11,8 @@
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.debug import debug_print
+from rpython.rtyper import rclass
+from rpython.rtyper.lltypesystem import llmemory
 from rpython.jit.metainterp.optimize import SpeculativeError
 
 
@@ -799,6 +801,21 @@
         if not (0 <= index < arraylength):
             raise SpeculativeError
 
+    @staticmethod
+    def _check_subclass(vtable1, vtable2): # checks that vtable1 is a subclass of vtable2
+        known_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable1),
+            rclass.CLASSTYPE)
+        expected_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable2),
+            rclass.CLASSTYPE)
+        # note: the test is for a range including 'max', but 'max'
+        # should never be used for actual classes.  Including it makes
+        # it easier to pass artificial tests.
+        return (expected_class.subclassrange_min
+                <= known_class.subclassrange_min
+                <= expected_class.subclassrange_max)
+
     def is_virtual(self, op):
         if op.type == 'r':
             opinfo = self.getptrinfo(op)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -324,37 +324,24 @@
             return
         self.emit_operation(op)
 
-    def _check_subclass(self, vtable1, vtable2):
-        # checks that vtable1 is a subclass of vtable2
-        known_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable1),
-            rclass.CLASSTYPE)
-        expected_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable2),
-            rclass.CLASSTYPE)
-        if (expected_class.subclassrange_min
-                <= known_class.subclassrange_min
-                <= expected_class.subclassrange_max):
-            return True
-        return False
-
     def optimize_GUARD_SUBCLASS(self, op):
         info = self.getptrinfo(op.getarg(0))
+        optimizer = self.optimizer
         if info and info.is_constant():
             c = self.get_box_replacement(op.getarg(0))
-            vtable = self.optimizer.cpu.ts.cls_of_box(c).getint()
-            if self._check_subclass(vtable, op.getarg(1).getint()):
+            vtable = optimizer.cpu.ts.cls_of_box(c).getint()
+            if optimizer._check_subclass(vtable, op.getarg(1).getint()):
                 return
             raise InvalidLoop("GUARD_SUBCLASS(const) proven to always fail")
         if info is not None and info.is_about_object():
-            known_class = info.get_known_class(self.optimizer.cpu)
+            known_class = info.get_known_class(optimizer.cpu)
             if known_class:
-                if self._check_subclass(known_class.getint(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(known_class.getint(),
+                                             op.getarg(1).getint()):
                     return
             elif info.get_descr() is not None:
-                if self._check_subclass(info.get_descr().get_vtable(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(info.get_descr().get_vtable(),
+                                             op.getarg(1).getint()):
                     return
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -17,12 +17,13 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
      VirtualStateInfo
-from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.optimizeopt import info, optimizer
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.tool import oparser
 
 class FakeOptimizer(object):
     optearlyforce = None
+    optimizer = optimizer.Optimizer
 
     class cpu:
         remove_gctypeptr = True
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -102,6 +102,8 @@
     node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2)
     node_vtable3 = lltype.malloc(OBJECT_VTABLE, immortal=True)
     node_vtable3.name = rclass.alloc_array_name('node3')
+    node_vtable3.subclassrange_min = 3
+    node_vtable3.subclassrange_max = 3
     node_vtable_adr3 = llmemory.cast_ptr_to_adr(node_vtable3)
     cpu = runner.LLGraphCPU(None)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -3,7 +3,8 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
      VArrayStateInfo, not_virtual, VirtualState,\
-     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
+     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo,\
+     VirtualStateConstructor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
      InputArgFloat
@@ -26,6 +27,7 @@
     def __init__(self, cpu):
         self.cpu = cpu
         self.optearlyforce = None
+        self.optimizer = Optimizer
 
 class BaseTestGenerateGuards(BaseTest):
     def setup_class(self):
@@ -87,6 +89,33 @@
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == []
 
+    def test_make_inputargs_2(self):
+        # Ensure that make_inputargs properly errors with VirtualStatesCantMatch
+        # when the type information for a virtual field conflicts. In practice the
+        # expected and given field always share a common subclass.
+        # This check is needed as not all paths to make_inputargs in unroll.py
+        # are guarded by a call to generate_guards.
+        optimizer = FakeOptimizer(self.cpu)
+        classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        innervalue1 = info.InstancePtrInfo(known_class=classbox1, is_virtual=True, descr=self.valuedescr.get_parent_descr())
+        for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
+            innervalue1.setfield(field, None, ConstInt(42))
+        classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3))
+        innervalue2 = info.InstancePtrInfo(known_class=classbox2, is_virtual=True, descr=self.valuedescr3.get_parent_descr())
+        for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
+            innervalue2.setfield(field, None, ConstInt(42))
+
+        nodebox1 = InputArgRef(self.nodeaddr)
+        nodebox2 = InputArgRef(self.myptr3)
+        nodebox1.set_forwarded(innervalue1)
+        nodebox2.set_forwarded(innervalue2)
+
+        constr = VirtualStateConstructor(optimizer)
+        vs = constr.get_virtual_state([nodebox1])
+
+        with py.test.raises(VirtualStatesCantMatch):
+            args = vs.make_inputargs([nodebox2], optimizer, force_boxes=True)
+
     def test_position_generalization(self):
         def postest(info1, info2):
             info1.position = 0
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -177,6 +177,14 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
+    @staticmethod
+    def descr_issubclass(descr1, descr2, optimizer):
+        if not descr1.is_object() or not descr2.is_object():
+            return True
+        vtable1 = descr1.get_vtable()
+        vtable2 = descr2.get_vtable()
+        return optimizer._check_subclass(vtable1, vtable2)
+
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
@@ -184,10 +192,14 @@
             raise VirtualStatesCantMatch()
         else:
             assert isinstance(info, AbstractStructPtrInfo)
+
         for i in range(len(self.fielddescrs)):
             state = self.fieldstate[i]
+            descr = self.fielddescrs[i].get_parent_descr()
             if not state:
                 continue
+            if not self.descr_issubclass(info.descr, descr, optimizer.optimizer):
+                raise VirtualStatesCantMatch()
             if state.position > self.position:
                 fieldbox = info._fields[i]
                 state.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -461,6 +461,8 @@
                    save_err=rffi.RFFI_SAVE_ERRNO)
 c_fdatasync = external('fdatasync', [rffi.INT], rffi.INT,
                        save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    c_sync = external('sync', [], lltype.Void)
 
 @replace_os_function('ftruncate')
 def ftruncate(fd, length):
@@ -477,6 +479,9 @@
     validate_fd(fd)
     handle_posix_error('fdatasync', c_fdatasync(fd))
 
+def sync():
+    c_sync()
+
 #___________________________________________________________________
 
 c_chdir = external('chdir', [rffi.CCHARP], rffi.INT,
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -604,3 +604,7 @@
     assert rposix.get_inheritable(fd1) == False
     os.close(fd1)
     os.close(fd2)
+
+def test_sync():
+    if sys.platform != 'win32':
+        rposix.sync()

From pypy.commits at gmail.com  Tue Sep 13 13:09:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 10:09:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57d832de.a3adc20a.3122b.1621@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87086:b88005ea869f
Date: 2016-09-13 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/b88005ea869f/

Log:	hg merge py3k

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
 from rpython.jit.metainterp.executor import execute
 from rpython.jit.metainterp.optimize import InvalidLoop
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -11,6 +11,8 @@
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.debug import debug_print
+from rpython.rtyper import rclass
+from rpython.rtyper.lltypesystem import llmemory
 from rpython.jit.metainterp.optimize import SpeculativeError
 
 
@@ -799,6 +801,21 @@
         if not (0 <= index < arraylength):
             raise SpeculativeError
 
+    @staticmethod
+    def _check_subclass(vtable1, vtable2): # checks that vtable1 is a subclass of vtable2
+        known_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable1),
+            rclass.CLASSTYPE)
+        expected_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable2),
+            rclass.CLASSTYPE)
+        # note: the test is for a range including 'max', but 'max'
+        # should never be used for actual classes.  Including it makes
+        # it easier to pass artificial tests.
+        return (expected_class.subclassrange_min
+                <= known_class.subclassrange_min
+                <= expected_class.subclassrange_max)
+
     def is_virtual(self, op):
         if op.type == 'r':
             opinfo = self.getptrinfo(op)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -324,37 +324,24 @@
             return
         self.emit_operation(op)
 
-    def _check_subclass(self, vtable1, vtable2):
-        # checks that vtable1 is a subclass of vtable2
-        known_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable1),
-            rclass.CLASSTYPE)
-        expected_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable2),
-            rclass.CLASSTYPE)
-        if (expected_class.subclassrange_min
-                <= known_class.subclassrange_min
-                <= expected_class.subclassrange_max):
-            return True
-        return False
-
     def optimize_GUARD_SUBCLASS(self, op):
         info = self.getptrinfo(op.getarg(0))
+        optimizer = self.optimizer
         if info and info.is_constant():
             c = self.get_box_replacement(op.getarg(0))
-            vtable = self.optimizer.cpu.ts.cls_of_box(c).getint()
-            if self._check_subclass(vtable, op.getarg(1).getint()):
+            vtable = optimizer.cpu.ts.cls_of_box(c).getint()
+            if optimizer._check_subclass(vtable, op.getarg(1).getint()):
                 return
             raise InvalidLoop("GUARD_SUBCLASS(const) proven to always fail")
         if info is not None and info.is_about_object():
-            known_class = info.get_known_class(self.optimizer.cpu)
+            known_class = info.get_known_class(optimizer.cpu)
             if known_class:
-                if self._check_subclass(known_class.getint(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(known_class.getint(),
+                                             op.getarg(1).getint()):
                     return
             elif info.get_descr() is not None:
-                if self._check_subclass(info.get_descr().get_vtable(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(info.get_descr().get_vtable(),
+                                             op.getarg(1).getint()):
                     return
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -17,12 +17,13 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
      VirtualStateInfo
-from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.optimizeopt import info, optimizer
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.tool import oparser
 
 class FakeOptimizer(object):
     optearlyforce = None
+    optimizer = optimizer.Optimizer
 
     class cpu:
         remove_gctypeptr = True
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -102,6 +102,8 @@
     node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2)
     node_vtable3 = lltype.malloc(OBJECT_VTABLE, immortal=True)
     node_vtable3.name = rclass.alloc_array_name('node3')
+    node_vtable3.subclassrange_min = 3
+    node_vtable3.subclassrange_max = 3
     node_vtable_adr3 = llmemory.cast_ptr_to_adr(node_vtable3)
     cpu = runner.LLGraphCPU(None)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -3,7 +3,8 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
      VArrayStateInfo, not_virtual, VirtualState,\
-     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
+     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo,\
+     VirtualStateConstructor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
      InputArgFloat
@@ -26,6 +27,7 @@
     def __init__(self, cpu):
         self.cpu = cpu
         self.optearlyforce = None
+        self.optimizer = Optimizer
 
 class BaseTestGenerateGuards(BaseTest):
     def setup_class(self):
@@ -87,6 +89,33 @@
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == []
 
+    def test_make_inputargs_2(self):
+        # Ensure that make_inputargs properly errors with VirtualStatesCantMatch
+        # when the type information for a virtual field conflicts. In practice the
+        # expected and given field always share a common subclass.
+        # This check is needed as not all paths to make_inputargs in unroll.py
+        # are guarded by a call to generate_guards.
+        optimizer = FakeOptimizer(self.cpu)
+        classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        innervalue1 = info.InstancePtrInfo(known_class=classbox1, is_virtual=True, descr=self.valuedescr.get_parent_descr())
+        for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
+            innervalue1.setfield(field, None, ConstInt(42))
+        classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3))
+        innervalue2 = info.InstancePtrInfo(known_class=classbox2, is_virtual=True, descr=self.valuedescr3.get_parent_descr())
+        for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
+            innervalue2.setfield(field, None, ConstInt(42))
+
+        nodebox1 = InputArgRef(self.nodeaddr)
+        nodebox2 = InputArgRef(self.myptr3)
+        nodebox1.set_forwarded(innervalue1)
+        nodebox2.set_forwarded(innervalue2)
+
+        constr = VirtualStateConstructor(optimizer)
+        vs = constr.get_virtual_state([nodebox1])
+
+        with py.test.raises(VirtualStatesCantMatch):
+            args = vs.make_inputargs([nodebox2], optimizer, force_boxes=True)
+
     def test_position_generalization(self):
         def postest(info1, info2):
             info1.position = 0
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -177,6 +177,14 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
+    @staticmethod
+    def descr_issubclass(descr1, descr2, optimizer):
+        if not descr1.is_object() or not descr2.is_object():
+            return True
+        vtable1 = descr1.get_vtable()
+        vtable2 = descr2.get_vtable()
+        return optimizer._check_subclass(vtable1, vtable2)
+
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
@@ -184,10 +192,14 @@
             raise VirtualStatesCantMatch()
         else:
             assert isinstance(info, AbstractStructPtrInfo)
+
         for i in range(len(self.fielddescrs)):
             state = self.fieldstate[i]
+            descr = self.fielddescrs[i].get_parent_descr()
             if not state:
                 continue
+            if not self.descr_issubclass(info.descr, descr, optimizer.optimizer):
+                raise VirtualStatesCantMatch()
             if state.position > self.position:
                 fieldbox = info._fields[i]
                 state.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -461,6 +461,8 @@
                    save_err=rffi.RFFI_SAVE_ERRNO)
 c_fdatasync = external('fdatasync', [rffi.INT], rffi.INT,
                        save_err=rffi.RFFI_SAVE_ERRNO)
+if not _WIN32:
+    c_sync = external('sync', [], lltype.Void)
 
 @replace_os_function('ftruncate')
 def ftruncate(fd, length):
@@ -477,6 +479,9 @@
     validate_fd(fd)
     handle_posix_error('fdatasync', c_fdatasync(fd))
 
+def sync():
+    c_sync()
+
 #___________________________________________________________________
 
 c_chdir = external('chdir', [rffi.CCHARP], rffi.INT,
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -604,3 +604,7 @@
     assert rposix.get_inheritable(fd1) == False
     os.close(fd1)
     os.close(fd2)
+
+def test_sync():
+    if sys.platform != 'win32':
+        rposix.sync()

From pypy.commits at gmail.com  Tue Sep 13 13:09:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 10:09:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: os.sync()
Message-ID: <57d832df.a2ddc20a.72320.0204@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87087:eb88834f82b2
Date: 2016-09-13 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/eb88834f82b2/

Log:	os.sync()

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -200,6 +200,9 @@
     if rposix.HAVE_PIPE2:
         interpleveldefs['pipe2'] = 'interp_posix.pipe2'
 
+    if not rposix._WIN32:
+        interpleveldefs['sync'] = 'interp_posix.sync'
+
     def startup(self, space):
         from pypy.module.posix import interp_posix
         from pypy.module.imp import importing
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -347,6 +347,10 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
+def sync(space):
+    """Force write of everything to disk."""
+    rposix.sync()
+
 def fchdir(space, w_fd):
     """Change to the directory of the given file descriptor.  fildes must be
 opened on a directory, not a file."""
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1134,6 +1134,10 @@
         assert os.get_inheritable(fd) == False
         os.close(fd)
 
+    if sys.platform != 'win32':
+        def test_sync(self):
+            self.posix.sync()   # does not raise
+
     def test_urandom(self):
         os = self.posix
         s = os.urandom(5)

From pypy.commits at gmail.com  Tue Sep 13 13:11:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 10:11:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: graft eb88834f82b2
Message-ID: <57d83353.4756c20a.dc392.d857@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87088:10c68ca3af89
Date: 2016-09-13 19:11 +0200
http://bitbucket.org/pypy/pypy/changeset/10c68ca3af89/

Log:	graft eb88834f82b2

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -193,6 +193,9 @@
     interpleveldefs['_have_functions'] = (
         'space.newlist([space.wrap(x) for x in interp_posix.have_functions])')
 
+    if not rposix._WIN32:
+        interpleveldefs['sync'] = 'interp_posix.sync'
+
     def startup(self, space):
         from pypy.module.posix import interp_posix
         interp_posix.get(space).startup(space)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -342,6 +342,10 @@
     except OSError as e:
         raise wrap_oserror(space, e)
 
+def sync(space):
+    """Force write of everything to disk."""
+    rposix.sync()
+
 def fchdir(space, w_fd):
     """Change to the directory of the given file descriptor.  fildes must be
 opened on a directory, not a file."""
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1079,6 +1079,10 @@
         x = f.read(1)
         assert x == 'e'
 
+    if sys.platform != 'win32':
+        def test_sync(self):
+            self.posix.sync()   # does not raise
+
     def test_urandom(self):
         os = self.posix
         s = os.urandom(5)

From pypy.commits at gmail.com  Tue Sep 13 13:23:29 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 13 Sep 2016 10:23:29 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Move import_module impl to sys_info
Message-ID: <57d83611.d42f1c0a.23f7a.0923@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87089:80b4b4aa0819
Date: 2016-09-13 17:50 +0100
http://bitbucket.org/pypy/pypy/changeset/80b4b4aa0819/

Log:	Move import_module impl to sys_info

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -55,7 +55,7 @@
         self.extra_libs = extra_libs
         self.ext = ext
 
-    def compile_extension_module(self, name, include_dirs=[],
+    def compile_extension_module(self, name, include_dirs=None,
             source_files=None, source_strings=None):
         """
         Build an extension module and return the filename of the resulting
@@ -67,6 +67,7 @@
         Any extra keyword arguments are passed on to ExternalCompilationInfo to
         build the module (so specify your source with one of those).
         """
+        include_dirs = include_dirs or []
         modname = name.split('.')[-1]
         dirname = (udir/uniquemodulename('module')).ensure(dir=1)
         if source_strings:
@@ -82,6 +83,32 @@
         soname.rename(pydname)
         return str(pydname)
 
+    def import_module(self, name, init=None, body='', filename=None,
+            include_dirs=None, PY_SSIZE_T_CLEAN=False):
+        """
+        init specifies the overall template of the module.
+
+        if init is None, the module source will be loaded from a file in this
+        test directory, give a name given by the filename parameter.
+
+        if filename is None, the module name will be used to construct the
+        filename.
+        """
+        if init is not None:
+            code = make_source(name, init, body, PY_SSIZE_T_CLEAN)
+            kwds = dict(source_strings=[code])
+        else:
+            assert not PY_SSIZE_T_CLEAN
+            if filename is None:
+                filename = name
+            filename = py.path.local(pypydir) / 'module' \
+                    / 'cpyext'/ 'test' / (filename + ".c")
+            kwds = dict(source_files=[filename])
+        mod = self.compile_extension_module(
+            name, include_dirs=include_dirs, **kwds)
+        return self.load_module(mod, name)
+
+
 class ExtensionCompiler(SystemCompilationInfo):
     """Extension compiler for appdirect mode"""
     def load_module(space, mod, name):
@@ -351,6 +378,13 @@
             skip("Windows Python >= 2.6 only")
         assert isinstance(sys.dllhandle, int)
 
+def _unwrap_include_dirs(space, w_include_dirs):
+    if w_include_dirs is None:
+        return None
+    else:
+        return [space.str_w(s) for s in space.listview(w_include_dirs)]
+
+
 class AppTestCpythonExtensionBase(LeakCheckingTest):
 
     def setup_class(cls):
@@ -405,34 +439,10 @@
         def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
                           PY_SSIZE_T_CLEAN=False):
-            """
-            init specifies the overall template of the module.
-
-            if init is None, the module source will be loaded from a file in this
-            test directory, give a name given by the filename parameter.
-
-            if filename is None, the module name will be used to construct the
-            filename.
-            """
-            if w_include_dirs is None:
-                include_dirs = []
-            else:
-                include_dirs = [space.str_w(s) for s in space.listview(w_include_dirs)]
-            if init is not None:
-                code = make_source(name, init, body, PY_SSIZE_T_CLEAN)
-                kwds = dict(source_strings=[code])
-            else:
-                assert not PY_SSIZE_T_CLEAN
-                if filename is None:
-                    filename = name
-                filename = py.path.local(pypydir) / 'module' \
-                        / 'cpyext'/ 'test' / (filename + ".c")
-                kwds = dict(source_files=[filename])
-            mod = self.sys_info.compile_extension_module(
-                name, include_dirs=include_dirs, **kwds)
-            w_result = self.sys_info.load_module(mod, name)
-            if not self.runappdirect:
-                self.record_imported_module(name)
+            include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+            w_result = self.sys_info.import_module(
+                name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN)
+            self.record_imported_module(name)
             return w_result
 
 
@@ -445,13 +455,14 @@
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
+            include_dirs = _unwrap_include_dirs(space, w_include_dirs)
             body = prologue + make_methods(functions, modname)
             init = """Py_InitModule("%s", methods);""" % (modname,)
             if more_init:
                 init += more_init
-            return import_module(space, name=modname, init=init, body=body,
-                                 w_include_dirs=w_include_dirs,
-                                 PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+            return self.sys_info.import_module(
+                name=modname, init=init, body=body, include_dirs=include_dirs,
+                PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
 
         def debug_collect(space):
             rawrefcount._collect()
@@ -474,13 +485,15 @@
                 return func
             self.sys_info = get_sys_info_app()
             self.compile_module = self.sys_info.compile_extension_module
+            self.load_module = self.sys_info.load_module
+            self.import_module = self.sys_info.import_module
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
             self.sys_info = get_cpyext_info(self.space)
             self.w_compile_module = wrap(interp2app(compile_module))
-        self.w_import_module = wrap(interp2app(import_module))
-        self.w_load_module = wrap(interp2app(load_module))
+            self.w_load_module = wrap(interp2app(load_module))
+            self.w_import_module = wrap(interp2app(import_module))
         self.w_import_extension = wrap(interp2app(import_extension))
         self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/')
         self.w_debug_collect = wrap(interp2app(debug_collect))

From pypy.commits at gmail.com  Tue Sep 13 13:23:30 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 13 Sep 2016 10:23:30 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Move import_extension impl to
 sys_info
Message-ID: <57d83612.274fc20a.d0260.fefe@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87090:8f2666e8e624
Date: 2016-09-13 18:22 +0100
http://bitbucket.org/pypy/pypy/changeset/8f2666e8e624/

Log:	Move import_extension impl to sys_info

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -108,6 +108,16 @@
             name, include_dirs=include_dirs, **kwds)
         return self.load_module(mod, name)
 
+        def import_extension(modname, functions, prologue="",
+                             include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
+            body = prologue + make_methods(functions, modname)
+            init = """Py_InitModule("%s", methods);""" % (modname,)
+            if more_init:
+                init += more_init
+            return self.import_module(
+                name=modname, init=init, body=body, include_dirs=include_dirs,
+                PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+
 
 class ExtensionCompiler(SystemCompilationInfo):
     """Extension compiler for appdirect mode"""
@@ -456,13 +466,11 @@
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
             include_dirs = _unwrap_include_dirs(space, w_include_dirs)
-            body = prologue + make_methods(functions, modname)
-            init = """Py_InitModule("%s", methods);""" % (modname,)
-            if more_init:
-                init += more_init
-            return self.sys_info.import_module(
-                name=modname, init=init, body=body, include_dirs=include_dirs,
-                PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+            w_result = self.sys_info.import_extension(
+                modname, functions, prologue, include_dirs, more_init,
+                PY_SSIZE_T_CLEAN)
+            self.record_imported_module(modname)
+            return w_result
 
         def debug_collect(space):
             rawrefcount._collect()
@@ -487,6 +495,7 @@
             self.compile_module = self.sys_info.compile_extension_module
             self.load_module = self.sys_info.load_module
             self.import_module = self.sys_info.import_module
+            self.import_extension = self.sys_info.import_extension
         else:
             interp2app = gateway.interp2app
             wrap = self.space.wrap
@@ -494,7 +503,7 @@
             self.w_compile_module = wrap(interp2app(compile_module))
             self.w_load_module = wrap(interp2app(load_module))
             self.w_import_module = wrap(interp2app(import_module))
-        self.w_import_extension = wrap(interp2app(import_extension))
+            self.w_import_extension = wrap(interp2app(import_extension))
         self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/')
         self.w_debug_collect = wrap(interp2app(debug_collect))
 

From pypy.commits at gmail.com  Tue Sep 13 13:31:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 10:31:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: SOMAXCONN, socket.listen(no argument)
Message-ID: <57d837f0.04a81c0a.67389.1708@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87091:22dc852ebf3b
Date: 2016-09-13 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/22dc852ebf3b/

Log:	SOMAXCONN, socket.listen(no argument)

diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -1,4 +1,6 @@
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rlib.rsocket import SOMAXCONN
+
 
 class Module(MixedModule):
 
@@ -12,6 +14,7 @@
         'herror'    :  'interp_socket.get_error(space, "herror")',
         'gaierror'  :  'interp_socket.get_error(space, "gaierror")',
         'timeout'   :  'interp_socket.get_error(space, "timeout")',
+        'SOMAXCONN' :  'space.wrap(%d)' % SOMAXCONN,
     }
 
     def startup(self, space):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -3,7 +3,7 @@
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rsocket import (
     RSocket, AF_INET, SOCK_STREAM, SocketError, SocketErrorWithErrno,
-    RSocketError
+    RSocketError, SOMAXCONN
 )
 from rpython.rtyper.lltypesystem import lltype, rffi
 
@@ -359,13 +359,16 @@
         return space.wrap(timeout)
 
     @unwrap_spec(backlog="c_int")
-    def listen_w(self, space, backlog):
+    def listen_w(self, space, backlog=min(SOMAXCONN, 128)):
         """listen(backlog)
 
         Enable a server to accept connections.  The backlog argument must be at
-        least 1; it specifies the number of unaccepted connection that the system
-        will allow before refusing new connections.
+        least 0 (if it is lower, it is set to 0); it specifies the number of
+        unaccepted connection that the system will allow before refusing new
+        connections. If not specified, a default reasonable value is chosen.
         """
+        if backlog < 0:
+            backlog = 0
         try:
             self.sock.listen(backlog)
         except SocketError as e:
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -584,6 +584,11 @@
         s.sendto(memoryview(b''), ('localhost', 9)) # Send to discard port.
         s.close()
 
+    def test_listen_default(self):
+        import _socket
+        _socket.socket().listen()
+        assert isinstance(_socket.SOMAXCONN, int)
+
     def test_unix_socket_connect(self):
         import _socket, os
         if not hasattr(_socket, 'AF_UNIX'):

From pypy.commits at gmail.com  Tue Sep 13 13:47:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 10:47:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement the -I option to pypy
Message-ID: <57d83bae.448e1c0a.913c8.1cde@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87092:29d2a8abaa5f
Date: 2016-09-13 19:46 +0200
http://bitbucket.org/pypy/pypy/changeset/29d2a8abaa5f/

Log:	Implement the -I option to pypy

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -12,6 +12,7 @@
 -h     : print this help message and exit (also --help)
 -i     : inspect interactively after running script; forces a prompt even
          if stdin does not appear to be a terminal; also PYTHONINSPECT=x
+-I     : isolate Python from the user's environment (implies -E and -s)
 -m mod : run library module as a script (terminates option list)
 -O     : skip assert statements; also PYTHONOPTIMIZE=x
 -OO    : remove docstrings when importing modules in addition to -O
@@ -349,6 +350,7 @@
     "bytes_warning",
     "quiet",
     "hash_randomization",
+    "isolated",
 )
 
 default_options = dict.fromkeys(
@@ -362,6 +364,11 @@
 def simple_option(options, name, iterargv):
     options[name] += 1
 
+def isolated_option(options, name, iterargv):
+    options[name] += 1
+    options["no_user_site"] += 1
+    options["ignore_environment"] += 1
+
 def c_option(options, runcmd, iterargv):
     options["run_command"] = runcmd
     return ['-c'] + list(iterargv)
@@ -388,6 +395,7 @@
     'B': (simple_option, 'dont_write_bytecode'),
     'd': (simple_option, 'debug'),
     'E': (simple_option, 'ignore_environment'),
+    'I': (isolated_option, 'isolated'),
     'i': (simple_option, 'interactive'),
     'O': (simple_option, 'optimize'),
     's': (simple_option, 'no_user_site'),
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -193,6 +193,7 @@
             ("ignore_environment", "-E", "1"),
             ("verbose", "-v", "1"),
             ("bytes_warning", "-b", "1"),
+            (["isolated", "no_user_site", "ignore_environment"], "-I", "1"),
         )
         for flag, opt, value in flags:
             if isinstance(flag, list):   # this is for inspect&interactive

From pypy.commits at gmail.com  Tue Sep 13 14:00:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 11:00:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: audioop.byteswap()
Message-ID: <57d83ed7.02c51c0a.cdea0.1085@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87093:df95ced30bec
Date: 2016-09-13 20:00 +0200
http://bitbucket.org/pypy/pypy/changeset/df95ced30bec/

Log:	audioop.byteswap()

diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -578,3 +578,18 @@
     lib.adcpm2lin(rv, cp, len(cp), size, state_ptr)
     return ffi.buffer(rv)[:], tuple(state_ptr)
 
+
+def byteswap(cp, size):
+    if len(cp) % size != 0:
+        raise error("not a whole number of frames")
+    sample_count = _sample_count(cp, size)
+    rv = ffi.new("unsigned char[]", len(cp))
+    base = size
+    next_bump = 0
+    bump = 2 * size
+    for i in range(len(cp)):
+        base -= 1
+        rv[i] = cp[base]
+        if base == next_bump:
+            base += bump
+    return ffi.buffer(rv)[:]

From pypy.commits at gmail.com  Tue Sep 13 14:14:25 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 13 Sep 2016 11:14:25 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: more detailed benchmarks,
 used for paper
Message-ID: <57d84201.e97bc20a.d2809.f1e4@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r87094:0477318b132d
Date: 2016-09-13 09:27 -0700
http://bitbucket.org/pypy/pypy/changeset/0477318b132d/

Log:	more detailed benchmarks, used for paper

diff --git a/pypy/module/cppyy/test/bench1.py b/pypy/module/cppyy/test/bench1.py
--- a/pypy/module/cppyy/test/bench1.py
+++ b/pypy/module/cppyy/test/bench1.py
@@ -1,5 +1,6 @@
-import commands, os, sys, time
+import commands, os, sys, time, math
 
+from math import atan
 NNN = 10000000
 
 
@@ -18,36 +19,69 @@
     print ':::: %s cost: %#6.3fs (%#4.1fx)' % (name, t_bench, float(t_bench)/t_cppref)
 
 def python_loop_offset():
-    for i in range(NNN):
+    for i in xrange(NNN):
         i
     return i
 
-class PyCintexBench1(object):
-    scale = 5
+class CPythonBench1(object):
+    scale = 1
     def __init__(self):
-        import PyCintex
-        self.lib = PyCintex.gbl.gSystem.Load("./example01Dict.so")
+        import ROOT
+        ROOT.gROOT.SetBatch(1)
+        ROOT.SetSignalPolicy(ROOT.kSignalFast)
+        import cppyy
+        self.lib = cppyy.gbl.gSystem.Load("./example01Dict.so")
 
-        self.cls   = PyCintex.gbl.example01
+        self.cls   = cppyy.gbl.example01
         self.inst  = self.cls(0)
 
     def __call__(self, repeat):
-        # note that PyCintex calls don't actually scale linearly, but worse
-        # than linear (leak or wrong filling of a cache??)
+        # TODO: check linearity of actual scaling
         instance = self.inst
         niter = repeat/self.scale
-        for i in range(niter):
+        self.cls.addDataToInt._threaded = True
+        for i in xrange(niter):
             instance.addDataToInt(i)
         return i
 
-class PyROOTBench1(PyCintexBench1):
+class CPythonBench1_Swig(object):
+    scale = 1
     def __init__(self):
-        import ROOT
-        self.lib = ROOT.gSystem.Load("./example01Dict.so")
+        import example
 
-        self.cls   = ROOT.example01
+        self.cls   = example.example01
         self.inst  = self.cls(0)
 
+    def __call__(self, repeat):
+        # TODO: check linearity of actual scaling
+        instance = self.inst
+        niter = repeat/self.scale
+        for i in xrange(niter):
+            instance.addDataToInt(i)
+        return i
+
+
+class PureBench1(object):
+    scale = 1
+    def __init__(self):
+        class example01(object):
+            def __init__(self, somedata):
+                self.m_somedata = somedata
+            def addDataToInt(self, a):
+                return self.m_somedata + int(atan(a))
+
+        self.cls   = example01
+        self.inst  = self.cls(0)
+
+    def __call__(self, repeat):
+        # TODO: check linearity of actual scaling
+        instance = self.inst
+        niter = repeat/self.scale
+        for i in xrange(niter):
+            instance.addDataToInt(i)
+        return i
+
+
 class CppyyInterpBench1(object):
     title = "cppyy interp"
     scale = 1
@@ -61,7 +95,7 @@
     def __call__(self, repeat):
         addDataToInt = self.cls.get_overload("addDataToInt")
         instance = self.inst
-        for i in range(repeat):
+        for i in xrange(repeat):
             addDataToInt.call(instance, i)
         return i
 
@@ -70,7 +104,7 @@
     def __call__(self, repeat):
         addDataToInt = self.cls.get_overload("overloadedAddDataToInt")
         instance = self.inst
-        for i in range(repeat):
+        for i in xrange(repeat):
             addDataToInt.call(instance, i)
         return i
 
@@ -79,7 +113,7 @@
     def __call__(self, repeat):
         addDataToInt = self.cls.get_overload("addDataToIntConstRef")
         instance = self.inst
-        for i in range(repeat):
+        for i in xrange(repeat):
             addDataToInt.call(instance, i)
         return i
 
@@ -95,7 +129,7 @@
 
     def __call__(self, repeat):
         instance = self.inst
-        for i in range(repeat):
+        for i in xrange(repeat):
             instance.addDataToInt(i)
         return i
 
@@ -106,7 +140,7 @@
         pl = cppyy.gbl.payload(3.14)
 
         instance = self.inst
-        for i in range(repeat):
+        for i in xrange(repeat):
             instance.copyCyclePayload(pl)
         return i
 
@@ -117,7 +151,7 @@
         pl = cppyy.gbl.payload(3.14)
 
         instance = self.inst
-        for i in range(repeat):
+        for i in xrange(repeat):
             instance.cyclePayload(pl)
         return i
 
@@ -131,27 +165,36 @@
     t2 = time.time()
     t_loop_offset = t2-t1
 
-    # special case for PyCintex (run under python, not pypy-c)
-    if '--pycintex' in sys.argv:
-        cintex_bench1 = PyCintexBench1()
-        print run_bench(cintex_bench1)
+    # special cases for CPython
+    if '-swig' in sys.argv:
+        # runs SWIG
+        cpython_bench1 = CPythonBench1_Swig()
+    elif '-pure' in sys.argv:
+        # runs pure python
+        cpython_bench1 = PureBench1()
+    elif not 'cppyy' in sys.builtin_module_names:
+        # runs ROOT/cppyy.py
+        cpython_bench1 = CPythonBench1()
+    try:
+        print run_bench(cpython_bench1)
         sys.exit(0)
-
-    # special case for PyROOT (run under python, not pypy-c)
-    if '--pyroot' in sys.argv:
-        pyroot_bench1 = PyROOTBench1()
-        print run_bench(pyroot_bench1)
-        sys.exit(0)
+    except NameError:
+        pass
 
     # get C++ reference point
     if not os.path.exists("bench1.exe") or\
             os.stat("bench1.exe").st_mtime < os.stat("bench1.cxx").st_mtime:
         print "rebuilding bench1.exe ... "
-        os.system( "g++ -O2 bench1.cxx example01.cxx -o bench1.exe" )
+        # the following is debatable, as pypy-c uses direct function
+        # pointers, whereas that is only true for virtual functions in
+        # the case of C++ (by default, anyway, it need not)
+        # yes, shared library use is what's going on ...
+#        os.system( "g++ -O2 bench1.cxx example01.cxx -o bench1.exe" )
+        os.system( "g++ -O2 bench1.cxx -L. -lexample01Dict -o bench1.exe" )
     stat, cppref = commands.getstatusoutput("./bench1.exe")
     t_cppref = float(cppref)
 
-    # created object
+    # create object
     benches = [
         CppyyInterpBench1(), CppyyInterpBench2(), CppyyInterpBench3(),
         CppyyPythonBench1(), CppyyPythonBench2(), CppyyPythonBench3() ]
@@ -167,7 +210,35 @@
     # test runs ...
     for bench in benches:
         print_bench(bench.title, run_bench(bench))
-    stat, t_cintex = commands.getstatusoutput("python bench1.py --pycintex")
-    print_bench("pycintex    ", float(t_cintex))
-    #stat, t_pyroot = commands.getstatusoutput("python bench1.py --pyroot")
-    #print_bench("pyroot      ", float(t_pyroot))
+
+    stat, t_cpython1 = commands.getstatusoutput("/home/wlav/aditi/pypy/bin/v5/pypy-c bench1.py - -pure")
+    if stat:
+        print 'CPython pure bench1 failed:'
+        os.write(sys.stdout.fileno(), t_cpython1)
+        print
+        exit(stat)
+    print_bench("pypy-c pure ", float(t_cpython1))
+
+    stat, t_cpython1 = commands.getstatusoutput("python bench1.py - -pure")
+    if stat:
+        print 'CPython pure bench1 failed:'
+        os.write(sys.stdout.fileno(), t_cpython1)
+        print
+        exit(stat)
+    print_bench("CPython pure", float(t_cpython1))
+
+    stat, t_cpython1 = commands.getstatusoutput("python bench1.py - -b")
+    if stat:
+        print 'CPython bench1 failed:'
+        os.write(sys.stdout.fileno(), t_cpython1)
+        print
+        exit(stat)
+    print_bench("CPython     ", float(t_cpython1))
+
+    #stat, t_cpython1 = commands.getstatusoutput("python bench1.py - -swig")
+    #if stat:
+    #    print 'SWIG bench1 failed:'
+    #    os.write(sys.stdout.fileno(), t_cpython1)
+    #    print
+    #    exit(stat)
+    #print_bench("SWIG        ", float(t_cpython1))

From pypy.commits at gmail.com  Tue Sep 13 14:14:27 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 13 Sep 2016 11:14:27 -0700 (PDT)
Subject: [pypy-commit] pypy cling-support: optimization as used in the paper
Message-ID: <57d84203.8f081c0a.6f895.5638@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r87095:ff0990bd1f29
Date: 2016-09-13 11:09 -0700
http://bitbucket.org/pypy/pypy/changeset/ff0990bd1f29/

Log:	optimization as used in the paper

diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py
--- a/pypy/module/cppyy/capi/__init__.py
+++ b/pypy/module/cppyy/capi/__init__.py
@@ -9,8 +9,8 @@
 # the selection of the desired backend (default is Reflex).
 
 # choose C-API access method:
-#from pypy.module.cppyy.capi.loadable_capi import *
-from pypy.module.cppyy.capi.builtin_capi import *
+from pypy.module.cppyy.capi.loadable_capi import *
+#from pypy.module.cppyy.capi.builtin_capi import *
 
 from pypy.module.cppyy.capi.capi_types import C_OBJECT,\
     C_NULL_TYPE, C_NULL_OBJECT
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -190,7 +190,6 @@
     [C_SCOPE, C_INDEX], C_FUNC_PTR,
     releasegil=ts_reflect,
     compilation_info=backend.eci,
-    elidable_function=True,
     random_effects_on_gcobjs=False)
 def c_get_function_address(space, cppscope, index):
     return _c_get_function_address(cppscope.handle, index)
@@ -215,8 +214,8 @@
     [], rffi.SIZE_T,
     releasegil=ts_memory,
     compilation_info=backend.eci,
-    elidable_function=True,
     random_effects_on_gcobjs=False)
+ at jit.elidable
 def c_function_arg_sizeof(space):
     return _c_function_arg_sizeof()
 _c_function_arg_typeoffset = rffi.llexternal(
@@ -224,8 +223,8 @@
     [], rffi.SIZE_T,
     releasegil=ts_memory,
     compilation_info=backend.eci,
-    elidable_function=True,
     random_effects_on_gcobjs=False)
+ at jit.elidable
 def c_function_arg_typeoffset(space):
     return _c_function_arg_typeoffset()
 
@@ -300,9 +299,8 @@
     [C_TYPE, C_TYPE], rffi.INT,
     releasegil=ts_reflect,
     compilation_info=backend.eci,
-    elidable_function=True,
     random_effects_on_gcobjs=False)
- at jit.elidable_promote('2')
+ at jit.elidable
 def c_is_subtype(space, derived, base):
     if derived == base:
         return 1
@@ -313,9 +311,8 @@
     [C_TYPE, C_TYPE, C_OBJECT, rffi.INT], rffi.SIZE_T,
     releasegil=ts_reflect,
     compilation_info=backend.eci,
-    elidable_function=True,
     random_effects_on_gcobjs=False)
- at jit.elidable_promote('1,2,4')
+ at jit.elidable
 def c_base_offset(space, derived, base, address, direction):
     if derived == base:
         return 0
@@ -564,3 +561,26 @@
     compilation_info=backend.eci)
 def c_stdstring2stdstring(space, cppobject):
     return _c_stdstring2stdstring(cppobject)
+
+_c_stdvector_valuetype = rffi.llexternal(
+    "cppyy_stdvector_valuetype",
+    [rffi.CCHARP], rffi.CCHARP,
+    releasegil=ts_helper,
+    compilation_info=backend.eci)
+def c_stdvector_valuetype(space, pystr):
+    cstr = rffi.str2charp(pystr)
+    result = _c_stdvector_valuetype(cstr)
+    rffi.free_charp(cstr)
+    if result:
+        return charp2str_free(space, result)
+    return ""
+_c_stdvector_valuesize = rffi.llexternal(
+    "cppyy_stdvector_valuesize",
+    [rffi.CCHARP], rffi.SIZE_T,
+    releasegil=ts_helper,
+    compilation_info=backend.eci)
+def c_stdvector_valuesize(space, pystr):
+    cstr = rffi.str2charp(pystr)
+    result = _c_stdvector_valuesize(cstr)
+    rffi.free_charp(cstr)
+    return result
diff --git a/pypy/module/cppyy/capi/cling_capi.py b/pypy/module/cppyy/capi/cling_capi.py
--- a/pypy/module/cppyy/capi/cling_capi.py
+++ b/pypy/module/cppyy/capi/cling_capi.py
@@ -1,12 +1,16 @@
 import py, os
 
+from pypy.objspace.std.iterobject import W_AbstractSeqIterObject
+
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app
 
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import libffi, rdynload
+from rpython.rlib import jit, libffi, rdynload
 
+from pypy.module._rawffi.array import W_ArrayInstance
 from pypy.module.cppyy.capi.capi_types import C_OBJECT
 
 __all__ = ['identify', 'std_string_name', 'eci', 'c_load_dictionary']
@@ -89,6 +93,9 @@
 
 # TODO: factor these out ...
 # pythonizations
+
+#
+# std::string behavior
 def stdstring_c_str(space, w_self):
     """Return a python string taking into account \0"""
 
@@ -96,6 +103,60 @@
     cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
     return space.wrap(c_stdstring2charp(space, cppstr._rawobject))
 
+#
+# std::vector behavior
+class W_STLVectorIter(W_AbstractSeqIterObject):
+    _immutable_fields_ = ['overload', 'len']#'data', 'converter', 'len', 'stride', 'vector']
+
+    def __init__(self, space, w_vector):
+        W_AbstractSeqIterObject.__init__(self, w_vector)
+        # TODO: this should live in rpythonize.py or something so that the
+        # imports can move to the top w/o getting circles
+        from pypy.module.cppyy import interp_cppyy
+        assert isinstance(w_vector, interp_cppyy.W_CPPInstance)
+        vector = space.interp_w(interp_cppyy.W_CPPInstance, w_vector)
+        self.overload = vector.cppclass.get_overload("__getitem__")
+
+        from pypy.module.cppyy import capi
+        v_type = capi.c_stdvector_valuetype(space, vector.cppclass.name)
+        v_size = capi.c_stdvector_valuesize(space, vector.cppclass.name)
+
+        if not v_type or not v_size:
+            raise NotImplementedError   # fallback on getitem
+
+        w_arr = vector.cppclass.get_overload("data").call(w_vector, [])
+        arr = space.interp_w(W_ArrayInstance, w_arr, can_be_None=True)
+        if not arr:
+            raise OperationError(space.w_StopIteration, space.w_None)
+
+        self.data = rffi.cast(rffi.VOIDP, space.uint_w(arr.getbuffer(space)))
+
+        from pypy.module.cppyy import converter
+        self.converter = converter.get_converter(space, v_type, '')
+        self.len     = space.uint_w(vector.cppclass.get_overload("size").call(w_vector, []))
+        self.stride  = v_size
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        if self.len <= self.index:
+            self.w_seq = None
+            raise OperationError(space.w_StopIteration, space.w_None)
+        try:
+            from pypy.module.cppyy import capi    # TODO: refector
+            offset = capi.direct_ptradd(rffi.cast(C_OBJECT, self.data), self.index*self.stride)
+            w_item = self.converter.from_memory(space, space.w_None, space.w_None, offset)
+        except OperationError as e:
+            self.w_seq = None
+            if not e.match(space, space.w_IndexError):
+                raise
+            raise OperationError(space.w_StopIteration, space.w_None)
+        self.index += 1
+        return w_item
+
+def stdvector_iter(space, w_self):
+    return W_STLVectorIter(space, w_self)
+
 # setup pythonizations for later use at run-time
 _pythonizations = {}
 def register_pythonizations(space):
@@ -106,6 +167,9 @@
         ### std::string
         stdstring_c_str,
 
+        ### std::vector
+        stdvector_iter,
+
     ]
 
     for f in allfuncs:
@@ -120,3 +184,13 @@
         space.setattr(w_pycppclass, space.wrap("c_str"), _pythonizations["stdstring_c_str"])
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
         _method_alias(space, w_pycppclass, "__str__",           "c_str")
+
+    if "vector" in name[:11]: # len('std::vector') == 11
+        from pypy.module.cppyy import capi
+        v_type = capi.c_stdvector_valuetype(space, name)
+        if v_type:
+            space.setattr(w_pycppclass, space.wrap("value_type"), space.wrap(v_type))
+        v_size = capi.c_stdvector_valuesize(space, name)
+        if v_size:
+            space.setattr(w_pycppclass, space.wrap("value_size"), space.wrap(v_size))
+        space.setattr(w_pycppclass, space.wrap("__iter__"), _pythonizations["stdvector_iter"])
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -225,6 +225,10 @@
             #stdstring2charp  actually takes an size_t* as last parameter, but this will do
             'stdstring2charp'          : ([c_object, c_voidp],        c_ccharp),
             'stdstring2stdstring'      : ([c_object],                 c_object),
+
+            'stdvector_valuetype'      : ([c_ccharp],                 c_ccharp),
+            'stdvector_valuesize'      : ([c_ccharp],                 c_size_t),
+
         }
 
         # size/offset are backend-specific but fixed after load
@@ -550,6 +554,14 @@
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_Arg(h=cppobject)]))
 
+def c_stdvector_valuetype(space, pystr):
+    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_Arg(s=pystr)]))
+
+def c_stdvector_valuetype(space, pystr):
+    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_Arg(s=pystr)]))
+def c_stdvector_valuesize(space, pystr):
+    return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_Arg(s=pystr)]))
+
 
 # TODO: factor these out ...
 # pythonizations
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -183,6 +183,11 @@
     RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
+    RPY_EXTERN
+    const char* cppyy_stdvector_valuetype(const char* clname);
+    RPY_EXTERN
+    size_t cppyy_stdvector_valuesize(const char* clname);
+
 #ifdef __cplusplus
 }
 #endif // ifdef __cplusplus
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -197,6 +197,8 @@
 
     @jit.unroll_safe
     def call(self, cppthis, args_w):
+        jit.promote(self)
+
         assert lltype.typeOf(cppthis) == capi.C_OBJECT
 
         # check number of given arguments against required (== total - defaults)
@@ -325,8 +327,8 @@
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
-        self._funcaddr = capi.c_get_function_address(self.space, self.scope, self.index)
-        if self._funcaddr and cppthis:      # methods only for now
+        funcaddr = capi.c_get_function_address(self.space, self.scope, self.index)
+        if funcaddr and cppthis:      # methods only for now
             state = self.space.fromcache(ffitypes.State)
 
             # argument type specification (incl. cppthis)
@@ -353,6 +355,9 @@
                     self.cif_descr = lltype.nullptr(jit_libffi.CIF_DESCRIPTION)
                 raise FastCallNotPossible
 
+            # success ...
+            self._funcaddr = funcaddr
+
     @jit.unroll_safe
     def prepare_arguments(self, args_w, call_local):
         args = capi.c_allocate_function_args(self.space, len(args_w))
@@ -828,6 +833,9 @@
             arg_defs.append((arg_type, arg_dflt))
         return CPPFunction(self.space, self, index, arg_defs, args_required)
 
+    def _build_methods(self):
+        pass       # force lazy lookups in namespaces
+
     def _make_datamember(self, dm_name, dm_idx):
         type_name = capi.c_datamember_type(self.space, self, dm_idx)
         offset = capi.c_datamember_offset(self.space, self, dm_idx)
diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -311,6 +311,7 @@
     else:
         return python_style_getitem(self, slice_or_idx)
 
+
 _pythonizations = {}
 def _pythonize(pyclass):
 
@@ -357,8 +358,8 @@
         pyclass.__iadd__ = __iadd__
 
     # map begin()/end() protocol to iter protocol on STL(-like) classes, but
-    # not on vector, for which otherwise the user has to make sure that the
-    # global == and != for its iterators are reflected, which is a hassle ...
+    # not on vector, which is pythonized in the capi (interp-level; there is
+    # also the fallback on the indexed __getitem__, but that is slower)
     if not 'vector' in pyclass.__name__[:11] and \
             ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
         if cppyy._scope_byname(pyclass.__name__+'::iterator') or \
diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx
--- a/pypy/module/cppyy/src/clingcwrapper.cxx
+++ b/pypy/module/cppyy/src/clingcwrapper.cxx
@@ -1540,4 +1540,24 @@
     return (cppyy_object_t)new std::string(*(std::string*)ptr);
 }
 
+const char* cppyy_stdvector_valuetype(const char* clname) {
+    const char* result = nullptr;
+    std::string name = clname;
+    TypedefInfo_t* ti = gInterpreter->TypedefInfo_Factory((name+"::value_type").c_str());
+    if (gInterpreter->TypedefInfo_IsValid(ti))
+        result = cppstring_to_cstring(gInterpreter->TypedefInfo_TrueName(ti));
+    gInterpreter->TypedefInfo_Delete(ti);
+    return result;
+}
+
+size_t cppyy_stdvector_valuesize(const char* clname) {
+    size_t result = 0;
+    std::string name = clname;
+    TypedefInfo_t* ti = gInterpreter->TypedefInfo_Factory((name+"::value_type").c_str());
+    if (gInterpreter->TypedefInfo_IsValid(ti))
+       result = (size_t)gInterpreter->TypedefInfo_Size(ti);
+    gInterpreter->TypedefInfo_Delete(ti);
+    return result;
+}
+   
 } // end C-linkage wrappers

From pypy.commits at gmail.com  Tue Sep 13 14:20:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 11:20:16 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: done
Message-ID: <57d84360.0575c20a.231dd.04ab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5703:358bf7e9fc12
Date: 2016-09-13 20:20 +0200
http://bitbucket.org/pypy/extradoc/changeset/358bf7e9fc12/

Log:	done

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -42,8 +42,6 @@
   accepting any object whatsoever(?!), which is supposedly a feature
   (see http://bugs.python.org/issue14705).  DONE
 
-* ``math.isclose()``
-
 * ``KeyError('pip.exceptions',) in weakref callback <function
   _get_module_lock.<locals>.cb at 0x00007f118e2c0020> ignored``
   we're getting them now on start-up, investigate

From pypy.commits at gmail.com  Tue Sep 13 14:20:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 11:20:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: math.isclose()
Message-ID: <57d84366.0205c20a.92e4b.04c8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87096:0523f9c54a03
Date: 2016-09-13 20:19 +0200
http://bitbucket.org/pypy/pypy/changeset/0523f9c54a03/

Log:	math.isclose()

diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py
--- a/pypy/module/math/__init__.py
+++ b/pypy/module/math/__init__.py
@@ -52,5 +52,6 @@
        'erfc'           : 'interp_math.erfc',
        'gamma'          : 'interp_math.gamma',
        'lgamma'         : 'interp_math.lgamma',
+       'isclose'        : 'interp_math.isclose',
 }
 
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -3,6 +3,7 @@
 
 from rpython.rlib import rfloat
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 
 class State:
     def __init__(self, space):
@@ -424,3 +425,53 @@
     """Compute the natural logarithm of the gamma function for x."""
     return math1(space, rfloat.lgamma, w_x)
 
+ at unwrap_spec(w_rel_tol=WrappedDefault(1e-09), w_abs_tol=WrappedDefault(0.0))
+def isclose(space, w_a, w_b, __kwonly__, w_rel_tol, w_abs_tol):
+    """isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) -> bool
+
+Determine whether two floating point numbers are close in value.
+
+   rel_tol
+       maximum difference for being considered "close", relative to the
+       magnitude of the input values
+   abs_tol
+       maximum difference for being considered "close", regardless of the
+       magnitude of the input values
+
+Return True if a is close in value to b, and False otherwise.
+
+For the values to be considered close, the difference between them
+must be smaller than at least one of the tolerances.
+
+-inf, inf and NaN behave similarly to the IEEE 754 Standard.  That
+is, NaN is not close to anything, even itself.  inf and -inf are
+only close to themselves."""
+    a = _get_double(space, w_a)
+    b = _get_double(space, w_b)
+    rel_tol = _get_double(space, w_rel_tol)
+    abs_tol = _get_double(space, w_abs_tol)
+    #
+    # sanity check on the inputs
+    if rel_tol < 0.0 or abs_tol < 0.0:
+        raise oefmt(space.w_ValueError, "tolerances must be non-negative")
+    #
+    # short circuit exact equality -- needed to catch two infinities of
+    # the same sign. And perhaps speeds things up a bit sometimes.
+    if a == b:
+        return space.w_True
+    #
+    # This catches the case of two infinities of opposite sign, or
+    # one infinity and one finite number. Two infinities of opposite
+    # sign would otherwise have an infinite relative tolerance.
+    # Two infinities of the same sign are caught by the equality check
+    # above.
+    if rfloat.isinf(a) or rfloat.isinf(b):
+        return space.w_False
+    #
+    # now do the regular computation
+    # this is essentially the "weak" test from the Boost library
+    diff = math.fabs(b - a)
+    result = ((diff <= math.fabs(rel_tol * b) or
+               diff <= math.fabs(rel_tol * a)) or
+              diff <= abs_tol)
+    return space.newbool(result)
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -343,3 +343,23 @@
         assert round(math.lgamma(6.0), 9) == round(math.log(120.0), 9)
         assert raises(ValueError, math.gamma, -1)
         assert round(math.lgamma(0.5), 9) == round(math.log(math.pi ** 0.5), 9)
+
+    def test_isclose(self):
+        import math
+        assert math.isclose(0, 1) is False
+        assert math.isclose(0, 0.0) is True
+        assert math.isclose(1000.1, 1000.2, abs_tol=0.2) is True
+        assert math.isclose(1000.1, 1000.2, rel_tol=1e-3) is True
+        assert math.isclose(1000.1, 1000.2, abs_tol=0.02) is False
+        assert math.isclose(1000.1, 1000.2, rel_tol=1e-5) is False
+        assert math.isclose(float("inf"), float("inf")) is True
+        assert math.isclose(float("-inf"), float("-inf")) is True
+        assert math.isclose(float("inf"), float("-inf")) is False
+        assert math.isclose(float("-inf"), float("inf")) is False
+        assert math.isclose(float("-inf"), 12.34) is False
+        assert math.isclose(float("-inf"), float("nan")) is False
+        assert math.isclose(float("nan"), 12.34) is False
+        assert math.isclose(float("nan"), float("nan")) is False
+        #
+        raises(TypeError, math.isclose, 0, 1, rel_tol=None)
+        raises(TypeError, math.isclose, 0, 1, abs_tol=None)

From pypy.commits at gmail.com  Tue Sep 13 14:29:36 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 13 Sep 2016 11:29:36 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: fix
Message-ID: <57d84590.4abf1c0a.f00b7.5d17@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87097:0667f8e6dec8
Date: 2016-09-13 18:36 +0100
http://bitbucket.org/pypy/pypy/changeset/0667f8e6dec8/

Log:	fix

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -108,15 +108,15 @@
             name, include_dirs=include_dirs, **kwds)
         return self.load_module(mod, name)
 
-        def import_extension(modname, functions, prologue="",
-                             include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
-            body = prologue + make_methods(functions, modname)
-            init = """Py_InitModule("%s", methods);""" % (modname,)
-            if more_init:
-                init += more_init
-            return self.import_module(
-                name=modname, init=init, body=body, include_dirs=include_dirs,
-                PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+    def import_extension(self, modname, functions, prologue="",
+            include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
+        body = prologue + make_methods(functions, modname)
+        init = """Py_InitModule("%s", methods);""" % (modname,)
+        if more_init:
+            init += more_init
+        return self.import_module(
+            name=modname, init=init, body=body, include_dirs=include_dirs,
+            PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
 
 
 class ExtensionCompiler(SystemCompilationInfo):

From pypy.commits at gmail.com  Tue Sep 13 14:29:38 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 13 Sep 2016 11:29:38 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Stop using StdObjSpace and
 FakeSpace for -A tests
Message-ID: <57d84592.a970c20a.1ed82.0620@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87098:5c5e9537dcee
Date: 2016-09-13 19:28 +0100
http://bitbucket.org/pypy/pypy/changeset/5c5e9537dcee/

Log:	Stop using StdObjSpace and FakeSpace for -A tests

diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -2,6 +2,8 @@
 import pytest
 
 def pytest_configure(config):
+    if config.option.runappdirect:
+        return
     from pypy.tool.pytest.objspace import gettestobjspace
     # For some reason (probably a ll2ctypes cache issue on linux64)
     # it's necessary to run "import time" at least once before any
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -5,7 +5,7 @@
 import py, pytest
 
 from pypy import pypydir
-from pypy.interpreter import gateway
+from pypy.interpreter.gateway import unwrap_spec, interp2app
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
@@ -18,6 +18,7 @@
 
 from .support import c_compile
 
+HERE = py.path.local(pypydir) / 'module' / 'cpyext' / 'test'
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 @api.cpython_api([], api.PyObject)
@@ -101,8 +102,7 @@
             assert not PY_SSIZE_T_CLEAN
             if filename is None:
                 filename = name
-            filename = py.path.local(pypydir) / 'module' \
-                    / 'cpyext'/ 'test' / (filename + ".c")
+            filename = HERE / (filename + ".c")
             kwds = dict(source_files=[filename])
         mod = self.compile_extension_module(
             name, include_dirs=include_dirs, **kwds)
@@ -253,23 +253,6 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
-class FakeSpace(object):
-    """Like TinyObjSpace, but different"""
-    def __init__(self, config):
-        self.config = config
-
-    def passthrough(self, arg):
-        return arg
-    listview = passthrough
-    str_w = passthrough
-    wrap = passthrough
-
-    def unwrap(self, args):
-        try:
-            return args.str_w(None)
-        except:
-            return args
-
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -360,9 +343,12 @@
             cls.w_libc = cls.space.wrap(get_libc_name())
 
     def setup_method(self, meth):
-        freeze_refcnts(self)
+        if not self.runappdirect:
+            freeze_refcnts(self)
 
     def teardown_method(self, meth):
+        if self.runappdirect:
+            return
         self.cleanup_references(self.space)
         # XXX: like AppTestCpythonExtensionBase.teardown_method:
         # find out how to disable check_and_print_leaks() if the
@@ -388,25 +374,30 @@
             skip("Windows Python >= 2.6 only")
         assert isinstance(sys.dllhandle, int)
 
+
 def _unwrap_include_dirs(space, w_include_dirs):
     if w_include_dirs is None:
         return None
     else:
         return [space.str_w(s) for s in space.listview(w_include_dirs)]
 
+def debug_collect(space):
+    rawrefcount._collect()
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
 
     def setup_class(cls):
         space = cls.space
-        space.getbuiltinmodule("cpyext")
-        # 'import os' to warm up reference counts
-        w_import = space.builtin.getdictvalue(space, '__import__')
-        space.call_function(w_import, space.wrap("os"))
-        #state = cls.space.fromcache(RefcountState) ZZZ
-        #state.non_heaptypes_w[:] = []
+        cls.w_here = space.wrap(str(HERE))
         if not cls.runappdirect:
             cls.w_runappdirect = space.wrap(cls.runappdirect)
+            space.getbuiltinmodule("cpyext")
+            # 'import os' to warm up reference counts
+            w_import = space.builtin.getdictvalue(space, '__import__')
+            space.call_function(w_import, space.wrap("os"))
+            #state = cls.space.fromcache(RefcountState) ZZZ
+            #state.non_heaptypes_w[:] = []
+            cls.w_debug_collect = space.wrap(interp2app(debug_collect))
 
     def record_imported_module(self, name):
         """
@@ -418,7 +409,7 @@
         self.imported_module_names.append(name)
 
     def setup_method(self, func):
-        @gateway.unwrap_spec(name=str)
+        @unwrap_spec(name=str)
         def compile_module(space, name,
                            w_source_files=None,
                            w_source_strings=None):
@@ -444,7 +435,7 @@
 
             return space.wrap(pydname)
 
-        @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
+        @unwrap_spec(name=str, init='str_or_None', body=str,
                      filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
         def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
@@ -456,11 +447,11 @@
             return w_result
 
 
-        @gateway.unwrap_spec(mod=str, name=str)
+        @unwrap_spec(mod=str, name=str)
         def load_module(space, mod, name):
             return self.sys_info.load_module(mod, name)
 
-        @gateway.unwrap_spec(modname=str, prologue=str,
+        @unwrap_spec(modname=str, prologue=str,
                              more_init=str, PY_SSIZE_T_CLEAN=bool)
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
@@ -472,46 +463,29 @@
             self.record_imported_module(modname)
             return w_result
 
-        def debug_collect(space):
-            rawrefcount._collect()
-
         # A list of modules which the test caused to be imported (in
         # self.space).  These will be cleaned up automatically in teardown.
         self.imported_module_names = []
 
         if self.runappdirect:
-            fake = FakeSpace(self.space.config)
-            def interp2app(func):
-                def run(*args, **kwargs):
-                    for k in kwargs.keys():
-                        if k not in func.unwrap_spec and not k.startswith('w_'):
-                            v = kwargs.pop(k)
-                            kwargs['w_' + k] = v
-                    return func(fake, *args, **kwargs)
-                return run
-            def wrap(func):
-                return func
             self.sys_info = get_sys_info_app()
             self.compile_module = self.sys_info.compile_extension_module
             self.load_module = self.sys_info.load_module
             self.import_module = self.sys_info.import_module
             self.import_extension = self.sys_info.import_extension
         else:
-            interp2app = gateway.interp2app
             wrap = self.space.wrap
             self.sys_info = get_cpyext_info(self.space)
             self.w_compile_module = wrap(interp2app(compile_module))
             self.w_load_module = wrap(interp2app(load_module))
             self.w_import_module = wrap(interp2app(import_module))
             self.w_import_extension = wrap(interp2app(import_extension))
-        self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/')
-        self.w_debug_collect = wrap(interp2app(debug_collect))
 
-        # create the file lock before we count allocations
-        self.space.call_method(self.space.sys.get("stdout"), "flush")
+            # create the file lock before we count allocations
+            self.space.call_method(self.space.sys.get("stdout"), "flush")
 
-        freeze_refcnts(self)
-        #self.check_and_print_leaks()
+            freeze_refcnts(self)
+            #self.check_and_print_leaks()
 
     def unimport_module(self, name):
         """
@@ -522,6 +496,8 @@
         self.space.delitem(w_modules, w_name)
 
     def teardown_method(self, func):
+        if self.runappdirect:
+            return
         for name in self.imported_module_names:
             self.unimport_module(name)
         self.cleanup_references(self.space)
@@ -656,15 +632,15 @@
         If `cherry.date` is an extension module which imports `apple.banana`,
         the latter is added to `sys.modules` for the `"apple.banana"` key.
         """
+        import sys, types, os
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", source_files=[self.here + 'banana.c'])
+            "apple.banana", source_files=[os.path.join(self.here, 'banana.c')])
         date = self.compile_module(
-            "cherry.date", source_files=[self.here + 'date.c'])
+            "cherry.date", source_files=[os.path.join(self.here, 'date.c')])
 
         # Set up some package state so that the extensions can actually be
         # imported.
-        import sys, types, os
         cherry = sys.modules['cherry'] = types.ModuleType('cherry')
         cherry.__path__ = [os.path.dirname(date)]
 

From pypy.commits at gmail.com  Tue Sep 13 14:38:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 11:38:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: math.gcd()
Message-ID: <57d847a4.4756c20a.501ec.0729@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87099:682481164902
Date: 2016-09-13 20:37 +0200
http://bitbucket.org/pypy/pypy/changeset/682481164902/

Log:	math.gcd()

diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py
--- a/pypy/module/math/__init__.py
+++ b/pypy/module/math/__init__.py
@@ -4,7 +4,8 @@
 
 class Module(MixedModule):
     appleveldefs = {
-       'factorial' : 'app_math.factorial'
+       'factorial' : 'app_math.factorial',
+       'gcd' :       'app_math.gcd',
     }
 
     interpleveldefs = {
diff --git a/pypy/module/math/app_math.py b/pypy/module/math/app_math.py
--- a/pypy/module/math/app_math.py
+++ b/pypy/module/math/app_math.py
@@ -42,3 +42,11 @@
 
     res, _, shift = _fac1(x)
     return res << shift
+
+def gcd(x, y):
+    """greatest common divisor of x and y"""
+    x = abs(x)
+    y = abs(y)
+    while x > 0:
+        x, y = y % x, x
+    return y
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -363,3 +363,9 @@
         #
         raises(TypeError, math.isclose, 0, 1, rel_tol=None)
         raises(TypeError, math.isclose, 0, 1, abs_tol=None)
+
+    def test_gcd(self):
+        import math
+        assert math.gcd(-4, -10) == 2
+        assert math.gcd(0, -10) == 10
+        assert math.gcd(0, 0) == 0

From pypy.commits at gmail.com  Tue Sep 13 15:12:35 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 13 Sep 2016 12:12:35 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Performance improvements
Message-ID: <57d84fa3.8759c20a.b32e1.1595@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87100:b4a1d5e8166e
Date: 2016-09-13 21:11 +0200
http://bitbucket.org/pypy/pypy/changeset/b4a1d5e8166e/

Log:	Performance improvements

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -1,6 +1,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
+from rpython.rlib.objectmodel import func_with_new_name
 from rpython.memory.gcheader import GCHeaderBuilder
 from rpython.memory.support import DEFAULT_CHUNK_SIZE
 from rpython.memory.support import get_address_stack, get_address_deque
@@ -234,6 +235,11 @@
             i += 1
     trace._annspecialcase_ = 'specialize:arg(2)'
 
+    tracei = func_with_new_name(trace, 'tracei')
+    tracei._annspecialcase_ = 'specialize:arg(2)'
+    tracei._always_inline_ = True
+
+
     def _trace_slow_path(self, obj, callback, arg):
         typeid = self.get_type_id(obj)
         if self.has_gcptr_in_varsize(typeid):
diff --git a/rpython/memory/gc/qcgc.py b/rpython/memory/gc/qcgc.py
--- a/rpython/memory/gc/qcgc.py
+++ b/rpython/memory/gc/qcgc.py
@@ -4,6 +4,7 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rlib.objectmodel import specialize
 
 QCGC_HAS_HASH = 0x100 # Upper half of flags for clients, lower half is reserved
 QCGC_PREBUILT_OBJECT = 0x2 # XXX: exploits knowledge about qcgc library
@@ -13,6 +14,10 @@
     moving_gc = False
     needs_write_barrier = True
     malloc_zero_filled = False
+
+    inline_simple_malloc = True
+    inline_simple_malloc_varsize = True
+
     prebuilt_gc_objects_are_static_roots = True # XXX: ?
     can_usually_pin_objects = False
     object_minimal_size = 0
@@ -82,6 +87,7 @@
         # XXX: Not supported
         pass
 
+    @specialize.arg(2)
     def id_or_identityhash(self, gcobj, is_hash):
         obj = llmemory.cast_ptr_to_adr(gcobj)
         hdr = self.header(obj)
@@ -92,6 +98,7 @@
             if is_hash:
                 i = mangle_hash(i)
         return i
+    id_or_identityhash._always_inline_ = True
 
     def id(self, gcobje):
         return self.id_or_identityhash(gcobj, False)
diff --git a/rpython/memory/gctransform/qcgcframework.py b/rpython/memory/gctransform/qcgcframework.py
--- a/rpython/memory/gctransform/qcgcframework.py
+++ b/rpython/memory/gctransform/qcgcframework.py
@@ -19,16 +19,15 @@
         #
         s_gcref = SomePtr(llmemory.GCREF)
 
-        #self.malloc_weakref_ptr = self._getfn(
-        #    GCClass.malloc_weakref.im_func,
-        #    [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True),
-        #     s_gcref], s_gcref)
-        #
+
         def invokecallback(root, visit_fn):
             ref = llmemory.cast_adr_to_ptr(root, rffi.VOIDPP)
             visit_fn(llmemory.cast_ptr_to_adr(ref[0]))
         def pypy_trace_cb(obj, visit_fn):
-            gc.trace(obj, invokecallback, visit_fn)
+            typeid = gc.get_type_id(obj)
+            if not gc.has_gcptr(typeid):
+                return
+            gc.tracei(obj, invokecallback, visit_fn)
         pypy_trace_cb.c_name = "pypy_trace_cb"
         self.autoregister_ptrs.append(
             getfn(pypy_trace_cb,
@@ -57,8 +56,8 @@
         return livevars
 
     def pop_roots(self, hop, livevars):
-        for _ in livevars: # Does not move, so no writing back
-            hop.genop("qcgc_pop_root", [])
+        c_count = rmodel.inputconst(lltype.Signed, len(livevars))
+        hop.genop("qcgc_pop_roots", [c_count])
 
     def gct_gc_fq_register(self, hop):
         pass
diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -888,17 +888,20 @@
 
     extern void pypy_trace_cb(void *, void (*)(void *));
 
-    void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object)) {
+    inline void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object)) {
         pypy_trace_cb((void *) object, (void (*)(void *)) visit);
     }
     """
 
     eci = ExternalCompilationInfo(
-            include_dirs = [library_dir],
+            include_dirs = [library_dir, os.path.join(library_dir, 'src')],
             includes = ["qcgc.h"],
             separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
-                ["qcgc.c"]],
+                ['qcgc.c', 'src/allocator.c', 'src/arena.c', 'src/bag.c',
+                 'src/collector.c', 'src/event_logger.c', 'src/gray_stack.c',
+                 'src/hugeblocktable.c', 'src/shadow_stack.c',
+                 'src/signal_handler.c', 'src/weakref.c']],
             )
     return eci
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -948,8 +948,9 @@
         obj = self.expr(op.args[0])
         return 'qcgc_push_root((object_t *) %s);' % (obj,)
 
-    def OP_QCGC_POP_ROOT(self, op):
-        return 'qcgc_pop_root();'
+    def OP_QCGC_POP_ROOTS(self, op):
+        count = self.expr(op.args[0])
+        return 'qcgc_pop_root((size_t) %s);' % (count,)
 
     def OP_QCGC_ALLOCATE(self, op):
         # XXX: SET typeid
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -0,0 +1,58 @@
+#pragma once
+
+#define CHECKED 0							// Enable runtime sanity checks
+#define DEBUG_ZERO_ON_SWEEP 0				// Zero memory on sweep (debug only)
+
+#define QCGC_INIT_ZERO 0					// Init new objects with zero bytes
+
+/**
+ * Event logger
+ */
+#define EVENT_LOG 1							// Enable event log
+#define LOGFILE "./qcgc_events.log"			// Default logfile
+#define LOG_ALLOCATION 0					// Enable allocation log (warning:
+											// significant performance impact)
+
+#define QCGC_SHADOWSTACK_SIZE 163840		// Total shadowstack size
+#define QCGC_ARENA_BAG_INIT_SIZE 16			// Initial size of the arena bag
+#define QCGC_ARENA_SIZE_EXP 20				// Between 16 (64kB) and 20 (1MB)
+#define QCGC_LARGE_ALLOC_THRESHOLD_EXP 14	// Less than QCGC_ARENA_SIZE_EXP
+#define QCGC_MARK_LIST_SEGMENT_SIZE 64		// TODO: Tune for performance
+#define QCGC_GRAY_STACK_INIT_SIZE 128		// TODO: Tune for performance
+#define QCGC_INC_MARK_MIN 64				// TODO: Tune for performance
+
+/**
+ * Fit allocator
+ */
+#define QCGC_LARGE_FREE_LIST_FIRST_EXP 5	// First exponent of large free list
+#define QCGC_LARGE_FREE_LIST_INIT_SIZE 4	// Initial size for large free lists
+#define QCGC_SMALL_FREE_LIST_INIT_SIZE 16	// Initial size for small free lists
+
+/**
+ * Auto Mark/Collect
+ */
+#define QCGC_MAJOR_COLLECTION_THRESHOLD (5 * (1<<QCGC_ARENA_SIZE_EXP))
+#define QCGC_INCMARK_THRESHOLD (1<<QCGC_ARENA_SIZE_EXP)
+
+/**
+ * DO NOT MODIFY BELOW HERE
+ */
+
+#if QCGC_LARGE_ALLOC_THRESHOLD_EXP >= QCGC_ARENA_SIZE_EXP
+#error	"Inconsistent configuration. Huge block threshold must be smaller " \
+		"than the arena size."
+#endif
+
+#ifdef TESTING
+#define QCGC_STATIC
+#define QCGC_INLINE
+#else
+#define QCGC_STATIC static
+#define QCGC_INLINE inline __attribute((always_inline))
+#endif
+
+#define MAX(a,b) (((a)>(b))?(a):(b))
+#define MIN(a,b) (((a)<(b))?(a):(b))
+#define UNUSED(x) (void)(x)
+#define LIKELY(x) __builtin_expect((x), 1)
+#define UNLIKELY(x) __builtin_expect((x), 0)
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -1,26 +1,199 @@
-#include "src/config.h"
+#include "qcgc.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <sys/mman.h>
+
 #include "src/allocator.h"
-#include "src/arena.h"
-#include "src/bag.h"
 #include "src/collector.h"
-#include "src/core.h"
 #include "src/event_logger.h"
 #include "src/gc_state.h"
-#include "src/gray_stack.h"
 #include "src/hugeblocktable.h"
-#include "src/object.h"
-#include "src/shadow_stack.h"
 #include "src/signal_handler.h"
-#include "src/weakref.h"
 
-#include "src/allocator.c"
-#include "src/arena.c"
-#include "src/bag.c"
-#include "src/collector.c"
-#include "src/core.c"
-#include "src/event_logger.c"
-#include "src/gray_stack.c"
-#include "src/hugeblocktable.c"
-#include "src/shadow_stack.c"
-#include "src/signal_handler.c"
-#include "src/weakref.c"
+#define env_or_fallback(var, env_name, fallback) do {			\
+	char *env_val = getenv(env_name);							\
+	if (env_val != NULL) {										\
+		if (1 != sscanf(env_val, "%zu", &var)) {				\
+			var = fallback;										\
+		}														\
+	} else {													\
+		var = fallback;											\
+	}															\
+} while(0)
+
+QCGC_STATIC QCGC_INLINE void initialize_shadowstack(void);
+QCGC_STATIC QCGC_INLINE void destroy_shadowstack(void);
+
+void qcgc_initialize(void) {
+	initialize_shadowstack();
+	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); // XXX
+	qcgc_state.weakrefs = qcgc_weakref_bag_create(16); // XXX
+	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
+	qcgc_state.gray_stack_size = 0;
+	qcgc_state.phase = GC_PAUSE;
+	qcgc_state.bytes_since_collection = 0;
+	qcgc_state.bytes_since_incmark = 0;
+	qcgc_state.free_cells = 0;
+	qcgc_state.largest_free_block = 0;
+	qcgc_allocator_initialize();
+	qcgc_hbtable_initialize();
+	qcgc_event_logger_initialize();
+
+	env_or_fallback(qcgc_state.major_collection_threshold,
+			"QCGC_MAJOR_COLLECTION", QCGC_MAJOR_COLLECTION_THRESHOLD);
+	env_or_fallback(qcgc_state.incmark_threshold,
+			"QCGC_INCMARK", QCGC_INCMARK_THRESHOLD);
+
+	setup_signal_handler();
+}
+
+void qcgc_destroy(void) {
+	qcgc_event_logger_destroy();
+	qcgc_hbtable_destroy();
+	qcgc_allocator_destroy();
+	destroy_shadowstack();
+	free(qcgc_state.prebuilt_objects);
+	free(qcgc_state.weakrefs);
+	free(qcgc_state.gp_gray_stack);
+}
+
+object_t *qcgc_allocate(size_t size) {
+#if LOG_ALLOCATION
+	qcgc_event_logger_log(EVENT_ALLOCATE_START, sizeof(size_t),
+			(uint8_t *) &size);
+#endif
+	object_t *result;
+
+	if (qcgc_state.bytes_since_collection >
+			qcgc_state.major_collection_threshold) {
+		qcgc_collect();
+	}
+	if (qcgc_state.bytes_since_incmark > qcgc_state.incmark_threshold) {
+		qcgc_mark(true);
+	}
+
+	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
+		// Use bump / fit allocator
+		//if (qcgc_allocator_state.use_bump_allocator) {
+		if (true) {
+			result = qcgc_bump_allocate(size);
+		} else {
+			result = qcgc_fit_allocate(size);
+
+			// Fallback to bump allocator
+			if (result == NULL) {
+				result = qcgc_bump_allocate(size);
+			}
+		}
+	} else {
+		// Use huge block allocator
+		result = qcgc_large_allocate(size);
+	}
+
+	// XXX: Should we use cells instead of bytes?
+	qcgc_state.bytes_since_collection += size;
+	qcgc_state.bytes_since_incmark += size;
+
+
+#if LOG_ALLOCATION
+	qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
+			(uint8_t *) &result);
+#endif
+	return result;
+}
+
+void qcgc_collect(void) {
+	qcgc_mark(false);
+	qcgc_sweep();
+	qcgc_state.bytes_since_collection = 0;
+}
+
+void qcgc_write(object_t *object) {
+#if CHECKED
+	assert(object != NULL);
+#endif
+	if ((object->flags & QCGC_GRAY_FLAG) != 0) {
+		// Already gray, skip
+		return;
+	}
+	object->flags |= QCGC_GRAY_FLAG;
+
+	// Register prebuilt object if necessary
+	if (((object->flags & QCGC_PREBUILT_OBJECT) != 0) &&
+			((object->flags & QCGC_PREBUILT_REGISTERED) == 0)) {
+		object->flags |= QCGC_PREBUILT_REGISTERED;
+		qcgc_state.prebuilt_objects = qcgc_shadow_stack_push(
+				qcgc_state.prebuilt_objects, object);
+	}
+
+	if (qcgc_state.phase == GC_PAUSE) {
+		return; // We are done
+	}
+
+	// Triggered barrier, we must not collect now
+	qcgc_state.phase = GC_MARK;
+
+	// Test reachability of object and push if neccessary
+	if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
+		// NOTE: No mark test here, as prebuilt objects are always reachable
+		// Push prebuilt object to general purpose gray stack
+		qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+				qcgc_state.gp_gray_stack, object);
+	} else if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
+		if (qcgc_hbtable_is_marked(object)) {
+			// Push huge block to general purpose gray stack
+			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+					qcgc_state.gp_gray_stack, object);
+		}
+	} else {
+		if (qcgc_arena_get_blocktype(qcgc_arena_addr((cell_t *) object),
+					qcgc_arena_cell_index((cell_t *) object)) == BLOCK_BLACK) {
+			// This was black before, push it to gray stack again
+			arena_t *arena = qcgc_arena_addr((cell_t *) object);
+			arena->gray_stack = qcgc_gray_stack_push(
+					arena->gray_stack, object);
+		}
+	}
+}
+
+void qcgc_register_weakref(object_t *weakrefobj, object_t **target) {
+#if CHECKED
+	assert((weakrefobj->flags & QCGC_PREBUILT_OBJECT) == 0);
+	assert((object_t *) qcgc_arena_addr((cell_t *) weakrefobj) != weakrefobj);
+#endif
+	// NOTE: At this point, the target must point to a pointer to a valid
+	// object. We don't register any weakrefs to prebuilt objects as they
+	// are always valid.
+	if (((*target)->flags & QCGC_PREBUILT_OBJECT) == 0) {
+		qcgc_state.weakrefs = qcgc_weakref_bag_add(qcgc_state.weakrefs,
+				(struct weakref_bag_item_s) {
+					.weakrefobj = weakrefobj,
+					.target = target});
+	}
+}
+
+QCGC_STATIC QCGC_INLINE void *_trap_page_addr(object_t **shadow_stack) {
+	object_t **shadow_stack_end = shadow_stack + QCGC_SHADOWSTACK_SIZE;
+	char *in_trap_page = (((char *)shadow_stack_end) + 4095);
+	void *rounded_trap_page = (void *)(((uintptr_t)in_trap_page) & (~4095));
+	return rounded_trap_page;
+}
+
+QCGC_STATIC QCGC_INLINE void initialize_shadowstack(void) {
+	size_t stack_size = QCGC_SHADOWSTACK_SIZE * sizeof(object_t *);
+	// allocate stack + size for alignement + trap page
+	object_t **stack = (object_t **) malloc(stack_size + 8192);
+	assert(stack != NULL);
+	mprotect(_trap_page_addr(stack), 4096, PROT_NONE);
+
+	qcgc_shadowstack.top = stack;
+	qcgc_shadowstack.base = stack;
+}
+
+QCGC_STATIC void destroy_shadowstack(void) {
+	mprotect(_trap_page_addr(qcgc_shadowstack.base), 4096, PROT_READ |
+				PROT_WRITE);
+
+	free(qcgc_shadowstack.base);
+}
diff --git a/rpython/translator/c/src/qcgc/qcgc.h b/rpython/translator/c/src/qcgc/qcgc.h
--- a/rpython/translator/c/src/qcgc/qcgc.h
+++ b/rpython/translator/c/src/qcgc/qcgc.h
@@ -4,9 +4,30 @@
 
 #pragma once
 
+#include "config.h"
+
+#include <assert.h>
 #include <stddef.h>
+#include <stdint.h>
 
-#include "src/object.h"
+/**
+ * Object Layout.
+ */
+#define QCGC_GRAY_FLAG (1<<0)
+#define QCGC_PREBUILT_OBJECT (1<<1)
+#define QCGC_PREBUILT_REGISTERED (1<<2)
+
+typedef struct object_s {
+	uint32_t flags;
+} object_t;
+
+/**
+ * Shadow stack
+ */
+struct qcgc_shadowstack {
+	object_t **top;
+	object_t **base;
+} qcgc_shadowstack;
 
 /**
  * Initialize the garbage collector.
@@ -32,12 +53,20 @@
  *
  * @param	object	The root object
  */
-void qcgc_push_root(object_t *object);
+QCGC_STATIC QCGC_INLINE void qcgc_push_root(object_t *object) {
+	*qcgc_shadowstack.top = object;
+	qcgc_shadowstack.top++;
+}
 
 /**
- * Pop root object.
+ * Pop root objects.
+ *
+ * @param	count	Number of object to pop
  */
-void qcgc_pop_root(void);
+QCGC_STATIC QCGC_INLINE void qcgc_pop_root(size_t count) {
+	qcgc_shadowstack.top -= count;
+	assert(qcgc_shadowstack.base <= qcgc_shadowstack.top);
+}
 
 /**
  * Write barrier. Has to be called whenever a reference to another object is
diff --git a/rpython/translator/c/src/qcgc/src/allocator.c b/rpython/translator/c/src/qcgc/src/allocator.c
--- a/rpython/translator/c/src/qcgc/src/allocator.c
+++ b/rpython/translator/c/src/qcgc/src/allocator.c
@@ -22,9 +22,6 @@
 QCGC_STATIC cell_t *fit_allocator_large_fit(size_t index, size_t cells);
 QCGC_STATIC cell_t *fit_allocator_large_first_fit(size_t index, size_t cells);
 
-QCGC_STATIC bool valid_block(cell_t *ptr, size_t cells);
-QCGC_STATIC void free_list_consistency_check(void);
-
 void qcgc_allocator_initialize(void) {
 	qcgc_allocator_state.arenas =
 		qcgc_arena_bag_create(QCGC_ARENA_BAG_INIT_SIZE);
@@ -74,19 +71,6 @@
 }
 
 void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
-#if CHECKED
-	if (cells > 0) {
-		assert((((object_t *)ptr)->flags & QCGC_PREBUILT_OBJECT) == 0);
-		assert((cell_t *) qcgc_arena_addr(ptr) != ptr);
-		assert(qcgc_arena_get_blocktype(ptr) == BLOCK_FREE);
-		if (qcgc_arena_addr(ptr) == qcgc_arena_addr(ptr + cells)) {
-			assert(qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT);
-		}
-		for (size_t i = 1; i < cells; i++) {
-			assert(qcgc_arena_get_blocktype(ptr + i) == BLOCK_EXTENT);
-		}
-	}
-#endif
 	if (cells > 0) {
 		if (is_small(cells)) {
 			size_t index = small_index(cells);
@@ -113,9 +97,12 @@
 	assert(bytes <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP);
 #endif
 	size_t cells = bytes_to_cells(bytes);
-	if (cells > qcgc_allocator_state.bump_state.remaining_cells) {
-		if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
-			qcgc_arena_set_blocktype(qcgc_allocator_state.bump_state.bump_ptr,
+	if (UNLIKELY(cells > qcgc_allocator_state.bump_state.remaining_cells)) {
+		if (LIKELY(qcgc_allocator_state.bump_state.remaining_cells > 0)) {
+			qcgc_arena_set_blocktype(
+					qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr),
+					qcgc_arena_cell_index(
+						qcgc_allocator_state.bump_state.bump_ptr),
 					BLOCK_FREE);
 		}
 		bump_allocator_renew_block();
@@ -123,7 +110,8 @@
 	cell_t *mem = qcgc_allocator_state.bump_state.bump_ptr;
 	bump_allocator_advance(cells);
 
-	qcgc_arena_set_blocktype(mem, BLOCK_WHITE);
+	qcgc_arena_set_blocktype(qcgc_arena_addr(mem), qcgc_arena_cell_index(mem),
+			BLOCK_WHITE);
 	/*
 	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
 		qcgc_arena_set_blocktype(qcgc_allocator_state.bump_state.bump_ptr,
@@ -143,8 +131,10 @@
 	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
 		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
 				i++) {
-			assert(qcgc_arena_get_blocktype(
-						qcgc_allocator_state.bump_state.bump_ptr + i)
+			assert(qcgc_arena_get_blocktype(qcgc_arena_addr(
+							qcgc_allocator_state.bump_state.bump_ptr + i),
+						qcgc_arena_cell_index(
+							qcgc_allocator_state.bump_state.bump_ptr + i))
 					== BLOCK_EXTENT);
 		}
 	}
@@ -156,11 +146,16 @@
 #if CHECKED
 	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
 		assert(qcgc_arena_get_blocktype(
-					qcgc_allocator_state.bump_state.bump_ptr) == BLOCK_FREE);
+					qcgc_arena_addr( qcgc_allocator_state.bump_state.bump_ptr),
+					qcgc_arena_cell_index(
+						qcgc_allocator_state.bump_state.bump_ptr))
+				== BLOCK_FREE);
 		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
 				i++) {
-			assert(qcgc_arena_get_blocktype(
-						qcgc_allocator_state.bump_state.bump_ptr + i)
+			assert(qcgc_arena_get_blocktype(qcgc_arena_addr(
+							qcgc_allocator_state.bump_state.bump_ptr + i),
+						qcgc_arena_cell_index(
+							qcgc_allocator_state.bump_state.bump_ptr + i))
 					== BLOCK_EXTENT);
 		}
 	}
@@ -191,12 +186,17 @@
 		large_free_list[QCGC_LARGE_FREE_LISTS - 1] = free_list;
 #if CHECKED
 	assert(qcgc_allocator_state.bump_state.bump_ptr != NULL);
-	assert(qcgc_arena_get_blocktype(qcgc_allocator_state.bump_state.bump_ptr) ==
-			BLOCK_FREE);
+	assert(qcgc_arena_get_blocktype(
+				qcgc_arena_addr( qcgc_allocator_state.bump_state.bump_ptr),
+				qcgc_arena_cell_index(
+					qcgc_allocator_state.bump_state.bump_ptr))
+			== BLOCK_FREE);
 	for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
 			i++) {
-		assert(qcgc_arena_get_blocktype(
-					qcgc_allocator_state.bump_state.bump_ptr + i)
+		assert(qcgc_arena_get_blocktype(qcgc_arena_addr(
+						qcgc_allocator_state.bump_state.bump_ptr + i),
+					qcgc_arena_cell_index(
+						qcgc_allocator_state.bump_state.bump_ptr + i))
 				== BLOCK_EXTENT);
 	}
 #endif
@@ -204,9 +204,11 @@
 
 QCGC_STATIC void bump_allocator_assign(cell_t *ptr, size_t cells) {
 #if CHECKED
-	assert(qcgc_arena_get_blocktype(ptr) == BLOCK_FREE);
+	assert(qcgc_arena_get_blocktype(qcgc_arena_addr(ptr),
+				qcgc_arena_cell_index(ptr)) == BLOCK_FREE);
 	for (size_t i = 1; i < cells; i++) {
-		assert(qcgc_arena_get_blocktype(ptr + i) == BLOCK_EXTENT);
+		assert(qcgc_arena_get_blocktype(qcgc_arena_addr(ptr + i),
+					qcgc_arena_cell_index(ptr + i)) == BLOCK_EXTENT);
 	}
 #endif
 	qcgc_allocator_state.bump_state.bump_ptr = ptr;
@@ -289,9 +291,11 @@
 				qcgc_linear_free_list_remove_index(
 						qcgc_allocator_state.fit_state.small_free_list[index],
 						0);
-			qcgc_arena_set_blocktype(result, BLOCK_WHITE);
+			qcgc_arena_set_blocktype(qcgc_arena_addr(result),
+					qcgc_arena_cell_index(result), BLOCK_WHITE);
 			if (list_cell_size - cells > 0) {
-				qcgc_arena_set_blocktype(result + cells, BLOCK_FREE);
+				qcgc_arena_set_blocktype(qcgc_arena_addr(result + cells),
+						qcgc_arena_cell_index(result + cells), BLOCK_FREE);
 				qcgc_fit_allocator_add(result + cells, list_cell_size - cells);
 			}
 			return result;
@@ -334,9 +338,11 @@
 		qcgc_allocator_state.fit_state.large_free_list[index] =
 			qcgc_exp_free_list_remove_index(qcgc_allocator_state.fit_state.
 					large_free_list[index], best_fit_index);
-		qcgc_arena_set_blocktype(result, BLOCK_WHITE);
+		qcgc_arena_set_blocktype(qcgc_arena_addr(result),
+				qcgc_arena_cell_index(result), BLOCK_WHITE);
 		if (best_fit_cells - cells > 0) {
-			qcgc_arena_set_blocktype(result + cells, BLOCK_FREE);
+			qcgc_arena_set_blocktype(qcgc_arena_addr(result + cells),
+					qcgc_arena_cell_index(result + cells), BLOCK_FREE);
 			qcgc_fit_allocator_add(result + cells, best_fit_cells - cells);
 		}
 	} else {
@@ -360,9 +366,11 @@
 						0);
 
 			qcgc_arena_mark_allocated(item.ptr, cells);
-			qcgc_arena_set_blocktype(item.ptr, BLOCK_WHITE);
+			qcgc_arena_set_blocktype(qcgc_arena_addr(item.ptr),
+					qcgc_arena_cell_index(item.ptr), BLOCK_WHITE);
 			if (item.size - cells > 0) {
-				qcgc_arena_set_blocktype(item.ptr + cells, BLOCK_FREE);
+				qcgc_arena_set_blocktype(qcgc_arena_addr(item.ptr + cells),
+						qcgc_arena_cell_index(item.ptr + cells), BLOCK_FREE);
 				qcgc_fit_allocator_add(item.ptr + cells, item.size - cells);
 			}
 			return item.ptr;
@@ -404,41 +412,3 @@
 #endif
 	return index + 1;
 }
-
-QCGC_STATIC bool valid_block(cell_t *ptr, size_t cells) {
-#if CHECKED
-	assert(ptr != NULL);
-	assert(cells > 0);
-#endif
-	return (qcgc_arena_get_blocktype(ptr) == BLOCK_FREE && (
-				((qcgc_arena_addr(ptr + cells)) == (arena_t *) (ptr + cells)) ||
-				qcgc_arena_get_blocktype(ptr + cells) != BLOCK_EXTENT));
-}
-
-QCGC_STATIC void free_list_consistency_check(void) {
-	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
-		linear_free_list_t *free_list =
-			qcgc_allocator_state.fit_state.small_free_list[i];
-		for (size_t j = 0; j < free_list->count; j++) {
-			cell_t *item = free_list->items[j];
-			if  (qcgc_arena_get_blocktype(item) == BLOCK_FREE) {
-				for (size_t s = 1; s < small_index_to_cells(i); s++) {
-					assert(qcgc_arena_get_blocktype(item + s) == BLOCK_EXTENT);
-				}
-			}
-		}
-	}
-
-	for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
-		exp_free_list_t *free_list =
-			qcgc_allocator_state.fit_state.large_free_list[i];
-		for (size_t j = 0; j < free_list->count; j++) {
-			struct exp_free_list_item_s item = free_list->items[j];
-			if  (qcgc_arena_get_blocktype(item.ptr) == BLOCK_FREE) {
-				for (size_t s = 1; s < item.size; s++) {
-					assert(qcgc_arena_get_blocktype(item.ptr + s) == BLOCK_EXTENT);
-				}
-			}
-		}
-	}
-}
diff --git a/rpython/translator/c/src/qcgc/src/allocator.h b/rpython/translator/c/src/qcgc/src/allocator.h
--- a/rpython/translator/c/src/qcgc/src/allocator.h
+++ b/rpython/translator/c/src/qcgc/src/allocator.h
@@ -1,12 +1,9 @@
 #pragma once
 
-#include "config.h"
-
-#include <stddef.h>
+#include "../qcgc.h"
 
 #include "arena.h"
 #include "bag.h"
-#include "object.h"
 
 /**
  * Free lists:
diff --git a/rpython/translator/c/src/qcgc/src/arena.c b/rpython/translator/c/src/qcgc/src/arena.c
--- a/rpython/translator/c/src/qcgc/src/arena.c
+++ b/rpython/translator/c/src/qcgc/src/arena.c
@@ -13,12 +13,6 @@
 #include "event_logger.h"
 #include "gc_state.h"
 
-/**
- * Internal functions
- */
-QCGC_STATIC QCGC_INLINE blocktype_t get_blocktype(arena_t *arena, size_t index);
-QCGC_STATIC QCGC_INLINE void set_blocktype(arena_t *arena, size_t index, blocktype_t type);
-
 arena_t *qcgc_arena_create(void) {
 	qcgc_event_logger_log(EVENT_NEW_ARENA, 0, NULL);
 
@@ -48,7 +42,7 @@
 	}
 
 	// Init bitmaps: One large free block
-	qcgc_arena_set_bitmap_entry(result->mark_bitmap, QCGC_ARENA_FIRST_CELL_INDEX, 1);
+	result->mark_bitmap[QCGC_ARENA_FIRST_CELL_INDEX / 8] = 1;
 
 	// Create gray stack
 	result->gray_stack = qcgc_gray_stack_create(QCGC_GRAY_STACK_INIT_SIZE);
@@ -63,121 +57,36 @@
 	munmap((void *) arena, QCGC_ARENA_SIZE);
 }
 
-arena_t *qcgc_arena_addr(cell_t *ptr) {
-	return (arena_t *)((intptr_t) ptr & ~(QCGC_ARENA_SIZE - 1));
-}
-
-size_t qcgc_arena_cell_index(cell_t *ptr) {
-	return (size_t)((intptr_t) ptr & (QCGC_ARENA_SIZE - 1)) >> 4;
-}
-
-bool qcgc_arena_get_bitmap_entry(uint8_t *bitmap, size_t index) {
-#if CHECKED
-	assert(bitmap != NULL);
-#endif
-	size_t byte = index / 8;
-	uint8_t mask = 0x01 << (index % 8);
-	return ((bitmap[byte] & mask) == mask);
-}
-
-void qcgc_arena_set_bitmap_entry(uint8_t *bitmap, size_t index, uint8_t value) {
-#if CHECKED
-	assert(bitmap != NULL);
-#endif
-	size_t byte = index / 8;
-	uint8_t bit = (index % 8);
-	bitmap[byte] &= ~(1 << bit);
-	bitmap[byte] = (bitmap[byte] & ~(1 << bit)) | (value << bit);
-}
-
-QCGC_STATIC QCGC_INLINE blocktype_t get_blocktype(arena_t *arena, size_t index) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	uint8_t block_bit = qcgc_arena_get_bitmap_entry(arena->block_bitmap, index);
-	uint8_t mark_bit = qcgc_arena_get_bitmap_entry(arena->mark_bitmap, index);
-
-	if (block_bit) {
-		if (mark_bit) {
-			return BLOCK_BLACK;
-		} else {
-			return BLOCK_WHITE;
-		}
-	} else {
-		if (mark_bit) {
-			return BLOCK_FREE;
-		} else {
-			return BLOCK_EXTENT;
-		}
-	}
-}
-
-QCGC_STATIC QCGC_INLINE blocktype_t qcgc_arena_get_blocktype(cell_t *ptr) {
-	size_t index = qcgc_arena_cell_index(ptr);
-	arena_t *arena = qcgc_arena_addr(ptr);
-
-	return get_blocktype(arena, index);
-}
-
-QCGC_STATIC QCGC_INLINE void set_blocktype(arena_t *arena, size_t index, blocktype_t type) {
-#if CHECKED
-	assert(arena != NULL);
-#endif
-	switch(type) {
-		case BLOCK_EXTENT:
-			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, 0);
-			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, 0);
-			break;
-		case BLOCK_FREE:
-			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, 0);
-			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, 1);
-			break;
-		case BLOCK_WHITE:
-			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, 1);
-			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, 0);
-			break;
-		case BLOCK_BLACK:
-			qcgc_arena_set_bitmap_entry(arena->mark_bitmap, index, 1);
-			qcgc_arena_set_bitmap_entry(arena->block_bitmap, index, 1);
-			break;
-	}
-}
-
-QCGC_STATIC QCGC_INLINE void qcgc_arena_set_blocktype(cell_t *ptr,
-		blocktype_t type) {
-	size_t index = qcgc_arena_cell_index(ptr);
-	arena_t *arena = qcgc_arena_addr(ptr);
-	set_blocktype(arena, index, type);
-}
-
 void qcgc_arena_mark_allocated(cell_t *ptr, size_t cells) {
 	size_t index = qcgc_arena_cell_index(ptr);
 	arena_t *arena = qcgc_arena_addr(ptr);
 #if CHECKED
-	assert(get_blocktype(arena, index) == BLOCK_FREE);
+	assert(qcgc_arena_get_blocktype(arena, index) == BLOCK_FREE);
 	for (size_t i = 1; i < cells; i++) {
-		assert(get_blocktype(arena, index + i) == BLOCK_EXTENT);
+		assert(qcgc_arena_get_blocktype(arena, index + i) == BLOCK_EXTENT);
 	}
 #endif
-	set_blocktype(arena, index, BLOCK_WHITE);
+	qcgc_arena_set_blocktype(arena, index, BLOCK_WHITE);
 	size_t index_of_next_block = index + cells;
 	if (index_of_next_block < QCGC_ARENA_CELLS_COUNT &&
-			get_blocktype(arena, index_of_next_block) == BLOCK_EXTENT) {
-		set_blocktype(arena, index_of_next_block, BLOCK_FREE);
+			qcgc_arena_get_blocktype(arena, index_of_next_block) ==
+			BLOCK_EXTENT) {
+		qcgc_arena_set_blocktype(arena, index_of_next_block, BLOCK_FREE);
 	}
 #if CHECKED
-	assert(get_blocktype(arena, index) == BLOCK_WHITE);
+	assert(qcgc_arena_get_blocktype(arena, index) == BLOCK_WHITE);
 	for (size_t i = 1; i < cells; i++) {
-		assert(get_blocktype(arena, index + i) == BLOCK_EXTENT);
+		assert(qcgc_arena_get_blocktype(arena, index + i) == BLOCK_EXTENT);
 	}
 	if (index_of_next_block < QCGC_ARENA_CELLS_COUNT) {
-		assert(get_blocktype(arena, index + cells) != BLOCK_EXTENT);
+		assert(qcgc_arena_get_blocktype(arena, index + cells) != BLOCK_EXTENT);
 	}
 #endif
 }
 
 void qcgc_arena_mark_free(cell_t *ptr) {
-	qcgc_arena_set_blocktype(ptr, BLOCK_FREE);
+	qcgc_arena_set_blocktype(qcgc_arena_addr(ptr), qcgc_arena_cell_index(ptr),
+			BLOCK_FREE);
 	// No coalescing, collector will do this
 }
 
@@ -192,9 +101,9 @@
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
-		switch (get_blocktype(arena, cell)) {
+		switch (qcgc_arena_get_blocktype(arena, cell)) {
 			case BLOCK_BLACK:
-				set_blocktype(arena, cell, BLOCK_WHITE);
+				qcgc_arena_set_blocktype(arena, cell, BLOCK_WHITE);
 			case BLOCK_FREE: // Fall through
 			case BLOCK_EXTENT: // Fall through
 			case BLOCK_WHITE: // Fall through
@@ -228,36 +137,52 @@
 
 		arena->block_bitmap[i] = new_block;
 
-		for (size_t j = 0; j < 8; j++) {
-			size_t cell = i * 8 + j;
-			uint8_t mask = 1 << j;
-			if ((new_mark & mask) == mask) {
-				if (last_free_cell != 0) {
-					// Coalesce
-					new_mark &= ~mask;
-				} else {
-					last_free_cell = cell;
+		if (new_block == new_mark) {
+			// Both are 0
+			continue;
+		}
+
+		if (!new_block) {
+			// Only entries in the mark bitmap
+			if (last_free_cell != 0) {
+				// Coalesce
+				new_mark = 0;
+			} else {
+				uint8_t first = __builtin_ctz(new_mark);
+				new_mark = 1 << first;
+				last_free_cell = i * 8 + first;
+			}
+		} else {
+			for (size_t j = 0; j < 8; j++) {
+				size_t cell = i * 8 + j;
+				uint8_t m = (new_mark >> j) & 0x1;
+				uint8_t b = (new_block >> j) & 0x1;
+				uint8_t mask = 1 << j;
+				if (m) {
+					// Free
+					if (last_free_cell != 0) {
+						// Coalesce
+						new_mark &= ~mask;
+					} else {
+						last_free_cell = cell;
+					}
+				} else if (b) {
+					// White
+					free = false;
+					if (last_free_cell != 0) {
+						qcgc_fit_allocator_add(arena->cells + last_free_cell,
+								cell - last_free_cell);
+#if DEBUG_ZERO_ON_SWEEP
+						memset(arena->cells + last_free_cell, 0,
+								sizeof(cell_t) * (cell - last_free_cell));
+#endif
+						qcgc_state.free_cells += cell - last_free_cell;
+						qcgc_state.largest_free_block = MAX(
+								qcgc_state.largest_free_block,
+								cell - last_free_cell);
+						last_free_cell = 0;
+					}
 				}
-			} else if ((new_block & mask) == mask) {
-				free = false;
-				if (last_free_cell != 0) {
-					// Force update to satisfy precondition
-					arena->mark_bitmap[i] = new_mark;
-					qcgc_fit_allocator_add(arena->cells + last_free_cell,
-							cell - last_free_cell);
-#if DEBUG_ZERO_ON_SWEEP
-					memset(arena->cells + last_free_cell, 0,
-							sizeof(cell_t) * (cell - last_free_cell));
-#endif
-					qcgc_state.free_cells += cell - last_free_cell;
-					qcgc_state.largest_free_block = MAX(
-							qcgc_state.largest_free_block,
-							cell - last_free_cell);
-					last_free_cell = 0;
-				}
-				// White
-			} else {
-				// Extent
 			}
 		}
 		arena->mark_bitmap[i] = new_mark;
@@ -289,7 +214,7 @@
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+		switch (qcgc_arena_get_blocktype(arena, cell)) {
 			case BLOCK_WHITE: // Fall through
 			case BLOCK_BLACK:
 				return false;
@@ -310,7 +235,7 @@
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+		switch (qcgc_arena_get_blocktype(arena, cell)) {
 			case BLOCK_WHITE: // Fall through
 			case BLOCK_BLACK:
 				prev_was_free = false;
@@ -339,7 +264,7 @@
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+		switch (qcgc_arena_get_blocktype(arena, cell)) {
 			case BLOCK_WHITE: // Fall through
 			case BLOCK_BLACK:
 			case BLOCK_EXTENT:
@@ -361,7 +286,7 @@
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+		switch (qcgc_arena_get_blocktype(arena, cell)) {
 			case BLOCK_BLACK:	// Fall through
 			case BLOCK_EXTENT:
 			case BLOCK_FREE:
@@ -383,7 +308,7 @@
 	for (size_t cell = QCGC_ARENA_FIRST_CELL_INDEX;
 			cell < QCGC_ARENA_CELLS_COUNT;
 			cell++) {
-		switch (qcgc_arena_get_blocktype((void *) &arena->cells[cell])) {
+		switch (qcgc_arena_get_blocktype(arena, cell)) {
 			case BLOCK_WHITE: // Fall through
 			case BLOCK_FREE:
 			case BLOCK_EXTENT:
diff --git a/rpython/translator/c/src/qcgc/src/arena.h b/rpython/translator/c/src/qcgc/src/arena.h
--- a/rpython/translator/c/src/qcgc/src/arena.h
+++ b/rpython/translator/c/src/qcgc/src/arena.h
@@ -4,10 +4,9 @@
 
 #pragma once
 
-#include "config.h"
+#include "../qcgc.h"
 
 #include <stdbool.h>
-#include <stdint.h>
 #include <sys/types.h>
 
 #include "gray_stack.h"
@@ -70,66 +69,17 @@
 void qcgc_arena_destroy(arena_t *arena);
 
 /**
- * Arena pointer for given cell.
- *
- * @param	ptr		Pointer to cell for which you want to know the corresponding
- *					arena
- * @return	The arena the pointer belongs to
- */
-arena_t *qcgc_arena_addr(cell_t *ptr);
-
-/**
- * Index of cell in arena.
- *
- * @param	ptr		Pointer to cell for which you want to know the cell index
- * @return	Index of the cell to which ptr points to
- */
-size_t qcgc_arena_cell_index(cell_t *ptr);
-
-/**
- * Get bitmap value for given bitmap and cell index.
- *
- * @param	bitmap	Bitmap
- * @param	index	Index of cell
- * @return	true if bitmap entry is set, false otherwise
- */
-bool qcgc_arena_get_bitmap_entry(uint8_t *bitmap, size_t index);
-
-/**
- * Set bitmap value for given bitmap and cell index.
- *
- * @param	bitmap	Bitmap
- * @param	index	Index of cell
- * @param	value	1 to set entry, 0 to reset entry
- */
-void qcgc_arena_set_bitmap_entry(uint8_t *bitmap, size_t index, uint8_t value);
-
-/**
- * Get blocktype.
- *
- * @param	ptr		Pointer to cell for which you want to know the blocktype
- * @return	Blocktype
- */
-QCGC_STATIC QCGC_INLINE blocktype_t qcgc_arena_get_blocktype(cell_t *ptr);
-
-/**
- * Set blocktype.
- *
- * @param	ptr		Pointer to cell for which you want to set the blocktype
- * @param	type	Blocktype that should be set
- */
-QCGC_STATIC QCGC_INLINE void qcgc_arena_set_blocktype(cell_t *ptr, blocktype_t type);
-
-/**
  * Mark ptr as allocated area with given size.
+ * DEPRECATED
  *
  * @param	ptr		Pointer to first cell of area
  * @param	cells	Size in cells
  */
-QCGC_STATIC void qcgc_arena_mark_allocated(cell_t *ptr, size_t cells);
+void qcgc_arena_mark_allocated(cell_t *ptr, size_t cells);
 
 /**
  * Mark cell ptr point to as free (no coalescing).
+ * DEPRECATED
  *
  * @param	ptr		Pointer to cell that should be marked as free
  */
@@ -153,6 +103,101 @@
 
 
 /*******************************************************************************
+ * Inline functions
+ ******************************************************************************/
+
+/**
+ * Arena pointer for given cell.
+ *
+ * @param	ptr		Pointer to cell for which you want to know the corresponding
+ *					arena
+ * @return	The arena the pointer belongs to
+ */
+QCGC_STATIC QCGC_INLINE arena_t *qcgc_arena_addr(cell_t *ptr) {
+	return (arena_t *)((intptr_t) ptr & ~(QCGC_ARENA_SIZE - 1));
+}
+
+/**
+ * Index of cell in arena.
+ *
+ * @param	ptr		Pointer to cell for which you want to know the cell index
+ * @return	Index of the cell to which ptr points to
+ */
+QCGC_STATIC QCGC_INLINE size_t qcgc_arena_cell_index(cell_t *ptr) {
+	return (size_t)((intptr_t) ptr & (QCGC_ARENA_SIZE - 1)) >> 4;
+}
+
+/**
+ * Get blocktype.
+ *
+ * @param	arena		Arena in which to perform the lookup
+ * @param	index		Cell index of the block to look up
+ * @return	Blocktype
+ */
+QCGC_STATIC QCGC_INLINE blocktype_t qcgc_arena_get_blocktype(arena_t *arena,
+		size_t index) {
+#if CHECKED
+	assert(arena != NULL);
+	assert(index >= QCGC_ARENA_FIRST_CELL_INDEX);
+	assert(index < QCGC_ARENA_CELLS_COUNT);
+#endif
+	// Read bitmap entry
+	size_t byte = index / 8;
+	uint8_t mask = 0x01 << (index % 8);
+	bool block_bit = ((arena->block_bitmap[byte] & mask) == mask);
+	bool mark_bit  = ((arena->mark_bitmap[byte] & mask) == mask);
+
+	if (block_bit) {
+		if (mark_bit) {
+			return BLOCK_BLACK;
+		} else {
+			return BLOCK_WHITE;
+		}
+	} else {
+		if (mark_bit) {
+			return BLOCK_FREE;
+		} else {
+			return BLOCK_EXTENT;
+		}
+	}
+}
+
+/**
+ * Set blocktype.
+ *
+ * @param	ptr		Pointer to cell for which you want to set the blocktype
+ * @param	type	Blocktype that should be set
+ */
+QCGC_STATIC QCGC_INLINE void qcgc_arena_set_blocktype(arena_t *arena,
+		size_t index, blocktype_t type) {
+#if CHECKED
+	assert(arena != NULL);
+	assert(index >= QCGC_ARENA_FIRST_CELL_INDEX);
+	assert(index < QCGC_ARENA_CELLS_COUNT);
+#endif
+	size_t byte = index / 8;
+	uint8_t mask = 0x1 << (index % 8);
+	switch(type) {
+		case BLOCK_EXTENT:
+			arena->block_bitmap[byte] &= ~mask;
+			arena->mark_bitmap[byte] &= ~mask;
+			break;
+		case BLOCK_FREE:
+			arena->block_bitmap[byte] &= ~mask;
+			arena->mark_bitmap[byte] |= mask;
+			break;
+		case BLOCK_WHITE:
+			arena->block_bitmap[byte] |= mask;
+			arena->mark_bitmap[byte] &= ~mask;
+			break;
+		case BLOCK_BLACK:
+			arena->block_bitmap[byte] |= mask;
+			arena->mark_bitmap[byte] |= mask;
+			break;
+	}
+}
+
+/*******************************************************************************
  * Debug functions                                                             *
  ******************************************************************************/
 
diff --git a/rpython/translator/c/src/qcgc/src/bag.h b/rpython/translator/c/src/qcgc/src/bag.h
--- a/rpython/translator/c/src/qcgc/src/bag.h
+++ b/rpython/translator/c/src/qcgc/src/bag.h
@@ -1,8 +1,7 @@
 #pragma once
 
-#include "config.h"
+#include "../qcgc.h"
 
-#include <stddef.h>
 #include <stdlib.h>
 
 #include "arena.h"
diff --git a/rpython/translator/c/src/qcgc/src/collector.c b/rpython/translator/c/src/qcgc/src/collector.c
--- a/rpython/translator/c/src/qcgc/src/collector.c
+++ b/rpython/translator/c/src/qcgc/src/collector.c
@@ -1,8 +1,5 @@
 #include "collector.h"
 
-#include <assert.h>
-#include <stddef.h>
-
 #include "arena.h"
 #include "allocator.h"
 #include "gc_state.h"
@@ -13,7 +10,7 @@
 QCGC_STATIC QCGC_INLINE void qcgc_pop_object(object_t *object);
 QCGC_STATIC QCGC_INLINE void qcgc_push_object(object_t *object);
 
-QCGC_STATIC void qcgc_mark(bool incremental) {
+void qcgc_mark(bool incremental) {
 	{
 		struct log_info_s {
 			bool incremental;
@@ -42,8 +39,8 @@
 	qcgc_state.phase = GC_MARK;
 
 	// Always push all roots to make shadowstack pushes faster
-	for (object_t **it = qcgc_state.shadow_stack_base;
-		it < qcgc_state.shadow_stack;
+	for (object_t **it = qcgc_shadowstack.base;
+		it < qcgc_shadowstack.top;
 		it++) {
 		qcgc_push_object(*it);
 	}
@@ -110,7 +107,8 @@
 			(object->flags & QCGC_GRAY_FLAG) == QCGC_GRAY_FLAG);
 	if (((object->flags & QCGC_PREBUILT_OBJECT) == 0) &&
 		((object_t *) qcgc_arena_addr((cell_t *) object) != object)) {
-		assert(qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK);
+		assert(qcgc_arena_get_blocktype(qcgc_arena_addr((cell_t *) object),
+					qcgc_arena_cell_index((cell_t *) object)) == BLOCK_BLACK);
 	}
 #endif
 	object->flags &= ~QCGC_GRAY_FLAG;
@@ -122,22 +120,23 @@
 	assert(qcgc_state.phase == GC_MARK);
 #endif
 	if (object != NULL) {
-		if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
+		arena_t *arena = qcgc_arena_addr((cell_t *) object);
+		if ((object_t *) arena == object) {
 			if (qcgc_hbtable_mark(object)) {
 				// Did mark it / was white before
 				object->flags |= QCGC_GRAY_FLAG;
 				qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
 						qcgc_state.gp_gray_stack, object);
 			}
-			return; // Skip tests
+			return;
 		}
 		if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
-			return; // Prebuilt objects are always black, no pushing here
+			return;
 		}
-		if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_WHITE) {
+		size_t index = qcgc_arena_cell_index((cell_t *) object);
+		if (qcgc_arena_get_blocktype(arena, index) == BLOCK_WHITE) {
 			object->flags |= QCGC_GRAY_FLAG;
-			qcgc_arena_set_blocktype((cell_t *) object, BLOCK_BLACK);
-			arena_t *arena = qcgc_arena_addr((cell_t *) object);
+			qcgc_arena_set_blocktype(arena, index, BLOCK_BLACK);
 			arena->gray_stack = qcgc_gray_stack_push(arena->gray_stack, object);
 		}
 	}
diff --git a/rpython/translator/c/src/qcgc/src/collector.h b/rpython/translator/c/src/qcgc/src/collector.h
--- a/rpython/translator/c/src/qcgc/src/collector.h
+++ b/rpython/translator/c/src/qcgc/src/collector.h
@@ -1,12 +1,8 @@
 #pragma once
 
-#include "config.h"
+#include "../qcgc.h"
 
 #include <stdbool.h>
 
-#include "object.h"
-
-QCGC_STATIC void qcgc_mark(bool incremental);
-QCGC_STATIC void qcgc_sweep(void);
-
-extern void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object));
+void qcgc_mark(bool incremental);
+void qcgc_sweep(void);
diff --git a/rpython/translator/c/src/qcgc/src/config.h b/rpython/translator/c/src/qcgc/src/config.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/config.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#pragma once
-
-#define CHECKED 0							// Enable runtime sanity checks
-#define DEBUG_ZERO_ON_SWEEP 0				// Zero memory on sweep (debug only)
-
-#define QCGC_INIT_ZERO 0					// Init new objects with zero bytes
-
-/**
- * Event logger
- */
-#define EVENT_LOG 1							// Enable event log
-#define LOGFILE "./qcgc_events.log"			// Default logfile
-#define LOG_ALLOCATION 0					// Enable allocation log (warning:
-											// significant performance impact)
-
-#define QCGC_SHADOWSTACK_SIZE 163840		// Total shadowstack size
-#define QCGC_ARENA_BAG_INIT_SIZE 16			// Initial size of the arena bag
-#define QCGC_ARENA_SIZE_EXP 20				// Between 16 (64kB) and 20 (1MB)
-#define QCGC_LARGE_ALLOC_THRESHOLD_EXP 14	// Less than QCGC_ARENA_SIZE_EXP
-#define QCGC_MARK_LIST_SEGMENT_SIZE 64		// TODO: Tune for performance
-#define QCGC_GRAY_STACK_INIT_SIZE 128		// TODO: Tune for performance
-#define QCGC_INC_MARK_MIN 64				// TODO: Tune for performance
-
-/**
- * Fit allocator
- */
-#define QCGC_LARGE_FREE_LIST_FIRST_EXP 5	// First exponent of large free list
-#define QCGC_LARGE_FREE_LIST_INIT_SIZE 4	// Initial size for large free lists
-#define QCGC_SMALL_FREE_LIST_INIT_SIZE 16	// Initial size for small free lists
-
-/**
- * Auto Mark/Collect
- */
-#define QCGC_MAJOR_COLLECTION_THRESHOLD (5 * (1<<QCGC_ARENA_SIZE_EXP))
-#define QCGC_INCMARK_THRESHOLD (1<<QCGC_ARENA_SIZE_EXP)
-
-/**
- * DO NOT MODIFY BELOW HERE
- */
-
-#if QCGC_LARGE_ALLOC_THRESHOLD_EXP >= QCGC_ARENA_SIZE_EXP
-#error	"Inconsistent configuration. Huge block threshold must be smaller " \
-		"than the arena size."
-#endif
-
-#ifdef TESTING
-#define QCGC_STATIC
-#define QCGC_INLINE
-#else
-#define QCGC_STATIC static
-#define QCGC_INLINE inline __attribute((always_inline))
-#endif
-
-#define MAX(a,b) (((a)>(b))?(a):(b))
-#define MIN(a,b) (((a)<(b))?(a):(b))
-#define UNUSED(x) (void)(x)
-#define LIKELY(x) __builtin_expect((x), 1)
-#define UNLIKELY(x) __builtin_expect((x), 0)
diff --git a/rpython/translator/c/src/qcgc/src/core.c b/rpython/translator/c/src/qcgc/src/core.c
--- a/rpython/translator/c/src/qcgc/src/core.c
+++ b/rpython/translator/c/src/qcgc/src/core.c
@@ -59,15 +59,6 @@
 	free(qcgc_state.gp_gray_stack);
 }
 
-void qcgc_push_root(object_t *object) {
-	*qcgc_state.shadow_stack = object;
-	qcgc_state.shadow_stack++;
-}
-
-void qcgc_pop_root(void) {
-	qcgc_state.shadow_stack--;
-}
-
 object_t *qcgc_allocate(size_t size) {
 #if LOG_ALLOCATION
 	qcgc_event_logger_log(EVENT_ALLOCATE_START, sizeof(size_t),
diff --git a/rpython/translator/c/src/qcgc/src/core.h b/rpython/translator/c/src/qcgc/src/core.h
--- a/rpython/translator/c/src/qcgc/src/core.h
+++ b/rpython/translator/c/src/qcgc/src/core.h
@@ -9,8 +9,6 @@
 
 void qcgc_initialize(void);
 void qcgc_destroy(void);
-void qcgc_push_root(object_t *object);
-void qcgc_pop_root(void);
 object_t *qcgc_allocate(size_t size);
 void qcgc_collect(void);
 void qcgc_write(object_t *object);
diff --git a/rpython/translator/c/src/qcgc/src/event_logger.h b/rpython/translator/c/src/qcgc/src/event_logger.h
--- a/rpython/translator/c/src/qcgc/src/event_logger.h
+++ b/rpython/translator/c/src/qcgc/src/event_logger.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "config.h"
+#include "../config.h"
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/rpython/translator/c/src/qcgc/src/gc_state.h b/rpython/translator/c/src/qcgc/src/gc_state.h
--- a/rpython/translator/c/src/qcgc/src/gc_state.h
+++ b/rpython/translator/c/src/qcgc/src/gc_state.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include "../qcgc.h"
+
 #include <stddef.h>
 
 #include "bag.h"
@@ -25,8 +27,6 @@
  * Global state of the garbage collector
  */
 struct qcgc_state {
-	object_t **shadow_stack;
-	object_t **shadow_stack_base;
 	shadow_stack_t *prebuilt_objects;
 	weakref_bag_t *weakrefs;
 	gray_stack_t *gp_gray_stack;
@@ -43,6 +43,4 @@
 	size_t largest_free_block;	// Size of the largest free block.
 								// (Free arenas don't count as free blocks)
 								// Valid right after sweep
-};
-
-QCGC_STATIC struct qcgc_state qcgc_state;
+} qcgc_state;
diff --git a/rpython/translator/c/src/qcgc/src/gray_stack.h b/rpython/translator/c/src/qcgc/src/gray_stack.h
--- a/rpython/translator/c/src/qcgc/src/gray_stack.h
+++ b/rpython/translator/c/src/qcgc/src/gray_stack.h
@@ -1,10 +1,6 @@
 #pragma once
 
-#include "config.h"
-
-#include <stddef.h>
-
-#include "object.h"
+#include "../qcgc.h"
 
 typedef struct gray_stack_s {
 	size_t index;
diff --git a/rpython/translator/c/src/qcgc/src/hugeblocktable.c b/rpython/translator/c/src/qcgc/src/hugeblocktable.c
--- a/rpython/translator/c/src/qcgc/src/hugeblocktable.c
+++ b/rpython/translator/c/src/qcgc/src/hugeblocktable.c
@@ -2,8 +2,6 @@
 
 #include <assert.h>
 
-#include "gc_state.h"
-
 QCGC_STATIC size_t bucket(object_t *object);
 
 void qcgc_hbtable_initialize(void) {
diff --git a/rpython/translator/c/src/qcgc/src/hugeblocktable.h b/rpython/translator/c/src/qcgc/src/hugeblocktable.h
--- a/rpython/translator/c/src/qcgc/src/hugeblocktable.h
+++ b/rpython/translator/c/src/qcgc/src/hugeblocktable.h
@@ -1,11 +1,10 @@
 #pragma once
 
-#include "config.h"
+#include "../qcgc.h"
 
 #include <stdbool.h>
 
 #include "bag.h"
-#include "object.h"
 #include "gray_stack.h"
 
 // Choosing a prime number, hoping for good results
diff --git a/rpython/translator/c/src/qcgc/src/mark_list.c b/rpython/translator/c/src/qcgc/src/mark_list.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/mark_list.c
+++ /dev/null
@@ -1,180 +0,0 @@
-#include "mark_list.h"
-
-#include <assert.h>
-
-#include <stdlib.h>
-#include <string.h>
-
-QCGC_STATIC mark_list_t *qcgc_mark_list_grow(mark_list_t *list);
-QCGC_STATIC void qcgc_mark_list_check_invariant(mark_list_t *list);
-
-mark_list_t *qcgc_mark_list_create(size_t initial_size) {
-	size_t length = (initial_size + QCGC_MARK_LIST_SEGMENT_SIZE - 1) / QCGC_MARK_LIST_SEGMENT_SIZE;
-	length += (size_t) length == 0;
-	mark_list_t *result = (mark_list_t *)
-		malloc(sizeof(mark_list_t) + length * sizeof(object_t **));
-	result->head = 0;
-	result->tail = 0;
-	result->length = length;
-	result->insert_index = 0;
-	result->count = 0;
-	result->segments[result->head] = (object_t **)
-		calloc(QCGC_MARK_LIST_SEGMENT_SIZE, sizeof(object_t *));
-#if CHECKED
-	qcgc_mark_list_check_invariant(result);
-#endif
-	return result;
-}
-
-void qcgc_mark_list_destroy(mark_list_t *list) {
-#if CHECKED
-	qcgc_mark_list_check_invariant(list);
-#endif
-
-	size_t i = list->head;
-	while (i != list->tail) {
-		free(list->segments[i]);
-		i = (i + 1) % list->length;
-	}
-	free(list->segments[list->tail]);
-	free(list);
-}
-
-mark_list_t *qcgc_mark_list_push(mark_list_t *list, object_t *object) {
-#if CHECKED
-	assert(list != NULL);
-	assert(object != NULL);
-
-	qcgc_mark_list_check_invariant(list);
-	size_t old_count = list->count;
-#endif
-	if (list->insert_index >= QCGC_MARK_LIST_SEGMENT_SIZE) {
-		if ((list->tail + 1) % list->length == list->head) {
-			list = qcgc_mark_list_grow(list);
-		}
-		list->insert_index = 0;
-		list->tail = (list->tail + 1) % list->length;
-		list->segments[list->tail] = (object_t **)
-			calloc(QCGC_MARK_LIST_SEGMENT_SIZE, sizeof(object_t *));
-	}
-	list->segments[list->tail][list->insert_index] = object;
-	list->insert_index++;
-	list->count++;
-#if CHECKED
-	assert(list->count == old_count + 1);
-	assert(list->segments[list->tail][list->insert_index - 1] == object);
-	qcgc_mark_list_check_invariant(list);
-#endif
-	return list;
-}
-
-mark_list_t *qcgc_mark_list_push_all(mark_list_t *list,
-		object_t **objects, size_t count) {
-#if CHECKED
-	assert(list != NULL);
-	assert(objects != NULL);
-
-	qcgc_mark_list_check_invariant(list);
-
-	size_t old_count = list->count;
-	for (size_t i = 0; i < count; i++) {
-		assert(objects[i] != NULL);
-	}
-#endif
-	// FIXME: Optimize or remove
-	for (size_t i = 0; i < count; i++) {
-		list = qcgc_mark_list_push(list, objects[i]);
-	}
-#if CHECKED
-	assert(list->count == old_count + count);
-	qcgc_mark_list_check_invariant(list);
-#endif
-	return list;
-}
-
-object_t **qcgc_mark_list_get_head_segment(mark_list_t *list) {
-#if CHECKED
-	assert(list != NULL);
-	assert(list->segments[list->head] != NULL);
-	qcgc_mark_list_check_invariant(list);
-#endif
-	return list->segments[list->head];
-}
-
-mark_list_t *qcgc_mark_list_drop_head_segment(mark_list_t *list) {
-#if CHECKED
-	assert(list != NULL);
-	size_t old_head = list->head;
-	size_t old_tail = list->tail;
-	qcgc_mark_list_check_invariant(list);
-#endif
-	if (list->head != list->tail) {
-		free(list->segments[list->head]);
-		list->segments[list->head] = NULL;
-		list->head = (list->head + 1) % list->length;
-		list->count -= QCGC_MARK_LIST_SEGMENT_SIZE;
-	} else {
-		memset(list->segments[list->head], 0,
-				sizeof(object_t *) * QCGC_MARK_LIST_SEGMENT_SIZE);
-		list->insert_index = 0;
-		list->count = 0;
-	}
-#if CHECKED
-	assert(old_tail == list->tail);
-	if (old_head == old_tail) {
-		assert(old_head == list->head);
-	} else {
-		assert((old_head + 1) % list->length == list->head);
-	}
-	qcgc_mark_list_check_invariant(list);
-#endif
-	return list;
-}
-
-QCGC_STATIC mark_list_t *qcgc_mark_list_grow(mark_list_t *list) {
-#if CHECKED
-	assert(list != NULL);
-	size_t old_length = list->length;
-	size_t old_tail = list->tail;
-	qcgc_mark_list_check_invariant(list);
-#endif
-	mark_list_t *new_list = (mark_list_t *) realloc(list,
-			sizeof(mark_list_t) + 2 * list->length * sizeof(object_t **));
-	if (new_list->tail < new_list->head) {
-		memcpy(new_list->segments + new_list->length,
-				new_list->segments, (new_list->tail + 1) * sizeof(object_t **));
-		new_list->tail = new_list->length + new_list->tail;
-	}
-	new_list->length = 2 * new_list->length;
-#if CHECKED
-	assert(new_list->length == 2 * old_length);
-	if (old_tail < new_list->head) {
-		assert(new_list->tail == old_tail + old_length);
-		for (size_t i = 0; i < old_tail; i++) {
-			assert(new_list->segments[i] == new_list->segments[i + old_length]);
-		}
-	} else {
-		assert(new_list->tail == old_tail);
-	}
-	qcgc_mark_list_check_invariant(new_list);
-#endif
-	return new_list;
-}
-
-QCGC_STATIC void qcgc_mark_list_check_invariant(mark_list_t *list) {
-	assert(list->head < list->length);
-	assert(list->tail < list->length);
-	assert(list->count == (list->tail - list->head + list->length) % list->length * QCGC_MARK_LIST_SEGMENT_SIZE + list->insert_index);
-	for (size_t i = 0; i < list->length; i++) {
-		if ((list->head <= i && i <= list->tail) || (list->tail < list->head &&
-				(i <= list->tail || i >= list->head))) {
-			for (size_t j = 0; j < QCGC_MARK_LIST_SEGMENT_SIZE; j++) {
-				if (i != list->tail || j < list->insert_index) {
-					assert(list->segments[i][j] != NULL);
-				} else {
-					assert(list->segments[i][j] == NULL);
-				}
-			}
-		}
-	}
-}
diff --git a/rpython/translator/c/src/qcgc/src/mark_list.h b/rpython/translator/c/src/qcgc/src/mark_list.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/mark_list.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * @file	mark_list.h
- *
- * Object list for marking step
- */
-
-#pragma once
-
-#include "config.h"
-
-#include <stddef.h>
-
-#include "object.h"
-
-/**
- * Mark list - circular buffer.
- */
-typedef struct mark_list_s {
-	size_t head;
-	size_t tail;
-	size_t length;
-	size_t count;
-	size_t insert_index;
-	object_t **segments[];
-} mark_list_t;
-
-__attribute__ ((warn_unused_result))
-mark_list_t *qcgc_mark_list_create(size_t initial_size);
-void qcgc_mark_list_destroy(mark_list_t *list);
-
-__attribute__ ((warn_unused_result))
-mark_list_t *qcgc_mark_list_push(mark_list_t *list, object_t *object);
-
-__attribute__ ((warn_unused_result))
-mark_list_t *qcgc_mark_list_push_all(mark_list_t *list,
-		object_t **objects, size_t count);
-
-object_t **qcgc_mark_list_get_head_segment(mark_list_t *list);
-
-__attribute__ ((warn_unused_result))
-mark_list_t *qcgc_mark_list_drop_head_segment(mark_list_t *list);
diff --git a/rpython/translator/c/src/qcgc/src/object.h b/rpython/translator/c/src/qcgc/src/object.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/object.h
+++ /dev/null
@@ -1,17 +0,0 @@
-#pragma once
-
-
-#include "config.h"
-#include <stdint.h>
-
-/**
- * The lower half of flags is reserved for the library, the upper half for
- * clients
- */
-#define QCGC_GRAY_FLAG (1<<0)
-#define QCGC_PREBUILT_OBJECT (1<<1)
-#define QCGC_PREBUILT_REGISTERED (1<<2)
-
-typedef struct object_s {
-	uint32_t flags;
-} object_t;
diff --git a/rpython/translator/c/src/qcgc/src/shadow_stack.h b/rpython/translator/c/src/qcgc/src/shadow_stack.h
--- a/rpython/translator/c/src/qcgc/src/shadow_stack.h
+++ b/rpython/translator/c/src/qcgc/src/shadow_stack.h
@@ -1,10 +1,6 @@
 #pragma once
 
-#include "config.h"
-
-#include <stddef.h>
-
-#include "object.h"
+#include "../qcgc.h"
 
 typedef struct shadow_stack_s {
 	size_t count;
diff --git a/rpython/translator/c/src/qcgc/src/signal_handler.c b/rpython/translator/c/src/qcgc/src/signal_handler.c
--- a/rpython/translator/c/src/qcgc/src/signal_handler.c
+++ b/rpython/translator/c/src/qcgc/src/signal_handler.c
@@ -7,7 +7,6 @@
 
 #include "arena.h"
 #include "allocator.h"
-#include "gc_state.h"
 
 QCGC_STATIC void handle_error(int signo, siginfo_t *siginfo, void *context);
 QCGC_STATIC bool is_stack_overflow(void *addr);
@@ -44,7 +43,7 @@
 }
 
 QCGC_STATIC bool is_stack_overflow(void *addr) {
-	void *shadow_stack_end = (void *)(qcgc_state.shadow_stack_base +
+	void *shadow_stack_end = (void *)(qcgc_shadowstack.base +
 		QCGC_SHADOWSTACK_SIZE);
 	return (addr >= shadow_stack_end && addr < shadow_stack_end + 8192);
 }
diff --git a/rpython/translator/c/src/qcgc/src/signal_handler.h b/rpython/translator/c/src/qcgc/src/signal_handler.h
--- a/rpython/translator/c/src/qcgc/src/signal_handler.h
+++ b/rpython/translator/c/src/qcgc/src/signal_handler.h
@@ -1,5 +1,5 @@
 #pragma once
 
-#include "config.h"
+#include "../qcgc.h"
 
 void setup_signal_handler(void);
diff --git a/rpython/translator/c/src/qcgc/src/weakref.c b/rpython/translator/c/src/qcgc/src/weakref.c
--- a/rpython/translator/c/src/qcgc/src/weakref.c
+++ b/rpython/translator/c/src/qcgc/src/weakref.c
@@ -2,33 +2,20 @@
 
 #include <assert.h>
 
+#include "arena.h"
 #include "bag.h"
 #include "gc_state.h"
 #include "hugeblocktable.h"
 
-void qcgc_register_weakref(object_t *weakrefobj, object_t **target) {
-#if CHECKED
-	assert((weakrefobj->flags & QCGC_PREBUILT_OBJECT) == 0);
-	assert((object_t *) qcgc_arena_addr((cell_t *) weakrefobj) != weakrefobj);
-#endif
-	// NOTE: At this point, the target must point to a pointer to a valid
-	// object. We don't register any weakrefs to prebuilt objects as they
-	// are always valid.
-	if (((*target)->flags & QCGC_PREBUILT_OBJECT) == 0) {
-		qcgc_state.weakrefs = qcgc_weakref_bag_add(qcgc_state.weakrefs,
-				(struct weakref_bag_item_s) {
-					.weakrefobj = weakrefobj,
-					.target = target});
-	}
-}
-
-QCGC_STATIC void update_weakrefs(void) {
+void update_weakrefs(void) {
 	size_t i = 0;
 	while (i < qcgc_state.weakrefs->count) {
 		struct weakref_bag_item_s item = qcgc_state.weakrefs->items[i];
 		// Check whether weakref object itself was collected
 		// We know the weakref object is a normal object
-		switch(qcgc_arena_get_blocktype((cell_t *) item.weakrefobj)) {
+		switch (qcgc_arena_get_blocktype(
+					qcgc_arena_addr((cell_t *) item.weakrefobj),
+					qcgc_arena_cell_index((cell_t *) item.weakrefobj))) {
 			case BLOCK_EXTENT: // Fall through
 			case BLOCK_FREE:
 				// Weakref itself was collected, forget it
@@ -57,7 +44,9 @@
 			}
 		} else {
 			// Normal object
-			switch(qcgc_arena_get_blocktype((cell_t *) points_to)) {
+			switch (qcgc_arena_get_blocktype(
+						qcgc_arena_addr((cell_t *) points_to),
+						qcgc_arena_cell_index((cell_t *) points_to))) {
 				case BLOCK_BLACK: // Still valid
 				case BLOCK_WHITE:
 					i++;
diff --git a/rpython/translator/c/src/qcgc/src/weakref.h b/rpython/translator/c/src/qcgc/src/weakref.h
--- a/rpython/translator/c/src/qcgc/src/weakref.h
+++ b/rpython/translator/c/src/qcgc/src/weakref.h
@@ -1,8 +1,5 @@
 #pragma once
 
-#include "config.h"
+#include "../qcgc.h"
 
-#include "object.h"
-
-void qcgc_register_weakref(object_t *weakrefobj, object_t **target);
-QCGC_STATIC void update_weakrefs(void);
+void update_weakrefs(void);
diff --git a/rpython/translator/platform/linux.py b/rpython/translator/platform/linux.py
--- a/rpython/translator/platform/linux.py
+++ b/rpython/translator/platform/linux.py
@@ -14,7 +14,7 @@
     extra_libs = ('-lrt',)
     cflags = tuple(
              ['-O3', '-pthread', '-fomit-frame-pointer',
-              '-Wall', '-Wno-unused']
+              '-Wall', '-Wno-unused', '-g']
              + os.environ.get('CFLAGS', '').split())
     standalone_only = ()
     shared_only = ('-fPIC',)

From pypy.commits at gmail.com  Wed Sep 14 02:44:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 23:44:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: profopt was accidentally silently
 disabled on all platforms, by
Message-ID: <57d8f1b7.0145c20a.5164e.bdd8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87103:10eaf9cac6ad
Date: 2016-09-14 08:43 +0200
http://bitbucket.org/pypy/pypy/changeset/10eaf9cac6ad/

Log:	profopt was accidentally silently disabled on all platforms, by
	7b21b053dd02.

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -275,6 +275,7 @@
         if retval and self.translator.platform.name == 'msvc':
             raise ValueError('Cannot do profile based optimization on MSVC,'
                     'it is not supported in free compiler version')
+        return retval
 
     def getentrypointptr(self):
         # XXX check that the entrypoint has the correct

From pypy.commits at gmail.com  Wed Sep 14 02:49:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Sep 2016 23:49:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: Write down a comment about profopt in
 JIT-enabled binaries
Message-ID: <57d8f308.8aabc20a.696bb.c32d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87104:4b6221995df7
Date: 2016-09-14 08:49 +0200
http://bitbucket.org/pypy/pypy/changeset/4b6221995df7/

Log:	Write down a comment about profopt in JIT-enabled binaries

diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt
--- a/pypy/doc/config/translation.profopt.txt
+++ b/pypy/doc/config/translation.profopt.txt
@@ -3,3 +3,14 @@
 RPython program) to gather profile data. Example for pypy-c: "-c 'from
 richards import main;main(); from test import pystone;
 pystone.main()'"
+
+NOTE: be aware of what this does in JIT-enabled executables.  What it
+does is instrument and later optimize the C code that happens to run in
+the example you specify, ignoring any execution of the JIT-generated
+assembler.  That means that you have to choose the example wisely.  If
+it is something that will just generate assembler and stay there, there
+is little value.  If it is something that exercises heavily library
+routines that are anyway written in C, then it will optimize that.  Most
+interesting would be something that causes a lot of JIT-compilation,
+like running a medium-sized test suite several times in a row, in order
+to optimize the warm-up in general.

From pypy.commits at gmail.com  Wed Sep 14 07:01:05 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 14 Sep 2016 04:01:05 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: load, store, add,
 subtract vector versions
Message-ID: <57d92df1.669fc20a.e4731.2180@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87105:4802a50159bf
Date: 2016-09-14 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/4802a50159bf/

Log:	load, store, add, subtract vector versions

diff --git a/rpython/jit/backend/zarch/detect_feature.py b/rpython/jit/backend/zarch/detect_feature.py
--- a/rpython/jit/backend/zarch/detect_feature.py
+++ b/rpython/jit/backend/zarch/detect_feature.py
@@ -5,18 +5,16 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.rmmap import alloc, free
 from rpython.rlib.rstruct.runpack import runpack
-from rpython.translator.platform.arch.s390x import s390x_cpu_revision
+from rpython.translator.platform.arch.s390x import s390x_detect_vx
 
 SYSTEM = platform.system()
 
 def detect_simd_z_linux():
-    return False
+    return s390x_detect_vx()
 
 def detect_simd_z():
     if SYSTEM == 'Linux':
-        machine = s390x_cpu_revision()
-        if machine == "z13":
-            return True
+        return detect_simd_z_linux()
     return False
 
 if __name__ == '__main__':
diff --git a/rpython/jit/backend/zarch/instruction_builder.py b/rpython/jit/backend/zarch/instruction_builder.py
--- a/rpython/jit/backend/zarch/instruction_builder.py
+++ b/rpython/jit/backend/zarch/instruction_builder.py
@@ -5,7 +5,7 @@
 
 def dummy_argument(arg):
     """ NOT_RPYTHON """
-    if arg in ('r', 'r/m', 'm', 'f', '-', 'eo'):
+    if arg in ('r', 'r/m', 'm', 'f', '-', 'eo', 'v'):
         return 0
     if arg.startswith('i') or arg.startswith('u'):
         return 0
@@ -23,6 +23,7 @@
         -      - unused
         f      - floating point register
         r      - register
+        v      - vector register (128 bit)
         m      - mask
         eo     - even odd pair (= the even register)
         r/m    - register or mask
@@ -447,7 +448,6 @@
     return encode_rxe
 
 def build_ris(mnemonic, (opcode1,opcode2), argtypes='r,i8,r/m,bd'):
-    br = is_branch_relative(mnemonic)
     @builder.arguments(argtypes)
     def encode_rie_c(self, reg1, imm8, mask, basedisp):
         self.writechar(opcode1)
@@ -459,6 +459,50 @@
         self.writechar(opcode2)
     return encode_rie_c
 
+def build_vrx(mnemonic, (opcode1,opcode2), argtypes='v,bid,m'):
+    @builder.arguments(argtypes)
+    def encode_vrx(self, v1, bid, mask=0):
+        self.writechar(opcode1)
+        rbx = (v1 >= 16) << 3
+        idx = bid.index
+        byte = (v1 & BIT_MASK_4) << 4 | (idx & BIT_MASK_4)
+        self.writechar(chr(byte))
+        encode_base_displace(self, bid)
+        self.writechar(chr((mask & BIT_MASK_4 << 4) | (rbx & BIT_MASK_4)))
+        self.writechar(opcode2)
+    return encode_vrx
+
+def build_vrr_a(mnemonic, (opcode1,opcode2), argtypes='v,v'):
+    @builder.arguments(argtypes)
+    def encode_vrr_a(self, v1, v2):
+        self.writechar(opcode1)
+        rbx =  (v1 >= 16) << 3
+        rbx |= (v2 >= 16) << 2
+        byte = (v1 & BIT_MASK_4) << 4 | (v2 & BIT_MASK_4)
+        self.writechar(chr(byte))
+        self.writechar(chr(0))
+        self.writechar(chr(0))
+        self.writechar(chr(rbx & BIT_MASK_4))
+        self.writechar(opcode2)
+    return encode_vrr_a
+
+def build_vrr_c(mnemonic, (opcode1,opcode2), argtypes='v,v,v,m,m'):
+    @builder.arguments(argtypes)
+    def encode_vrr_c(self, v1, v2, v3, mask1, mask2):
+        self.writechar(opcode1)
+        rbx =  (v1 >= 16) << 3
+        rbx |= (v2 >= 16) << 2
+        rbx |= (v3 >= 16) << 1
+        byte = (v1 & BIT_MASK_4) << 4 | (v2 & BIT_MASK_4)
+        self.writechar(chr(byte))
+        byte = (v3 & BIT_MASK_4) << 4
+        self.writechar(chr(byte))
+        self.writechar(chr(mask2 & BIT_MASK_4))
+        self.writechar(chr((mask1 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
+        self.writechar(opcode2)
+    return encode_vrr_c
+
+
 def build_unpack_func(mnemonic, func):
     @always_inline
     def check_arg_type(arg, type):
@@ -502,7 +546,8 @@
         if argtype == '-':
             return 0
         elif argtype == 'r' or argtype == 'r/m' or \
-             argtype == 'f' or argtype == 'eo':
+             argtype == 'f' or argtype == 'eo' or \
+             argtype == 'v':
             return arg.value
         elif argtype.startswith('i') or argtype.startswith('u') or argtype.startswith('h'):
             return arg.value
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -292,11 +292,26 @@
 
     'STFLE':   ('s',     ['\xB2','\xB0']),
 }
+
+vector_mnemonic_codes = {
+    'VL':     ('vrx', ['\xE7','\x06'], 'v,bid'),
+    'VLR':    ('vrr_a', ['\xE7','\x56']),
+
+    'VST':    ('vrx', ['\xE7','\x0E'], 'v,bid'),
+
+    # floating point
+    'VFA':   ('vrr_c', ['\xE7','\xE3']),
+    'VFS':   ('vrr_c', ['\xE7','\xE2']),
+
+    # '': ('', ['','']),
+}
+
 all_mnemonic_codes.update(arith_mnemonic_codes)
 all_mnemonic_codes.update(logic_mnemonic_codes)
 all_mnemonic_codes.update(memory_mnemonic_codes)
 all_mnemonic_codes.update(floatingpoint_mnemonic_codes)
 all_mnemonic_codes.update(branch_mnemonic_codes)
+all_mnemonic_codes.update(vector_mnemonic_codes)
 
 
 if __name__ == "__main__":
diff --git a/rpython/jit/backend/zarch/locations.py b/rpython/jit/backend/zarch/locations.py
--- a/rpython/jit/backend/zarch/locations.py
+++ b/rpython/jit/backend/zarch/locations.py
@@ -107,6 +107,26 @@
     def is_float(self):
         return True
 
+class VectorRegisterLocation(RegisterLocation):
+    _immutable_ = True
+    type = FLOAT
+    width = DOUBLE_WORD*2
+
+    def __repr__(self):
+        return 'v%d' % self.value
+
+    def is_core_reg(self):
+        return False
+
+    def is_fp_reg(self):
+        return True
+
+    def as_key(self):            # 16 <= as_key <= 32
+        return self.value + 32
+
+    def is_float(self):
+        return True
+
 class ImmLocation(AssemblerLocation):
     _immutable_ = True
     width = WORD
@@ -176,6 +196,9 @@
         if length:
             self.length = length.value
 
+    def __repr__(self):
+        return 'addr(base=r%d,idx=r%d,len=%d)' % (self.base, self.index, self.length)
+
 class PoolLoc(AddressLocation):
     _immutable_ = True
     width = WORD
diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py
--- a/rpython/jit/backend/zarch/regalloc.py
+++ b/rpython/jit/backend/zarch/regalloc.py
@@ -109,6 +109,46 @@
         self.temp_boxes.append(box)
         return reg
 
+class VectorRegisterManager(RegisterManager):
+    all_regs              = r.MANAGED_VECTOR_REGS
+    box_types             = [FLOAT, INT]
+    save_around_call_regs = [] # calling not allowed in vectorized traces!
+    assert set(save_around_call_regs).issubset(all_regs)
+    pool = None
+
+    def __init__(self, longevity, frame_manager=None, assembler=None):
+        RegisterManager.__init__(self, longevity, frame_manager, assembler)
+
+    def call_result_location(self, v):
+        return None
+
+    def convert_to_imm(self, c):
+        return l.pool(self.assembler.pool.get_offset(c), float=True)
+
+    def ensure_reg_or_pool(self, box):
+        if isinstance(box, Const):
+            offset = self.assembler.pool.get_offset(box)
+            return l.pool(offset, float=True)
+        else:
+            assert box in self.temp_boxes
+            loc = self.make_sure_var_in_reg(box,
+                    forbidden_vars=self.temp_boxes)
+        return loc
+
+    def ensure_reg(self, box):
+        assert box in self.temp_boxes
+        loc = self.make_sure_var_in_reg(box,
+                forbidden_vars=self.temp_boxes)
+        return loc
+
+    def get_scratch_reg(self, selected_reg=None):
+        # TODO
+        box = TempFloat()
+        reg = self.force_allocate_reg(box, forbidden_vars=self.temp_boxes, selected_reg=selected_reg)
+        self.temp_boxes.append(box)
+        return reg
+
+
 
 class ZARCHRegisterManager(RegisterManager):
     all_regs              = r.MANAGED_REGS
@@ -389,8 +429,9 @@
         assert isinstance(loc, l.StackLocation)
         return loc.position
 
+from rpython.jit.backend.zarch import vector_ext
 
-class Regalloc(BaseRegalloc):
+class Regalloc(BaseRegalloc, vector_ext.VectorRegalloc):
 
     def __init__(self, assembler=None):
         self.cpu = assembler.cpu
@@ -415,6 +456,9 @@
         self.fprm = FPRegisterManager(self.longevity, frame_manager = self.fm,
                                       assembler = self.assembler)
         self.fprm.pool = self.assembler.pool
+        self.vrm = VectorRegisterManager(self.longevity, frame_manager = self.fm,
+                                         assembler = self.assembler)
+        self.vrm.pool = self.assembler.pool
         return operations
 
     def prepare_loop(self, inputargs, operations, looptoken, allgcrefs):
@@ -470,13 +514,17 @@
         self.fm.finish_binding()
         self.rm._check_invariants()
         self.fprm._check_invariants()
+        self.vrm._check_invariants()
 
     def get_final_frame_depth(self):
         return self.fm.get_frame_depth()
 
     def possibly_free_var(self, var):
         if var is not None:
-            if var.type == FLOAT:
+            if var.is_vector():
+                if var.type != VOID:
+                    self.vrm.possibly_free_var(var)
+            elif var.type == FLOAT:
                 self.fprm.possibly_free_var(var)
             else:
                 self.rm.possibly_free_var(var)
@@ -533,6 +581,7 @@
             self.assembler.mc.mark_op(op)
             self.rm.position = i
             self.fprm.position = i
+            self.vrm.position = i
             opnum = op.getopnum()
             if rop.has_no_side_effect(opnum) and op not in self.longevity:
                 i += 1
@@ -541,7 +590,10 @@
             #
             for j in range(op.numargs()):
                 box = op.getarg(j)
-                if box.type != FLOAT:
+                if box.is_vector():
+                    if box.type != VOID:
+                        self.vrm.temp_boxes.append(box)
+                elif box.type != FLOAT:
                     self.rm.temp_boxes.append(box)
                 else:
                     self.fprm.temp_boxes.append(box)
@@ -555,6 +607,7 @@
             self.possibly_free_var(op)
             self.rm._check_invariants()
             self.fprm._check_invariants()
+            self.vrm._check_invariants()
             if self.assembler.mc.get_relative_pos() > self.limit_loop_break:
                 self.assembler.break_long_loop()
                 self.limit_loop_break = (self.assembler.mc.get_relative_pos() +
@@ -562,6 +615,7 @@
             i += 1
         assert not self.rm.reg_bindings
         assert not self.fprm.reg_bindings
+        assert not self.vrm.reg_bindings
         self.flush_loop()
         self.assembler.mc.mark_op(None) # end of the loop
         self.operations = None
@@ -677,6 +731,7 @@
         # temporary boxes and all the current operation's arguments
         self.rm.free_temp_vars()
         self.fprm.free_temp_vars()
+        self.vrm.free_temp_vars()
 
     def compute_hint_frame_locations(self, operations):
         # optimization only: fill in the 'hint_frame_locations' dictionary
diff --git a/rpython/jit/backend/zarch/registers.py b/rpython/jit/backend/zarch/registers.py
--- a/rpython/jit/backend/zarch/registers.py
+++ b/rpython/jit/backend/zarch/registers.py
@@ -1,8 +1,9 @@
-from rpython.jit.backend.zarch.locations import FloatRegisterLocation
-from rpython.jit.backend.zarch.locations import RegisterLocation
+from rpython.jit.backend.zarch.locations import (FloatRegisterLocation,
+        RegisterLocation, VectorRegisterLocation)
 
 registers = [RegisterLocation(i) for i in range(16)]
 fpregisters = [FloatRegisterLocation(i) for i in range(16)]
+vregisters = [VectorRegisterLocation(16+i) for i in range(16)]
 
 [r0,r1,r2,r3,r4,r5,r6,r7,r8,
  r9,r10,r11,r12,r13,r14,r15] = registers
@@ -30,6 +31,8 @@
 MANAGED_FP_REGS = fpregisters[:-1]
 VOLATILES_FLOAT = [f0,f1,f2,f3,f4,f5,f6,f7]
 
+MANAGED_VECTOR_REGS = vregisters
+
 # The JITFRAME_FIXED_SIZE is measured in words, and should be the
 # number of registers that need to be saved into the jitframe when
 # failing a guard, for example.
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -15,6 +15,7 @@
 import rpython.jit.backend.zarch.registers as r
 import rpython.jit.backend.zarch.conditions as c
 import rpython.jit.backend.zarch.locations as l
+from rpython.jit.backend.zarch.locations import imm
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.jit.codewriter import longlong
@@ -92,35 +93,16 @@
         pass
 
     def emit_vec_load_f(self, op, arglocs, regalloc):
-        resloc, baseloc, indexloc, size_loc, ofs, integer_loc = arglocs
-        indexloc = self._apply_offset(indexloc, ofs)
-        itemsize = size_loc.value
-        if integer_loc.value:
-            self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
-        elif itemsize == 4:
-            self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
-        elif itemsize == 8:
-            self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
-        else:
-            not_implemented("vec_load_f itemsize %d" % itemsize)
+        resloc, baseloc, indexloc, size_loc, offsetloc, integer_loc = arglocs
+        addrloc = self._load_address(baseloc, indexloc, offsetloc)
+        self.mc.VL(resloc, addrloc)
 
     emit_vec_load_i = emit_vec_load_f
 
     def emit_vec_store(self, op, arglocs, regalloc):
-        baseloc, indexloc, valueloc, sizeloc, baseofs, \
-            integer_loc = arglocs
-        indexloc = self._apply_offset(indexloc, baseofs)
-        assert baseofs.value == 0
-        if integer_loc.value:
-            self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
-        else:
-            itemsize = sizeloc.value
-            if itemsize == 4:
-                self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value)
-            elif itemsize == 8:
-                self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
-            else:
-                not_implemented("vec_store itemsize %d" % itemsize)
+        baseloc, indexloc, valueloc, sizeloc, offsetloc, integer_loc = arglocs
+        addrloc = self._load_address(baseloc, indexloc, offsetloc)
+        self.mc.VST(valueloc, addrloc)
 
     def emit_vec_int_add(self, op, arglocs, regalloc):
         resloc, loc0, loc1, size_loc = arglocs
@@ -152,18 +134,18 @@
     def emit_vec_float_add(self, op, arglocs, regalloc):
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
-        if itemsize == 4:
-            self.mc.xvaddsp(resloc.value, loc0.value, loc1.value)
-        elif itemsize == 8:
-            self.mc.xvadddp(resloc.value, loc0.value, loc1.value)
+        if itemsize == 8:
+            self.mc.VFA(resloc, loc0, loc1, 3, 0)
+            return
+        not_implemented("vec_float_add of size %d" % itemsize)
 
     def emit_vec_float_sub(self, op, arglocs, regalloc):
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
-        if itemsize == 4:
-            self.mc.xvsubsp(resloc.value, loc0.value, loc1.value)
-        elif itemsize == 8:
-            self.mc.xvsubdp(resloc.value, loc0.value, loc1.value)
+        if itemsize == 8:
+            self.mc.VFS(resloc, loc0, loc1, 3, 0)
+            return
+        not_implemented("vec_float_add of size %d" % itemsize)
 
     def emit_vec_float_mul(self, op, arglocs, regalloc):
         resloc, loc0, loc1, itemsize_loc = arglocs
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -38,6 +38,18 @@
 
     return ids
 
+def s390x_detect_vx():
+    with open("/proc/cpuinfo", "rb") as fd:
+        lines = fd.read().splitlines()
+        for line in lines:
+            if line.startswith("features"):
+                colonidx = line.find(':')
+                split = line[colonidx+1:].strip().split(' ')
+                if 'vx' in split:
+                    return True
+                break
+
+    return False
 
 def s390x_cpu_revision():
     # linux kernel does the same classification

From pypy.commits at gmail.com  Wed Sep 14 07:01:07 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 14 Sep 2016 04:01:07 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: more vector ops, add,
 subtract, logic or, and, xor, float mul, float div
Message-ID: <57d92df3.822e1c0a.6689a.7cd3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87106:155d0af20ad7
Date: 2016-09-14 12:54 +0200
http://bitbucket.org/pypy/pypy/changeset/155d0af20ad7/

Log:	more vector ops, add, subtract, logic or, and, xor, float mul, float
	div

diff --git a/rpython/jit/backend/zarch/instruction_builder.py b/rpython/jit/backend/zarch/instruction_builder.py
--- a/rpython/jit/backend/zarch/instruction_builder.py
+++ b/rpython/jit/backend/zarch/instruction_builder.py
@@ -488,7 +488,7 @@
 
 def build_vrr_c(mnemonic, (opcode1,opcode2), argtypes='v,v,v,m,m'):
     @builder.arguments(argtypes)
-    def encode_vrr_c(self, v1, v2, v3, mask1, mask2):
+    def encode_vrr_c(self, v1, v2, v3, mask1=0, mask2=0):
         self.writechar(opcode1)
         rbx =  (v1 >= 16) << 3
         rbx |= (v2 >= 16) << 2
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -299,9 +299,22 @@
 
     'VST':    ('vrx', ['\xE7','\x0E'], 'v,bid'),
 
+    # integral
+    # -> arith
+    'VA':     ('vrr_c', ['\xE7','\xF3'], 'v,v,v,m'),
+    'VS':     ('vrr_c', ['\xE7','\xF7'], 'v,v,v,m'),
+
+    # -> logic
+    'VO':     ('vrr_c', ['\xE7','\x6A'], 'v,v,v'),
+    'VNO':    ('vrr_c', ['\xE7','\x6B'], 'v,v,v'),
+    'VN':     ('vrr_c', ['\xE7','\x68'], 'v,v,v'),
+    'VX':     ('vrr_c', ['\xE7','\x6D'], 'v,v,v'),
+
     # floating point
     'VFA':   ('vrr_c', ['\xE7','\xE3']),
     'VFS':   ('vrr_c', ['\xE7','\xE2']),
+    'VFM':   ('vrr_c', ['\xE7','\xE7']),
+    'VFD':   ('vrr_c', ['\xE7','\xE5']),
 
     # '': ('', ['','']),
 }
diff --git a/rpython/jit/backend/zarch/locations.py b/rpython/jit/backend/zarch/locations.py
--- a/rpython/jit/backend/zarch/locations.py
+++ b/rpython/jit/backend/zarch/locations.py
@@ -250,3 +250,20 @@
 imm1 = imm(1)
 imm0 = imm(0)
 
+MASK_VEC_BYTE = 0
+MASK_VEC_HWORD = 1
+MASK_VEC_WORD = 2
+MASK_VEC_DWORD = 3
+
+def itemsize_to_mask(v):
+    if v == 16:
+        return MASK_VEC_QWORD
+    elif v == 8:
+        return MASK_VEC_DWORD
+    elif v == 4:
+        return MASK_VEC_WORD
+    elif v == 2:
+        return MASK_VEC_HWORD
+    elif v == 1:
+        return MASK_VEC_BYTE
+    assert 0, "not supported itemsize to mask!"
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -107,29 +107,13 @@
     def emit_vec_int_add(self, op, arglocs, regalloc):
         resloc, loc0, loc1, size_loc = arglocs
         size = size_loc.value
-        if size == 1:
-            self.mc.vaddubm(resloc.value, loc0.value, loc1.value)
-        elif size == 2:
-            self.mc.vadduhm(resloc.value, loc0.value, loc1.value)
-        elif size == 4:
-            self.mc.vadduwm(resloc.value, loc0.value, loc1.value)
-        elif size == 8:
-            self.mc.vaddudm(resloc.value, loc0.value, loc1.value)
+        mask = l.itemsize_to_mask(size_loc.value)
+        self.mc.VA(resloc, loc0, loc1, mask)
 
     def emit_vec_int_sub(self, op, arglocs, regalloc):
         resloc, loc0, loc1, size_loc = arglocs
-        size = size_loc.value
-        if size == 1:
-            # TODO verify if unsigned subtract is the wanted feature
-            self.mc.vsububm(resloc.value, loc0.value, loc1.value)
-        elif size == 2:
-            # TODO verify if unsigned subtract is the wanted feature
-            self.mc.vsubuhm(resloc.value, loc0.value, loc1.value)
-        elif size == 4:
-            # TODO verify if unsigned subtract is the wanted feature
-            self.mc.vsubuwm(resloc.value, loc0.value, loc1.value)
-        elif size == 8:
-            self.mc.vsubudm(resloc.value, loc0.value, loc1.value)
+        mask = l.itemsize_to_mask(size_loc.value)
+        self.mc.VS(resloc, loc0, loc1, mask)
 
     def emit_vec_float_add(self, op, arglocs, regalloc):
         resloc, loc0, loc1, itemsize_loc = arglocs
@@ -145,39 +129,40 @@
         if itemsize == 8:
             self.mc.VFS(resloc, loc0, loc1, 3, 0)
             return
-        not_implemented("vec_float_add of size %d" % itemsize)
+        not_implemented("vec_float_sub of size %d" % itemsize)
 
     def emit_vec_float_mul(self, op, arglocs, regalloc):
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
-        if itemsize == 4:
-            self.mc.xvmulsp(resloc.value, loc0.value, loc1.value)
-        elif itemsize == 8:
-            self.mc.xvmuldp(resloc.value, loc0.value, loc1.value)
+        if itemsize == 8:
+            self.mc.VFM(resloc, loc0, loc1, 3, 0)
+            return
+        not_implemented("vec_float_mul of size %d" % itemsize)
 
     def emit_vec_float_truediv(self, op, arglocs, regalloc):
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
-        if itemsize == 4:
-            self.mc.xvdivsp(resloc.value, loc0.value, loc1.value)
-        elif itemsize == 8:
-            self.mc.xvdivdp(resloc.value, loc0.value, loc1.value)
+        if itemsize == 8:
+            self.mc.VFD(resloc, loc0, loc1, 3, 0)
+            return
+        not_implemented("vec_float_truediv of size %d" % itemsize)
 
     def emit_vec_int_and(self, op, arglocs, regalloc):
         resloc, loc0, loc1, sizeloc = arglocs
-        self.mc.vand(resloc.value, loc0.value, loc1.value)
+        self.mc.VN(resloc, loc0, loc1)
 
     def emit_vec_int_or(self, op, arglocs, regalloc):
         resloc, loc0, loc1, sizeloc = arglocs
-        self.mc.vor(resloc.value, loc0.value, loc1.value)
+        self.mc.VO(resloc, loc0, loc1)
 
     def emit_vec_int_xor(self, op, arglocs, regalloc):
         resloc, loc0, loc1, sizeloc = arglocs
-        self.mc.vxor(resloc.value, loc0.value, loc1.value)
+        self.mc.VX(resloc, loc0, loc1)
 
     def emit_vec_int_signext(self, op, arglocs, regalloc):
         resloc, loc0 = arglocs
-        # TODO
+        # signext is only allowed if the data type sizes do not change.
+        # e.g. [byte,byte] = sign_ext([byte, byte]), a simple move is sufficient!
         self.regalloc_mov(loc0, resloc)
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):

From pypy.commits at gmail.com  Wed Sep 14 10:20:14 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 14 Sep 2016 07:20:14 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: more vector ops, int->float,
 comparison float
Message-ID: <57d95c9e.822e1c0a.6689a.cc04@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87107:4c3e946b5c05
Date: 2016-09-14 16:19 +0200
http://bitbucket.org/pypy/pypy/changeset/4c3e946b5c05/

Log:	more vector ops, int->float, comparison float

diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py
--- a/rpython/jit/backend/zarch/assembler.py
+++ b/rpython/jit/backend/zarch/assembler.py
@@ -919,6 +919,10 @@
         return frame_depth
 
     def regalloc_mov(self, prev_loc, loc):
+        if prev_loc.is_vector_reg():
+            assert loc.is_vector_reg()
+            self.mc.VLR(loc, prev_loc)
+            return
         if prev_loc.is_imm():
             value = prev_loc.getint()
             # move immediate value to register
diff --git a/rpython/jit/backend/zarch/conditions.py b/rpython/jit/backend/zarch/conditions.py
--- a/rpython/jit/backend/zarch/conditions.py
+++ b/rpython/jit/backend/zarch/conditions.py
@@ -18,7 +18,7 @@
 # normal branch instructions
 FLOAT = ConditionLocation(0x10)
 
-EQ = ConditionLocation(0x8)
+VEQI = EQ = ConditionLocation(0x8)
 LT = ConditionLocation(0x4)
 GT = ConditionLocation(0x2)
 OF = ConditionLocation(0x1) # overflow
@@ -27,7 +27,7 @@
 FLE = ConditionLocation(EQ.value | LT.value)
 GE = ConditionLocation(EQ.value | GT.value | OF.value)
 FGE = ConditionLocation(EQ.value | GT.value)
-NE = ConditionLocation(LT.value | GT.value | OF.value)
+VNEI = NE = ConditionLocation(LT.value | GT.value | OF.value)
 NO = ConditionLocation(0xe) # NO overflow
 
 FGT = ConditionLocation(GT.value | OF.value)
diff --git a/rpython/jit/backend/zarch/instruction_builder.py b/rpython/jit/backend/zarch/instruction_builder.py
--- a/rpython/jit/backend/zarch/instruction_builder.py
+++ b/rpython/jit/backend/zarch/instruction_builder.py
@@ -468,27 +468,27 @@
         byte = (v1 & BIT_MASK_4) << 4 | (idx & BIT_MASK_4)
         self.writechar(chr(byte))
         encode_base_displace(self, bid)
-        self.writechar(chr((mask & BIT_MASK_4 << 4) | (rbx & BIT_MASK_4)))
+        self.writechar(chr((mask & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
         self.writechar(opcode2)
     return encode_vrx
 
-def build_vrr_a(mnemonic, (opcode1,opcode2), argtypes='v,v'):
+def build_vrr_a(mnemonic, (opcode1,opcode2), argtypes='v,v,m,m,m'):
     @builder.arguments(argtypes)
-    def encode_vrr_a(self, v1, v2):
+    def encode_vrr_a(self, v1, v2, mask3=0, mask4=0, mask5=0):
         self.writechar(opcode1)
         rbx =  (v1 >= 16) << 3
         rbx |= (v2 >= 16) << 2
         byte = (v1 & BIT_MASK_4) << 4 | (v2 & BIT_MASK_4)
         self.writechar(chr(byte))
         self.writechar(chr(0))
-        self.writechar(chr(0))
-        self.writechar(chr(rbx & BIT_MASK_4))
+        self.writechar(chr((mask5 & BIT_MASK_4) << 4 | (mask4 & BIT_MASK_4)))
+        self.writechar(chr((mask3 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
         self.writechar(opcode2)
     return encode_vrr_a
 
-def build_vrr_c(mnemonic, (opcode1,opcode2), argtypes='v,v,v,m,m'):
+def build_vrr_c(mnemonic, (opcode1,opcode2), argtypes='v,v,v,m,m,m'):
     @builder.arguments(argtypes)
-    def encode_vrr_c(self, v1, v2, v3, mask1=0, mask2=0):
+    def encode_vrr_c(self, v1, v2, v3, mask4=0, mask5=0, mask6=0):
         self.writechar(opcode1)
         rbx =  (v1 >= 16) << 3
         rbx |= (v2 >= 16) << 2
@@ -497,11 +497,40 @@
         self.writechar(chr(byte))
         byte = (v3 & BIT_MASK_4) << 4
         self.writechar(chr(byte))
-        self.writechar(chr(mask2 & BIT_MASK_4))
-        self.writechar(chr((mask1 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
+        self.writechar(chr((mask6 & BIT_MASK_4) << 4 | (mask5 & BIT_MASK_4)))
+        self.writechar(chr((mask4 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
         self.writechar(opcode2)
     return encode_vrr_c
 
+def build_vrr_e(mnemonic, (opcode1,opcode2), argtypes='v,v,v,v,m,m'):
+    @builder.arguments(argtypes)
+    def encode_vrr_e(self, v1, v2, v3, v4, mask5=0, mask6=0):
+        self.writechar(opcode1)
+        rbx =  (v1 >= 16) << 3
+        rbx |= (v2 >= 16) << 2
+        rbx |= (v3 >= 16) << 1
+        rbx |= (v4 >= 16)
+        byte = (v1 & BIT_MASK_4) << 4 | (v2 & BIT_MASK_4)
+        self.writechar(chr(byte))
+        byte = (v3 & BIT_MASK_4) << 4 | (mask6 & BIT_MASK_4) << 4
+        self.writechar(chr(byte))
+        self.writechar(chr((mask5 & BIT_MASK_4)))
+        self.writechar(chr((v4 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
+        self.writechar(opcode2)
+    return encode_vrr_e
+
+def build_vri_a(mnemonic, (opcode1,opcode2), argtypes='v,i16,m'):
+    @builder.arguments(argtypes)
+    def encode_vri_a(self, v1, i2, mask3):
+        self.writechar(opcode1)
+        rbx =  (v1 >= 16) << 3
+        byte = (v1 & BIT_MASK_4) << 4
+        self.writechar(chr(byte))
+        self.write_i16(i2 & BIT_MASK_16)
+        self.writechar(chr((mask3 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
+        self.writechar(opcode2)
+    return encode_vri_a
+
 
 def build_unpack_func(mnemonic, func):
     @always_inline
@@ -555,13 +584,12 @@
             return arg
     unpack_arg._annspecialcase_ = 'specialize:arg(1)'
     argtypes = func._arguments_[:]
-    #while len(argtypes) > 0 and argtypes[-1] == '-':
-    #    argtypes.pop()
     at = argtypes[0] if len(argtypes) >= 1 else '-'
     bt = argtypes[1] if len(argtypes) >= 2 else '-'
     ct = argtypes[2] if len(argtypes) >= 3 else '-'
     dt = argtypes[3] if len(argtypes) >= 4 else '-'
     et = argtypes[4] if len(argtypes) >= 5 else '-'
+    ft = argtypes[5] if len(argtypes) >= 6 else '-'
     def function0(self):
         return func(self)
     def function1(self, a):
@@ -601,6 +629,14 @@
         i = unpack_arg(d, dt)
         j = unpack_arg(e, et)
         return func(self, f, g, h, i, j)
+    def function6(self, a, b, c, d, e, f):
+        g = unpack_arg(a, at)
+        h = unpack_arg(b, bt)
+        i = unpack_arg(c, ct)
+        j = unpack_arg(d, dt)
+        k = unpack_arg(e, et)
+        l = unpack_arg(f, ft)
+        return func(self, g, h, i, j, k, l)
     if len(argtypes) == 0:
         function = function0
     elif len(argtypes) == 1:
@@ -622,6 +658,8 @@
             function = function4_last_default
     elif len(argtypes) == 5:
         function = function5
+    elif len(argtypes) == 6:
+        function = function6
     else:
         assert 0, "implement function for argtypes %s" % (argtypes,)
     function.__name__ = mnemonic
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -295,7 +295,7 @@
 
 vector_mnemonic_codes = {
     'VL':     ('vrx', ['\xE7','\x06'], 'v,bid'),
-    'VLR':    ('vrr_a', ['\xE7','\x56']),
+    'VLR':    ('vrr_a', ['\xE7','\x56'], 'v,v'),
 
     'VST':    ('vrx', ['\xE7','\x0E'], 'v,bid'),
 
@@ -316,6 +316,16 @@
     'VFM':   ('vrr_c', ['\xE7','\xE7']),
     'VFD':   ('vrr_c', ['\xE7','\xE5']),
 
+    # conversion
+    'VCDG':  ('vrr_a', ['\xE7','\xC3']),
+
+    # compare, sign, ...
+    'VFPSO': ('vrr_a', ['\xE7','\xCC']),
+    'VFCE':  ('vrr_c', ['\xE7','\xE8']),
+    'VSEL':  ('vrr_e', ['\xE7','\x8D'], 'v,v,v,v'),
+    'VPERM': ('vrr_e', ['\xE7','\x8C'], 'v,v,v,v'),
+    'VREPI': ('vri_a', ['\xE7','\x45']),
+
     # '': ('', ['','']),
 }
 
diff --git a/rpython/jit/backend/zarch/locations.py b/rpython/jit/backend/zarch/locations.py
--- a/rpython/jit/backend/zarch/locations.py
+++ b/rpython/jit/backend/zarch/locations.py
@@ -25,6 +25,9 @@
     def is_fp_reg(self):
         return False
 
+    def is_vector_reg(self):
+        return False
+
     def is_imm_float(self):
         return False
 
@@ -118,13 +121,10 @@
     def is_core_reg(self):
         return False
 
-    def is_fp_reg(self):
-        return True
-
     def as_key(self):            # 16 <= as_key <= 32
         return self.value + 32
 
-    def is_float(self):
+    def is_vector_reg(self):
         return True
 
 class ImmLocation(AssemblerLocation):
diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py
--- a/rpython/jit/backend/zarch/regalloc.py
+++ b/rpython/jit/backend/zarch/regalloc.py
@@ -54,6 +54,12 @@
     def __repr__(self):
         return "<TempFloat at %s>" % (id(self),)
 
+class TempVector(TempVar):
+    type = 'V'
+
+    def __repr__(self):
+        return "<TempVector at %s>" % (id(self),)
+
 
 class FPRegisterManager(RegisterManager):
     all_regs              = r.MANAGED_FP_REGS
@@ -142,8 +148,7 @@
         return loc
 
     def get_scratch_reg(self, selected_reg=None):
-        # TODO
-        box = TempFloat()
+        box = TempVector()
         reg = self.force_allocate_reg(box, forbidden_vars=self.temp_boxes, selected_reg=selected_reg)
         self.temp_boxes.append(box)
         return reg
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -15,6 +15,7 @@
 import rpython.jit.backend.zarch.registers as r
 import rpython.jit.backend.zarch.conditions as c
 import rpython.jit.backend.zarch.locations as l
+import rpython.jit.backend.zarch.masks as m
 from rpython.jit.backend.zarch.locations import imm
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -27,42 +28,22 @@
         llop.debug_print(lltype.Void, msg)
     raise NotImplementedError(msg)
 
-def flush_vec_cc(asm, regalloc, condition, size, result_loc):
+def flush_vec_cc(asm, regalloc, condition, size, resultloc):
     # After emitting an instruction that leaves a boolean result in
-    # a condition code (cc), call this.  In the common case, result_loc
+    # a condition code (cc), call this.  In the common case, resultloc
     # will be set to SPP by the regalloc, which in this case means
     # "propagate it between this operation and the next guard by keeping
-    # it in the cc".  In the uncommon case, result_loc is another
+    # it in the cc".  In the uncommon case, resultloc is another
     # register, and we emit a load from the cc into this register.
 
-    # Possibly invert the bit in the CR
-    #bit, invert = c.encoding[condition]
-    #assert 24 <= bit <= 27
-    #if invert == 12:
-    #    pass
-    #elif invert == 4:
-    #    asm.mc.crnor(bit, bit, bit)
-    #else:
-    #    assert 0
-    #assert asm.guard_success_cc == c.cond_none
-    ##
-    #if result_loc is r.SPP:
-    #    asm.guard_success_cc = condition
-    #else:
-    #    resval = result_loc.value
-    #    # either doubleword integer 1 (2x) or word integer 1 (4x)
-    #    ones = regalloc.vrm.get_scratch_reg(type=INT).value
-    #    zeros = regalloc.vrm.get_scratch_reg(type=INT).value
-    #    asm.mc.vxor(zeros, zeros, zeros)
-    #    if size == 4:
-    #        asm.mc.vspltisw(ones, 1)
-    #    else:
-    #        assert size == 8
-    #        tloc = regalloc.rm.get_scratch_reg()
-    #        asm.mc.load_imm(tloc, asm.VEC_DOUBLE_WORD_ONES)
-    #        asm.mc.lvx(ones, 0, tloc.value)
-    #    asm.mc.vsel(resval, zeros, ones, resval)
-    pass
+    if resultloc is r.SPP:
+        asm.guard_success_cc = condition
+    else:
+        ones = regalloc.vrm.get_scratch_reg()
+        zeros = regalloc.vrm.get_scratch_reg()
+        asm.mc.VX(zeros, zeros, zeros)
+        asm.mc.VREPI(ones, l.imm(1), l.itemsize_to_mask(size))
+        asm.mc.VSEL(resultloc, ones, zeros, resultloc)
 
 class ZSIMDVectorExt(VectorExt):
     def setup_once(self, asm):
@@ -119,7 +100,7 @@
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
         if itemsize == 8:
-            self.mc.VFA(resloc, loc0, loc1, 3, 0)
+            self.mc.VFA(resloc, loc0, loc1, 3, 0, 0)
             return
         not_implemented("vec_float_add of size %d" % itemsize)
 
@@ -127,7 +108,7 @@
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
         if itemsize == 8:
-            self.mc.VFS(resloc, loc0, loc1, 3, 0)
+            self.mc.VFS(resloc, loc0, loc1, 3, 0, 0)
             return
         not_implemented("vec_float_sub of size %d" % itemsize)
 
@@ -135,7 +116,7 @@
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
         if itemsize == 8:
-            self.mc.VFM(resloc, loc0, loc1, 3, 0)
+            self.mc.VFM(resloc, loc0, loc1, 3, 0, 0)
             return
         not_implemented("vec_float_mul of size %d" % itemsize)
 
@@ -143,7 +124,7 @@
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
         if itemsize == 8:
-            self.mc.VFD(resloc, loc0, loc1, 3, 0)
+            self.mc.VFD(resloc, loc0, loc1, 3, 0, 0)
             return
         not_implemented("vec_float_truediv of size %d" % itemsize)
 
@@ -168,22 +149,18 @@
     def emit_vec_float_abs(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
         size = sizeloc.value
-        if size == 4:
-            self.mc.xvabssp(resloc.value, argloc.value)
-        elif size == 8:
-            self.mc.xvabsdp(resloc.value, argloc.value)
-        else:
-            not_implemented("float abs for size %d" % size)
+        if size == 8:
+            self.mc.VFPSO(resloc, argloc, 3, 0, 2)
+            return
+        not_implemented("vec_float_abs of size %d" % itemsize)
 
     def emit_vec_float_neg(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
         size = sizeloc.value
-        if size == 4:
-            self.mc.xvnegsp(resloc.value, argloc.value)
-        elif size == 8:
-            self.mc.xvnegdp(resloc.value, argloc.value)
-        else:
-            not_implemented("float neg for size %d" % size)
+        if size == 8:
+            self.mc.VFPSO(resloc, argloc, 3, 0, 0)
+            return
+        not_implemented("vec_float_abs of size %d" % itemsize)
 
     def emit_vec_guard_true(self, guard_op, arglocs, regalloc):
         self._emit_guard(guard_op, arglocs)
@@ -212,42 +189,9 @@
             self._accum_reduce(op, scalar_arg, vector_loc, scalar_loc)
             accum_info = accum_info.next()
 
-    def _accum_reduce(self, op, arg, accumloc, targetloc):
-        # Currently the accumulator can ONLY be the biggest
-        # 64 bit float/int
-        tgt = targetloc.value
-        acc = accumloc.value
-        if arg.type == FLOAT:
-            # r = (r[0]+r[1],r[0]+r[1])
-            if IS_BIG_ENDIAN:
-                self.mc.xxpermdi(tgt, acc, acc, 0b00)
-            else:
-                self.mc.xxpermdi(tgt, acc, acc, 0b10)
-            if op == '+':
-                self.mc.xsadddp(tgt, tgt, acc)
-            elif op == '*':
-                self.mc.xsmuldp(tgt, tgt, acc)
-            else:
-                not_implemented("sum not implemented")
-            return
-        else:
-            assert arg.type == INT
-            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
-            self.mc.stvx(acc, r.SCRATCH2.value, r.SP.value)
-            self.mc.load(tgt, r.SP.value, PARAM_SAVE_AREA_OFFSET)
-            self.mc.load(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
-            if op == '+':
-                self.mc.add(tgt, tgt, acc)
-            elif op == '*':
-                self.mc.mulld(tgt, tgt, acc)
-            else:
-                not_implemented("sum not implemented")
-            return
-
-        not_implemented("reduce sum for %s not impl." % arg)
-
     def emit_vec_int_is_true(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
+        # TODO
         resloc, argloc, sizeloc = arglocs
         size = sizeloc.value
         tmp = regalloc.vrm.get_scratch_reg(type=INT).value
@@ -266,22 +210,13 @@
 
     def emit_vec_float_eq(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        resloc, loc1, loc2, sizeloc = arglocs
+        resloc, loc0, loc1, sizeloc = arglocs
         size = sizeloc.value
-        tmp = regalloc.vrm.get_scratch_reg().value
-        offloc = regalloc.rm.get_scratch_reg()
-        off = offloc.value
-        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
-        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
-        if size == 4:
-            self.mc.xvcmpeqspx(tmp, loc1.value, loc2.value)
-            self.mc.stxvw4x(tmp, off, r.SP.value)
-        elif size == 8:
-            self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value)
-            self.mc.stxvd2x(tmp, off, r.SP.value)
+        if size == 8:
+            # bit 3 in last argument sets the condition code
+            self.mc.VFCE(resloc, loc0, loc1, 3, 0, 1)
         else:
             not_implemented("[zarch/assembler] float == for size %d" % size)
-        self.mc.lvx(resloc.value, off, r.SP.value)
         flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
 
     def emit_vec_float_xor(self, op, arglocs, regalloc):
@@ -314,14 +249,16 @@
         flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc)
 
     def emit_vec_cast_int_to_float(self, op, arglocs, regalloc):
-        res, l0 = arglocs
+        resloc, loc0 = arglocs
         offloc = regalloc.rm.get_scratch_reg()
         off = offloc.value
         # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
-        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
-        self.mc.stvx(l0.value, off, r.SP.value)
-        self.mc.lxvd2x(res.value, off, r.SP.value)
-        self.mc.xvcvsxddp(res.value, res.value)
+        # bit 1 on mask4 -> supresses inexact exception
+        self.mc.VCDG(resloc, loc0, 3, 4, m.RND_TOZERO.value)
+        #self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
+        #self.mc.stvx(l0.value, off, r.SP.value)
+        #self.mc.lxvd2x(res.value, off, r.SP.value)
+        #self.mc.xvcvsxddp(res.value, res.value)
 
     def emit_vec_int_eq(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
@@ -354,6 +291,15 @@
         self.mc.vnor(res.value, res.value, res.value)
         flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res)
 
+    def emit_vec_cast_float_to_int(self, op, arglocs, regalloc):
+        res, l0 = arglocs
+        offloc = regalloc.rm.get_scratch_reg()
+        v0 = regalloc.vrm.get_scratch_reg(type=INT)
+        off = offloc.value
+        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
+        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
+        self.mc.xvcvdpsxds(res.value, l0.value)
+
     def emit_vec_expand_f(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, srcloc = arglocs
@@ -540,14 +486,40 @@
             return
         not_implemented("unpack for combination src %d -> res %d" % (srcidx, residx))
 
-    def emit_vec_cast_float_to_int(self, op, arglocs, regalloc):
-        res, l0 = arglocs
-        offloc = regalloc.rm.get_scratch_reg()
-        v0 = regalloc.vrm.get_scratch_reg(type=INT)
-        off = offloc.value
-        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
-        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
-        self.mc.xvcvdpsxds(res.value, l0.value)
+    def _accum_reduce(self, op, arg, accumloc, targetloc):
+        # Currently the accumulator can ONLY be the biggest
+        # 64 bit float/int
+        # TODO
+        tgt = targetloc.value
+        acc = accumloc.value
+        if arg.type == FLOAT:
+            # r = (r[0]+r[1],r[0]+r[1])
+            if IS_BIG_ENDIAN:
+                self.mc.xxpermdi(tgt, acc, acc, 0b00)
+            else:
+                self.mc.xxpermdi(tgt, acc, acc, 0b10)
+            if op == '+':
+                self.mc.xsadddp(tgt, tgt, acc)
+            elif op == '*':
+                self.mc.xsmuldp(tgt, tgt, acc)
+            else:
+                not_implemented("sum not implemented")
+            return
+        else:
+            assert arg.type == INT
+            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
+            self.mc.stvx(acc, r.SCRATCH2.value, r.SP.value)
+            self.mc.load(tgt, r.SP.value, PARAM_SAVE_AREA_OFFSET)
+            self.mc.load(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
+            if op == '+':
+                self.mc.add(tgt, tgt, acc)
+            elif op == '*':
+                self.mc.mulld(tgt, tgt, acc)
+            else:
+                not_implemented("sum not implemented")
+            return
+
+        not_implemented("reduce sum for %s not impl." % arg)
 
     def emit_vec_f(self, op, arglocs, regalloc):
         pass
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -133,9 +133,9 @@
 
     vec_float_unary = functools.partial(vec_int_unary, _vector_float_unary)
 
-    test_vec_abs_float = \
+    test_vec_float_abs = \
             vec_float_unary(lambda v: abs(v), rffi.DOUBLE)
-    test_vec_neg_float = \
+    test_vec_float_neg = \
             vec_float_unary(lambda v: -v, rffi.DOUBLE)
 
     # FLOAT BINARY

From pypy.commits at gmail.com  Wed Sep 14 11:45:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 08:45:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Pff,
 took me too long to figure out this was causing an obscure error at
Message-ID: <57d970ac.05d71c0a.9b532.93ac@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87108:0419bc15bcf5
Date: 2016-09-14 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/0419bc15bcf5/

Log:	Pff, took me too long to figure out this was causing an obscure
	error at start-up of pypy-c

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -105,8 +105,9 @@
     bytes_warning = structseqfield(9)
     quiet = structseqfield(10)
     hash_randomization = structseqfield(11)
+    isolated = structseqfield(12)
 
-null_sysflags = sysflags((0,)*12)
+null_sysflags = sysflags((0,)*13)
 null__xoptions = {}
 
 

From pypy.commits at gmail.com  Wed Sep 14 11:45:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 08:45:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Skip the micronumpy tests on py3.5.
 Micronumpy is anyway not compiled
Message-ID: <57d970ae.a970c20a.1ed82.9337@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87109:b093c8afa8ce
Date: 2016-09-14 17:45 +0200
http://bitbucket.org/pypy/pypy/changeset/b093c8afa8ce/

Log:	Skip the micronumpy tests on py3.5. Micronumpy is anyway not
	compiled into a default pypy-c in this branch.

diff --git a/pypy/module/micronumpy/test/conftest.py b/pypy/module/micronumpy/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/conftest.py
@@ -0,0 +1,5 @@
+import py
+
+def pytest_collect_directory(path, parent):
+    py.test.skip("micronumpy tests skipped for now on py3.5")
+pytest_collect_file = pytest_collect_directory

From pypy.commits at gmail.com  Wed Sep 14 11:47:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 08:47:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Write down a message to avoid being hit
 by 0419bc15bcf5 again
Message-ID: <57d9711a.812e1c0a.50ea0.9aec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87110:d72441d11e7e
Date: 2016-09-14 16:56 +0100
http://bitbucket.org/pypy/pypy/changeset/d72441d11e7e/

Log:	Write down a message to avoid being hit by 0419bc15bcf5 again

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -336,7 +336,6 @@
     return stream
 
 
-# Order is significant!
 sys_flags = (
     "debug",
     "inspect",
@@ -352,6 +351,7 @@
     "hash_randomization",
     "isolated",
 )
+# ^^^ Order is significant!  Keep in sync with module.sys.app.sysflags
 
 default_options = dict.fromkeys(
     sys_flags +

From pypy.commits at gmail.com  Wed Sep 14 12:27:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 09:27:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: First test: space.next() must
 not accept coroutines (and not by *name*
Message-ID: <57d97a6c.87941c0a.ca05b.a065@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87112:4f2a6936d18e
Date: 2016-09-14 18:26 +0200
http://bitbucket.org/pypy/pypy/changeset/4f2a6936d18e/

Log:	First test: space.next() must not accept coroutines (and not by
	*name* anyway!)

diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -0,0 +1,10 @@
+
+class AppTestCoroutine:
+
+    def test_cannot_iterate(self): """
+        async def f(x):
+            pass
+        raises(TypeError, "for i in f(5): pass")
+        raises(TypeError, iter, f(5))
+        raises(TypeError, next, f(5))
+        """
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -298,10 +298,7 @@
         return w_iter
 
     def next(space, w_obj):
-        if space.type(w_obj).name == 'coroutine':
-            w_descr = space.lookup(w_obj, '__anext__')
-        else:
-            w_descr = space.lookup(w_obj, '__next__')
+        w_descr = space.lookup(w_obj, '__next__')
         if w_descr is None:
             raise oefmt(space.w_TypeError,
                         "'%T' object is not an iterator", w_obj)

From pypy.commits at gmail.com  Wed Sep 14 12:27:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 09:27:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: A branch to add the
 coroutine_wrapper type from CPython
Message-ID: <57d97a6b.11331c0a.1b838.a1e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87111:0df496486e9c
Date: 2016-09-14 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/0df496486e9c/

Log:	A branch to add the coroutine_wrapper type from CPython and review
	our coroutines


From pypy.commits at gmail.com  Wed Sep 14 13:36:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 10:36:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: in-progress
Message-ID: <57d98a97.c186c20a.3163.ba04@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87113:a1f0e3bcdc6f
Date: 2016-09-14 19:35 +0200
http://bitbucket.org/pypy/pypy/changeset/a1f0e3bcdc6f/

Log:	in-progress

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -7,38 +7,41 @@
 
 
 class GeneratorOrCoroutine(W_Root):
-    """XXX: move the common functionality here!"""
-
-    def descr_close(self):
-        raise NotImplementedError
-
-
-class GeneratorIterator(GeneratorOrCoroutine):
-    "An iterator created by a generator."
     _immutable_fields_ = ['pycode']
 
-    def __init__(self, frame):
+    def __init__(self, frame, name=None, qualname=None):
         self.space = frame.space
         self.frame = frame     # turned into None when frame_finished_execution
-        self.pycode = frame.pycode
+        self.pycode = frame.pycode  # usually never None but see __setstate__
         self.running = False
+        self._name = name           # may be null, use get_name()
+        self._qualname = qualname   # may be null, use get_qualname()
         if self.pycode.co_flags & CO_YIELD_INSIDE_TRY:
             self.register_finalizer(self.space)
 
+    def get_name(self):
+        if self._name is not None:
+            return self._name
+        elif self.pycode is None:
+            return "<finished>"
+        else:
+            return self.pycode.co_name
+
+    def get_qualname(self):
+        if self._qualname is not None:
+            return self._qualname
+        return self.get_name()
+
     def descr__repr__(self, space):
-        if self.pycode is None:
-            code_name = '<finished>'
-        else:
-            code_name = self.pycode.co_name
         addrstring = self.getaddrstring(space)
-        return space.wrap("<generator object %s at 0x%s>" %
-                          (code_name, addrstring))
+        return space.wrap("<%s object %s at 0x%s>" %
+                          (self.KIND, self.get_qualname(), addrstring))
 
     def descr__reduce__(self, space):
         from pypy.interpreter.mixedmodule import MixedModule
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
-        new_inst = mod.get('generator_new')
+        new_inst = mod.get(self.KIND + '_new')
         w        = space.wrap
         if self.frame:
             w_frame = self.frame._reduce_state(space)
@@ -64,15 +67,16 @@
         else:
             frame = instantiate(space.FrameClass)   # XXX fish
             frame.descr__setstate__(space, w_framestate)
-            GeneratorIterator.__init__(self, frame)
+            if isinstance(self, GeneratorIterator):
+                GeneratorIterator.__init__(self, frame)
+            elif isinstance(self, Coroutine):
+                Coroutine.__init__(self, frame)
+            else:
+                assert False
         self.running = self.space.is_true(w_running)
 
-    def descr__iter__(self):
-        """x.__iter__() <==> iter(x)"""
-        return self.space.wrap(self)
-
     def descr_send(self, w_arg=None):
-        """send(arg) -> send 'arg' into generator,
+        """send(arg) -> send 'arg' into generator/coroutine,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
 
@@ -87,9 +91,12 @@
     def _send_ex(self, w_arg, operr):
         space = self.space
         if self.running:
-            raise oefmt(space.w_ValueError, "generator already executing")
+            raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
         frame = self.frame
         if frame is None:
+            if isinstance(self, Coroutine):
+                raise oefmt(space.w_RuntimeError,
+                            "cannot reuse already awaited coroutine")
             # xxx a bit ad-hoc, but we don't want to go inside
             # execute_frame() if the frame is actually finished
             if operr is None:
@@ -100,8 +107,8 @@
         if last_instr == -1:
             if w_arg and not space.is_w(w_arg, space.w_None):
                 raise oefmt(space.w_TypeError,
-                            "can't send non-None value to a just-started "
-                            "generator")
+                            "can't send non-None value to a just-started %s",
+                            self.KIND)
         else:
             if not w_arg:
                 w_arg = space.w_None
@@ -129,7 +136,7 @@
             self.running = False
 
     def descr_throw(self, w_type, w_val=None, w_tb=None):
-        """x.throw(typ[,val[,tb]]) -> raise exception in generator,
+        """throw(typ[,val[,tb]]) -> raise exception in generator/coroutine,
 return next yielded value or raise StopIteration."""
         if w_val is None:
             w_val = self.space.w_None
@@ -219,16 +226,14 @@
                 operr.set_traceback(tb)
         return self.send_ex(space.w_None, operr)
 
-    def descr_next(self):
-        """x.__next__() <==> next(x)"""
-        return self.send_ex(self.space.w_None)
-
     def descr_close(self):
-        """x.close(arg) -> raise GeneratorExit inside generator."""
+        """close() -> raise GeneratorExit inside generator/coroutine."""
+        _PyGen_yf()....
+        if self.frame is None:
+            return     # nothing to do in this case
         space = self.space
         try:
-            w_retval = self.throw(space.w_GeneratorExit, space.w_None,
-                                  space.w_None)
+            w_retval = self.send_ex..?..(space.w_GeneratorExit..)
         except OperationError as e:
             if e.match(space, space.w_StopIteration) or \
                     e.match(space, space.w_GeneratorExit):
@@ -237,23 +242,46 @@
 
         if w_retval is not None:
             raise oefmt(space.w_RuntimeError,
-                        "generator ignored GeneratorExit")
+                        "%s ignored GeneratorExit", self.KIND)
 
-    def descr_gi_frame(self, space):
+    def descr_gicr_frame(self, space):
         if self.frame is not None and not self.frame.frame_finished_execution:
             return self.frame
         else:
             return space.w_None
 
-    def descr_gi_code(self, space):
-        return self.pycode
+    def descr__name__(self, space):
+        return space.wrap(self.get_name())
 
-    def descr__name__(self, space):
-        if self.pycode is None:
-            code_name = '<finished>'
-        else:
-            code_name = self.pycode.co_name
-        return space.wrap(code_name)
+    def descr__qualname__(self, space):
+        return space.wrap(self.get_qualname())
+
+    def _finalize_(self):
+        # This is only called if the CO_YIELD_INSIDE_TRY flag is set
+        # on the code object.  If the frame is still not finished and
+        # finally or except blocks are present at the current
+        # position, then raise a GeneratorExit.  Otherwise, there is
+        # no point.
+        if self.frame is not None:
+            block = self.frame.lastblock
+            while block is not None:
+                if not isinstance(block, LoopBlock):
+                    self.descr_close()
+                    break
+                block = block.previous
+
+
+class GeneratorIterator(GeneratorOrCoroutine):
+    "An iterator created by a generator."
+    KIND = "generator"
+
+    def descr__iter__(self):
+        """x.__iter__() <==> iter(x)"""
+        return self.space.wrap(self)
+
+    def descr_next(self):
+        """x.__next__() <==> next(x)"""
+        return self.send_ex(self.space.w_None)
 
     # Results can be either an RPython list of W_Root, or it can be an
     # app-level W_ListObject, which also has an append() method, that's why we
@@ -297,20 +325,6 @@
     unpack_into = _create_unpack_into()
     unpack_into_w = _create_unpack_into()
 
-    def _finalize_(self):
-        # This is only called if the CO_YIELD_INSIDE_TRY flag is set
-        # on the code object.  If the frame is still not finished and
-        # finally or except blocks are present at the current
-        # position, then raise a GeneratorExit.  Otherwise, there is
-        # no point.
-        if self.frame is not None:
-            block = self.frame.lastblock
-            while block is not None:
-                if not isinstance(block, LoopBlock):
-                    self.descr_close()
-                    break
-                block = block.previous
-    
     def _GetAwaitableIter(self, space):
         #check if generator is a coroutine
         if self.pycode.co_flags & consts.CO_ITERABLE_COROUTINE:
@@ -333,267 +347,18 @@
                         "of type '%T'", res)
         return res
 
-def get_printable_coroutine_location_genentry(bytecode):
-    return '%s <coroutine>' % (bytecode.get_repr(),)
-coroutineentry_driver = jit.JitDriver(greens=['pycode'],
-                                      reds=['gen', 'w_arg', 'operr'],
-                                      get_printable_location = get_printable_coroutine_location_genentry,
-                                      name='coroutineentry')
 
 class Coroutine(GeneratorOrCoroutine):
     "A coroutine object."
-    _immutable_fields_ = ['pycode']
-
-    def __init__(self, frame):
-        self.space = frame.space
-        self.frame = frame     # turned into None when frame_finished_execution
-        self.pycode = frame.pycode
-        self.running = False
-        if self.pycode.co_flags & CO_YIELD_INSIDE_TRY:
-            self.register_finalizer(self.space)
+    KIND = "coroutine"
 
     def descr__await__(self, space):
         # implement this function:
         # https://github.com/python/cpython/blob/3.5/Objects/genobject.c#L786
         # you need a new CoroutineWrapper object + CoroutineWrapperType
         return self
-    
-    def descr__reduce__(self, space):
-        from pypy.interpreter.mixedmodule import MixedModule
-        w_mod    = space.getbuiltinmodule('_pickle_support')
-        mod      = space.interp_w(MixedModule, w_mod)
-        new_inst = mod.get('coroutine_new')
-        w        = space.wrap
-        if self.frame:
-            w_frame = self.frame._reduce_state(space)
-        else:
-            w_frame = space.w_None
 
-        tup = [
-            w_frame,
-            w(self.running),
-            ]
-
-        return space.newtuple([new_inst, space.newtuple([]),
-                               space.newtuple(tup)])
-
-    def descr__setstate__(self, space, w_args):
-        from rpython.rlib.objectmodel import instantiate
-        args_w = space.unpackiterable(w_args)
-        w_framestate, w_running = args_w
-        if space.is_w(w_framestate, space.w_None):
-            self.frame = None
-            self.space = space
-            self.pycode = None
-        else:
-            frame = instantiate(space.FrameClass)   # XXX fish
-            frame.descr__setstate__(space, w_framestate)
-            Coroutine.__init__(self, frame)
-        self.running = self.space.is_true(w_running)
-
-    def descr__iter__(self):
-        """x.__iter__() <==> iter(x)"""
-        return self.space.wrap(self)
-
-    def descr_send(self, w_arg=None):
-        """send(arg) -> send 'arg' into coroutine,
-return next yielded value or raise StopIteration."""
-        return self.send_ex(w_arg)
-
-    def descr__repr__(self, space):
-        if self.pycode is None:
-            code_name = '<finished>'
-        else:
-            code_name = self.pycode.co_name
-        addrstring = self.getaddrstring(space)
-        return space.wrap("<coroutine object %s at 0x%s>" %
-                          (code_name, addrstring))
-    
-    def descr_next(self):
-        """x.__next__() <==> next(x)"""
-        return self.send_ex(self.space.w_None)
-    
-    def descr_close(self):
-        """x.close(arg) -> raise GeneratorExit inside coroutine."""
-        space = self.space
-        try:
-            w_retval = self.throw(space.w_GeneratorExit, space.w_None,
-                                  space.w_None)
-        except OperationError as e:
-            if e.match(space, space.w_StopIteration) or \
-                    e.match(space, space.w_GeneratorExit):
-                return space.w_None
-            raise
-
-        if w_retval is not None:
-            raise oefmt(space.w_RuntimeError,
-                        "coroutine ignored GeneratorExit")
-            
-    def descr_throw(self, w_type, w_val=None, w_tb=None):
-        """x.throw(typ[,val[,tb]]) -> raise exception in coroutine,
-return next iterated value or raise StopIteration."""
-        if w_val is None:
-            w_val = self.space.w_None
-        return self.throw(w_type, w_val, w_tb)
-    
-    def _get_yield_from(self):
-        # Probably a hack (but CPython has the same):
-        # If the current frame is stopped in a "yield from",
-        # return the paused generator.
-        if not self.frame:
-            return None
-        co_code = self.frame.pycode.co_code
-        opcode = ord(co_code[self.frame.last_instr + 1])
-        if opcode == YIELD_FROM:
-            return self.frame.peekvalue()
-    
-    def throw(self, w_type, w_val, w_tb):
-        space = self.space
-
-        w_yf = self._get_yield_from()
-        if w_yf is not None:
-            # Paused in a "yield from", pass the throw to the inner generator.
-            return self._throw_delegate(space, w_yf, w_type, w_val, w_tb)
-        else:
-            # Not paused in a "yield from", quit this generator
-            return self._throw_here(space, w_type, w_val, w_tb)
-        
-    def _throw_delegate(self, space, w_yf, w_type, w_val, w_tb):
-        if space.is_w(w_type, space.w_GeneratorExit):
-            try:
-                w_close = space.getattr(w_yf, space.wrap("close"))
-            except OperationError as e:
-                if not e.match(space, space.w_AttributeError):
-                    e.write_unraisable(space, "generator.close()")
-            else:
-                self.running = True
-                try:
-                    space.call_function(w_close)
-                except OperationError as operr:
-                    self.running = False
-                    return self.send_ex(space.w_None, operr)
-                finally:
-                    self.running = False
-            return self._throw_here(space, w_type, w_val, w_tb)
-        else:
-            try:
-                w_throw = space.getattr(w_yf, space.wrap("throw"))
-            except OperationError as e:
-                if not e.match(space, space.w_AttributeError):
-                    raise
-                return self._throw_here(space, w_type, w_val, w_tb)
-            self.running = True
-            try:
-                return space.call_function(w_throw, w_type, w_val, w_tb)
-            except OperationError as operr:
-                self.running = False
-                # Pop subiterator from stack.
-                w_subiter = self.frame.popvalue()
-                assert space.is_w(w_subiter, w_yf)
-                # Termination repetition of YIELD_FROM
-                self.frame.last_instr += 1
-                if operr.match(space, space.w_StopIteration):
-                    operr.normalize_exception(space)
-                    w_val = space.getattr(operr.get_w_value(space),
-                                          space.wrap("value"))
-                    return self.send_ex(w_val)
-                else:
-                    return self.send_ex(space.w_None, operr)
-            finally:
-                self.running = False
-
-    def _throw_here(self, space, w_type, w_val, w_tb):
-        from pypy.interpreter.pytraceback import check_traceback
-
-        msg = "throw() third argument must be a traceback object"
-        if space.is_none(w_tb):
-            tb = None
-        else:
-            tb = check_traceback(space, w_tb, msg)
-
-        operr = OperationError(w_type, w_val, tb)
-        operr.normalize_exception(space)
-        if tb is None:
-            tb = space.getattr(operr.get_w_value(space),
-                               space.wrap('__traceback__'))
-            if not space.is_w(tb, space.w_None):
-                operr.set_traceback(tb)
-        return self.send_ex(space.w_None, operr)
-    
-    def send_ex(self, w_arg, operr=None):
-        pycode = self.pycode
-        if pycode is not None:
-            if jit.we_are_jitted() and should_not_inline(pycode):
-                coroutineentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                      operr=operr, pycode=pycode)
-        return self._send_ex(w_arg, operr)
-
-    def _send_ex(self, w_arg, operr):
-        space = self.space
-        if self.running:
-            raise oefmt(space.w_ValueError, "coroutine already executing")
-        frame = self.frame
-        if frame is None:
-            # xxx a bit ad-hoc, but we don't want to go inside
-            # execute_frame() if the frame is actually finished
-            if operr is None:
-                operr = OperationError(space.w_StopIteration, space.w_None)
-            raise operr
-
-        last_instr = jit.promote(frame.last_instr)
-        if last_instr == -1:
-            if w_arg and not space.is_w(w_arg, space.w_None):
-                raise oefmt(space.w_TypeError,
-                            "can't send non-None value to a just-started "
-                            "coroutine")
-        else:
-            if not w_arg:
-                w_arg = space.w_None
-        self.running = True
-        try:
-            try:
-                w_result = frame.execute_frame(w_arg, operr)
-            except OperationError:
-                # errors finish a frame
-                self.frame = None
-                raise
-            # if the frame is now marked as finished, it was RETURNed from
-            if frame.frame_finished_execution:
-                self.frame = None
-                if space.is_none(w_result):
-                    # Delay exception instantiation if we can
-                    raise OperationError(space.w_StopIteration, space.w_None)
-                else:
-                    raise OperationError(space.w_StopIteration,
-                                         space.newtuple([w_result]))
-            else:
-                return w_result     # YIELDed
-        finally:
-            frame.f_backref = jit.vref_None
-            self.running = False
-        
-    def descr_gi_frame(self, space):
-        if self.frame is not None and not self.frame.frame_finished_execution:
-            return self.frame
-        else:
-            return space.w_None
-
-    def descr_gi_code(self, space):
-        return self.pycode
-
-    def descr__name__(self, space):
-        if self.pycode is None:
-            code_name = '<finished>'
-        else:
-            code_name = self.pycode.co_name
-        return space.wrap(code_name)
-    
     def _finalize_(self):
-        # This is only called if the CO_YIELD_INSIDE_TRY flag is set
-        # on the code object.  If the frame is still not finished and
-        # finally or except blocks are present at the current
-        # position, then raise a GeneratorExit.  Otherwise, there is
-        # no point.
         # If coroutine was never awaited on issue a RuntimeWarning.
         if self.pycode is not None and \
            self.frame is not None and \
@@ -601,18 +366,11 @@
             # XXX PyErr_Occured in condition?
             raise oefmt(self.space.w_RuntimeWarning,
                         "coroutine '%s' was never awaited",
-                        self.pycode.co_name)
-        if self.frame is not None:
-            block = self.frame.lastblock
-            while block is not None:
-                if not isinstance(block, LoopBlock):
-                    self.descr_close()
-                    break
-                block = block.previous
-    
+                        self.get_qualname())
+        GeneratorOrCoroutine._finalize_(self)
+
     def _GetAwaitableIter(self, space):
         return self
-        
 
 
 def get_printable_location_genentry(bytecode):
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -793,22 +793,25 @@
     __iter__   = interp2app(GeneratorIterator.descr__iter__,
                             descrmismatch='__iter__'),
     gi_running = interp_attrproperty('running', cls=GeneratorIterator),
-    gi_frame   = GetSetProperty(GeneratorIterator.descr_gi_frame),
-    gi_code    = GetSetProperty(GeneratorIterator.descr_gi_code),
+    gi_frame   = GetSetProperty(GeneratorIterator.descr_gicr_frame),
+    gi_code    = interp_attrproperty_w('pycode', cls=GeneratorIterator),
+    gi_yieldfrom = XXX,
     __name__   = GetSetProperty(GeneratorIterator.descr__name__),
+    __qualname__ = GetSetProperty(GeneratorIterator.descr__qualname__),
     __weakref__ = make_weakref_descr(GeneratorIterator),
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
 
-# TODO: to have the same distinction (Coroutine | Iterator) as in cpython 3.5,
-# a wrapper typedef with __anext__ has to be created, and __anext__ has to be
-# removed in coroutine
+CoroutineWrapper.typedef = TypeDef("coroutine_wrapper",
+    __anext__   = interp2app(CoroutineWrapper.descr_next,
+                            descrmismatch='__anext__'),
+)
+assert not CoroutineWrapper.typedef.acceptable_as_base_class  # no __new__
+
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
     __reduce__   = interp2app(Coroutine.descr__reduce__),
     __setstate__ = interp2app(Coroutine.descr__setstate__),
-    __anext__   = interp2app(Coroutine.descr_next,
-                            descrmismatch='__anext__'),
     send       = interp2app(Coroutine.descr_send,
                             descrmismatch='send'),
     throw      = interp2app(Coroutine.descr_throw,
@@ -817,10 +820,12 @@
                             descrmismatch='close'),
     __await__  = interp2app(Coroutine.descr__await__,
                             descrmismatch='__await__'),
-    gi_running = interp_attrproperty('running', cls=Coroutine),
-    gi_frame   = GetSetProperty(Coroutine.descr_gi_frame),
-    gi_code    = GetSetProperty(Coroutine.descr_gi_code),
+    cr_running = interp_attrproperty('running', cls=Coroutine),
+    cr_frame   = GetSetProperty(Coroutine.descr_gicr_frame),
+    cr_code    = interp_attrproperty_w('pycode', cls=Coroutine),
+    cr_await   = XXX,
     __name__   = GetSetProperty(Coroutine.descr__name__),
+    __qualname__ = GetSetProperty(Coroutine.descr__qualname__),
     __weakref__ = make_weakref_descr(Coroutine),
 )
 assert not Coroutine.typedef.acceptable_as_base_class  # no __new__

From pypy.commits at gmail.com  Wed Sep 14 13:58:38 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 14 Sep 2016 10:58:38 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: more tests,
 fixes for buffer interface on app-level classes
Message-ID: <57d98fce.861b1c0a.fd47c.21e3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87114:0497d086d9c5
Date: 2016-09-14 20:24 +0300
http://bitbucket.org/pypy/pypy/changeset/0497d086d9c5/

Log:	more tests, fixes for buffer interface on app-level classes

diff --git a/pypy/module/cpyext/test/test_abstract.py b/pypy/module/cpyext/test/test_abstract.py
--- a/pypy/module/cpyext/test/test_abstract.py
+++ b/pypy/module/cpyext/test/test_abstract.py
@@ -104,3 +104,17 @@
         assert raises(TypeError, buffer_support.readbuffer_as_string, 42)
         assert raises(TypeError, buffer_support.writebuffer_as_string, 42)
         assert raises(TypeError, buffer_support.charbuffer_as_string, 42)
+
+    def test_user_class(self):
+        class MyBuf(str):
+            pass
+        s = 'a\0x'
+        buf = MyBuf(s)
+        buffer_support = self.get_buffer_support()
+
+        assert buffer_support.check_readbuffer(buf)
+        assert s == buffer_support.readbuffer_as_string(buf)
+        assert raises(TypeError, buffer_support.writebuffer_as_string, buf)
+        assert s == buffer_support.charbuffer_as_string(buf)
+
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -514,10 +514,13 @@
 @cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
              header=None, error=-1)
 def bf_getreadbuffer(space, w_buf, segment, ref):
+    from rpython.rlib.buffer import StringBuffer
     if segment != 0:
         raise oefmt(space.w_SystemError,
                     "accessing non-existent segment")
     buf = space.readbuf_w(w_buf)
+    if isinstance(buf, StringBuffer):
+        return str_getreadbuffer(space, w_buf, segment, ref)
     address = buf.get_raw_address()
     ref[0] = address
     return len(buf)
@@ -533,7 +536,6 @@
     if segment != 0:
         raise oefmt(space.w_SystemError,
                     "accessing non-existent segment")
-
     buf = space.writebuf_w(w_buf)
     ref[0] = buf.get_raw_address()
     return len(buf)

From pypy.commits at gmail.com  Wed Sep 14 14:12:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 11:12:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: CO_FUTURE_GENERATOR_STOP,
 not tested so far
Message-ID: <57d992f4.02d31c0a.8a8d.2085@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87115:209720711d97
Date: 2016-09-14 20:11 +0200
http://bitbucket.org/pypy/pypy/changeset/209720711d97/

Log:	CO_FUTURE_GENERATOR_STOP, not tested so far

diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -25,7 +25,8 @@
 
 PyCF_MASK = (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT |
              CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION |
-             CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL)
+             CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL |
+             CO_FUTURE_GENERATOR_STOP)
 PyCF_SOURCE_IS_UTF8 = 0x0100
 PyCF_DONT_IMPLY_DEDENT = 0x0200
 PyCF_ONLY_AST = 0x0400
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -116,10 +116,15 @@
         try:
             try:
                 w_result = frame.execute_frame(w_arg, operr)
-            except OperationError:
+            except OperationError as e:
                 # errors finish a frame
-                self.frame = None
+                try:
+                    if e.match(space, space.w_StopIteration):
+                        self._leak_stopiteration(e)
+                finally:
+                    self.frame = None
                 raise
+            #
             # if the frame is now marked as finished, it was RETURNed from
             if frame.frame_finished_execution:
                 self.frame = None
@@ -135,6 +140,25 @@
             frame.f_backref = jit.vref_None
             self.running = False
 
+    def _leak_stopiteration(self, e):
+        # Check for __future__ generator_stop and conditionally turn
+        # a leaking StopIteration into RuntimeError (with its cause
+        # set appropriately).
+        space = self.space
+        if self.pycode.co_flags & (consts.CO_FUTURE_GENERATOR_STOP |
+                                   consts.CO_COROUTINE |
+                                   consts.CO_ITERABLE_COROUTINE):
+            e2 = OperationError(space.w_RuntimeError,
+                                space.wrap("%s raised StopIteration" %
+                                           self.KIND),
+                                w_cause=e.get_w_value(space))
+            e2.record_context(space, self.frame)
+            raise e2
+        else:
+            space.warn(space.wrap("generator '%s' raised StopIteration"
+                                  % self.get_qualname()),
+                       space.w_PendingDeprecationWarning)
+
     def descr_throw(self, w_type, w_val=None, w_tb=None):
         """throw(typ[,val[,tb]]) -> raise exception in generator/coroutine,
 return next yielded value or raise StopIteration."""
diff --git a/pypy/interpreter/pycompiler.py b/pypy/interpreter/pycompiler.py
--- a/pypy/interpreter/pycompiler.py
+++ b/pypy/interpreter/pycompiler.py
@@ -101,7 +101,7 @@
     """
     def __init__(self, space, override_version=None):
         PyCodeCompiler.__init__(self, space)
-        self.future_flags = future.futureFlags_3_2
+        self.future_flags = future.futureFlags_3_5
         self.parser = pyparse.PythonParser(space, self.future_flags)
         self.additional_rules = {}
         self.compiler_flags = self.future_flags.allowed_flags
diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -30,6 +30,7 @@
 futureFlags_2_5 = FutureFlags((2, 5, 0, 'final', 0))
 futureFlags_2_7 = FutureFlags((2, 7, 0, 'final', 0))
 futureFlags_3_2 = FutureFlags((3, 2, 0, 'final', 0))
+futureFlags_3_5 = FutureFlags((3, 5, 0, 'final', 0))
 
 
 class TokenIterator:
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -95,7 +95,7 @@
 
 class PythonParser(parser.Parser):
 
-    def __init__(self, space, future_flags=future.futureFlags_3_2,
+    def __init__(self, space, future_flags=future.futureFlags_3_5,
                  grammar=pygram.python_grammar):
         parser.Parser.__init__(self, grammar)
         self.space = space
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -641,3 +641,23 @@
             2,
         ]
 
+
+class AppTestGeneratorStop:
+
+    def test_past_generator_stop(self):
+        # how it works without 'from __future__' import generator_stop
+        def f(x):
+            raise StopIteration
+            yield x
+        raises(StopIteration, next, f(5))
+
+    def test_future_generator_stop(self):
+        d = {}
+        exec("""from __future__ import generator_stop
+
+def f(x):
+    raise StopIteration
+    yield x
+""", d)
+        f = d['f']
+        raises(RuntimeError, next, f(5))

From pypy.commits at gmail.com  Wed Sep 14 14:13:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 11:13:47 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57d9935b.e840c20a.9b5f.c7b9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r798:78fcbb44c7d2
Date: 2016-09-14 20:13 +0200
http://bitbucket.org/pypy/pypy.org/changeset/78fcbb44c7d2/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $65104 of $105000 (62.0%)
+   $65109 of $105000 (62.0%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5557 left
+    we have $5561 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Sep 14 15:24:50 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 14 Sep 2016 12:24:50 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: add test that passes -A fails
 untranslated, bf_get* slots are NULL in subclass
Message-ID: <57d9a402.812e1c0a.5d78b.21c3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87116:1079c793b219
Date: 2016-09-14 22:12 +0300
http://bitbucket.org/pypy/pypy/changeset/1079c793b219/

Log:	add test that passes -A fails untranslated, bf_get* slots are NULL
	in subclass

diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2394,14 +2394,29 @@
     (iternextfunc)arrayiter_next,               /* tp_iternext */
     0,                                          /* tp_methods */
 };
+static PyObject *
+readbuffer_as_string(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    const void *ptr;
+    Py_ssize_t size;
+
+    if (!PyArg_ParseTuple(args, "O", &obj)) {
+        return NULL;
+    }
+    if (PyObject_AsReadBuffer(obj, &ptr, &size) < 0)
+        return NULL;
+    return PyString_FromStringAndSize((char*)ptr, size);
+}
+
 
 
 /*********************** Install Module **************************/
 
-/* No functions in array module. */
 static PyMethodDef a_methods[] = {
     {"_reconstruct",   (PyCFunction)_reconstruct, METH_VARARGS, NULL},
     {"switch_multiply",   (PyCFunction)switch_multiply, METH_NOARGS, NULL},
+    {"readbuffer_as_string",   (PyCFunction)readbuffer_as_string, METH_VARARGS, NULL},
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -89,6 +89,7 @@
         assert res == [2, 4, 6]
 
     def test_subclass(self):
+        import struct
         module = self.import_module(name='array')
         class Sub(module.array):
             pass
@@ -97,3 +98,5 @@
         res = [1, 2, 3] * arr
         assert res == [1, 2, 3, 1, 2, 3]
         
+        val = module.readbuffer_as_string(arr)
+        assert val == struct.pack('i', 2)

From pypy.commits at gmail.com  Wed Sep 14 23:57:12 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 14 Sep 2016 20:57:12 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: fill in tp_as_buffer.*
 functions that have no dunder slot
Message-ID: <57da1c18.c186c20a.ff68f.5915@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87117:e5be2ebc1e87
Date: 2016-09-15 06:56 +0300
http://bitbucket.org/pypy/pypy/changeset/e5be2ebc1e87/

Log:	fill in tp_as_buffer.* functions that have no dunder slot

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -785,6 +785,22 @@
             pto.c_tp_getattro = base.c_tp_getattro
         if not pto.c_tp_as_buffer:
             pto.c_tp_as_buffer = base.c_tp_as_buffer
+        if base.c_tp_as_buffer:
+            # also inherit all the base.c_tp_as_buffer functions, since they
+            # do not have real __slots__ they are not filled in otherwise
+            # skip bf_getbuffer, which is __buffer__
+            pto_as = pto.c_tp_as_buffer
+            base_as = base.c_tp_as_buffer
+            if not pto_as.c_bf_getreadbuffer:
+                pto_as.c_bf_getreadbuffer = base_as.c_bf_getreadbuffer
+            if not pto_as.c_bf_getwritebuffer:
+                pto_as.c_bf_getwritebuffer = base_as.c_bf_getwritebuffer
+            if not pto_as.c_bf_getsegcount:
+                pto_as.c_bf_getsegcount = base_as.c_bf_getsegcount
+            if not pto_as.c_bf_getcharbuffer:
+                pto_as.c_bf_getcharbuffer = base_as.c_bf_getcharbuffer
+            if not pto_as.c_bf_getcharbuffer:
+                pto_as.c_bf_releasebuffer = base_as.c_bf_releasebuffer
     finally:
         Py_DecRef(space, base_pyo)
 

From pypy.commits at gmail.com  Thu Sep 15 00:36:52 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 14 Sep 2016 21:36:52 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: failing test
Message-ID: <57da2564.68adc20a.349d.6286@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87118:5b04c1bb7084
Date: 2016-09-15 07:36 +0300
http://bitbucket.org/pypy/pypy/changeset/5b04c1bb7084/

Log:	failing test

	is the decode (which is not the core of the test) valid on other
	platforms?

diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -100,3 +100,10 @@
         
         val = module.readbuffer_as_string(arr)
         assert val == struct.pack('i', 2)
+
+    def test_unicode_readbuffer(self):
+        # Not really part of array, refactor
+        import struct
+        module = self.import_module(name='array')
+        val = module.readbuffer_as_string(u'\u03a3')
+        assert val.decode('utf32') == u'\u03a3'
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -785,6 +785,7 @@
             pto.c_tp_getattro = base.c_tp_getattro
         if not pto.c_tp_as_buffer:
             pto.c_tp_as_buffer = base.c_tp_as_buffer
+        # XXX need to refactor: what about built-in objects i.e. W_Unicode
         if base.c_tp_as_buffer:
             # also inherit all the base.c_tp_as_buffer functions, since they
             # do not have real __slots__ they are not filled in otherwise

From pypy.commits at gmail.com  Thu Sep 15 02:53:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 23:53:36 -0700 (PDT)
Subject: [pypy-commit] cffi default: Warn when implicitly casting between
 "char *" and a different pointer
Message-ID: <57da4570.4398c20a.ed3b3.9005@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2768:7c0b1fadb4da
Date: 2016-09-15 08:39 +0200
http://bitbucket.org/cffi/cffi/changeset/7c0b1fadb4da/

Log:	Warn when implicitly casting between "char *" and a different
	pointer type, like C does (there is no special-case of "char *" in
	C)

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -128,7 +128,7 @@
 #define CT_VOID             512    /* void */
 
 /* other flags that may also be set in addition to the base flag: */
-#define CT_CAST_ANYTHING         1024    /* 'char *' and 'void *' only */
+#define CT_IS_VOIDCHAR_PTR       1024
 #define CT_PRIMITIVE_FITS_LONG   2048
 #define CT_IS_OPAQUE             4096
 #define CT_IS_ENUM               8192
@@ -1358,9 +1358,26 @@
             }
         }
         if (ctinit != ct) {
-            if ((ct->ct_flags & CT_CAST_ANYTHING) ||
-                (ctinit->ct_flags & CT_CAST_ANYTHING))
-                ;   /* accept void* or char* as either source or target */
+            int combined_flags = ct->ct_flags | ctinit->ct_flags;
+            if (combined_flags & CT_IS_VOID_PTR)
+                ;   /* accept "void *" as either source or target */
+            else if (combined_flags & CT_IS_VOIDCHAR_PTR) {
+                /* for backward compatibility, accept "char *" as either
+                   source of target.  This is not what C does, though,
+                   so emit a warning that will eventually turn into an
+                   error. */
+                char *msg = (ct->ct_flags & CT_IS_VOIDCHAR_PTR ?
+                    "implicit cast to 'char *' from a different pointer type: "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" :
+                    "implicit cast from 'char *' to a different pointer type: "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)");
+                if (PyErr_WarnEx(PyExc_UserWarning, msg, 1))
+                    return -1;
+            }
             else {
                 expected = "pointer to same type";
                 goto cannot_convert;
@@ -2476,7 +2493,7 @@
     if (PyBytes_Check(init)) {
         /* from a string: just returning the string here is fine.
            We assume that the C code won't modify the 'char *' data. */
-        if ((ctptr->ct_flags & CT_CAST_ANYTHING) ||
+        if ((ctptr->ct_flags & CT_IS_VOIDCHAR_PTR) ||
             ((ctitem->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED))
              && (ctitem->ct_size == sizeof(char)))) {
 #if defined(CFFI_MEM_DEBUG) || defined(CFFI_MEM_LEAK)
@@ -3929,7 +3946,7 @@
     if ((ctitem->ct_flags & CT_VOID) ||
         ((ctitem->ct_flags & CT_PRIMITIVE_CHAR) &&
          ctitem->ct_size == sizeof(char)))
-        td->ct_flags |= CT_CAST_ANYTHING;   /* 'void *' or 'char *' only */
+        td->ct_flags |= CT_IS_VOIDCHAR_PTR;   /* 'void *' or 'char *' only */
     unique_key[0] = ctitem;
     return get_unique_type(td, unique_key, 1);
 }
@@ -5903,7 +5920,7 @@
         return NULL;
     }
     ct = ((CDataObject *)arg)->c_type;
-    if (!(ct->ct_flags & CT_CAST_ANYTHING)) {
+    if (!(ct->ct_flags & CT_IS_VOIDCHAR_PTR)) {
         PyErr_Format(PyExc_TypeError,
                      "expected a 'cdata' object with a 'void *' out of "
                      "new_handle(), got '%s'", ct->ct_name);
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3676,3 +3676,25 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2

From pypy.commits at gmail.com  Thu Sep 15 02:53:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 23:53:38 -0700 (PDT)
Subject: [pypy-commit] cffi default: bump version number to 1.8.3
Message-ID: <57da4572.6937c20a.c44a4.91c2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2769:6abfa4fb79bc
Date: 2016-09-15 08:42 +0200
http://bitbucket.org/cffi/cffi/changeset/6abfa4fb79bc/

Log:	bump version number to 1.8.3

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.8.2"
+#define CFFI_VERSION  "1.8.3"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.2"
-__version_info__ = (1, 8, 2)
+__version__ = "1.8.3"
+__version_info__ = (1, 8, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.2"
+                               "\ncompiled with cffi version: 1.8.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.8'
 # The full version, including alpha/beta/rc tags.
-release = '1.8.2'
+release = '1.8.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.2.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.3.tar.gz
 
-   - MD5: 538f307b6c5169bba41fbfda2b070762
+   - MD5: ...
 
-   - SHA: 9d2722ba9241b232b980bb9243e12451513a8000
+   - SHA: ...
 
-   - SHA256: 2b636db1a179439d73ae0a090479e179a43df5d4eddc7e4c4067f960d4038530
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.8.2',
+        version='1.8.3',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Thu Sep 15 02:53:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Sep 2016 23:53:40 -0700 (PDT)
Subject: [pypy-commit] cffi default: Document the new warning
Message-ID: <57da4574.411d1c0a.1f8fe.4ccb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2770:5fc2d2cf2f5d
Date: 2016-09-15 08:53 +0200
http://bitbucket.org/cffi/cffi/changeset/5fc2d2cf2f5d/

Log:	Document the new warning

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3679,6 +3679,8 @@
 
 def test_char_pointer_conversion():
     import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
     BVoidP = new_pointer_type(new_void_type())
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,19 @@
 ======================
 
 
+v1.8.3
+======
+
+* When passing a ``void *`` argument to a function with a different
+  pointer type, or vice-versa, the cast occurs automatically, like in C.
+  The same occurs for initialization with ``ffi.new()`` and a few other
+  places.  However, I thought that ``char *`` had the same
+  property---but I was mistaken.  In C you get the usual warning if you
+  try to give a ``char *`` to a ``char **`` argument, for example.
+  Sorry about the confusion.  This has been fixed in CFFI by giving for
+  now a warning, too.  It will turn into an error in a future version.
+
+
 v1.8.2
 ======
 

From pypy.commits at gmail.com  Thu Sep 15 03:26:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 15 Sep 2016 00:26:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: Test and fix
Message-ID: <57da4d16.c3d41c0a.316d7.560a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87119:b509a7598625
Date: 2016-09-15 09:23 +0200
http://bitbucket.org/pypy/pypy/changeset/b509a7598625/

Log:	Test and fix

diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -59,6 +59,16 @@
                     lltype.free(self.cif_descr, flavor='raw')
                     self.cif_descr = lltype.nullptr(CIF_DESCRIPTION)
 
+    def is_unichar_ptr_or_array(self):
+        return False
+
+    def is_char_or_unichar_ptr_or_array(self):
+        return False
+
+    def string(self, cdataobj, maxlen):
+        # Can't use ffi.string() on a function pointer
+        return W_CType.string(self, cdataobj, maxlen)
+
     def new_ctypefunc_completing_argtypes(self, args_w):
         space = self.space
         nargs_declared = len(self.fargs)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -503,3 +503,10 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_bug_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        q = ffi.new("char[]", "abcd")
+        p = ffi.cast("char(*)(void)", q)
+        raises(TypeError, ffi.string, p)

From pypy.commits at gmail.com  Thu Sep 15 04:17:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 15 Sep 2016 01:17:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: Update to cffi/5fc2d2cf2f5d
Message-ID: <57da590f.898b1c0a.b0412.6ad8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87120:eba5b82d8e59
Date: 2016-09-15 10:09 +0200
http://bitbucket.org/pypy/pypy/changeset/eba5b82d8e59/

Log:	Update to cffi/5fc2d2cf2f5d

	Warn when implicitly casting between "char *" and a different
	pointer type, like C does (there is no special-case of "char *" in C

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.2
+Version: 1.8.3
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.2"
-__version_info__ = (1, 8, 2)
+__version__ = "1.8.3"
+__version_info__ = (1, 8, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.2"
+                               "\ncompiled with cffi version: 1.8.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1974,6 +1974,7 @@
     'ZeroDivisionError',
     'RuntimeWarning',
     'PendingDeprecationWarning',
+    'UserWarning',
 ]
 
 if sys.platform.startswith("win"):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.2"
+VERSION = "1.8.3"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer
 from pypy.module._cffi_backend import ctypeprim
 
 
@@ -22,6 +22,7 @@
     is_nonfunc_pointer_or_array = True
 
     def __init__(self, space, ctptr, length, arraysize, extra):
+        assert isinstance(ctptr, W_CTypePointer)
         W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
                                    ctptr.ctitem)
         self.length = length
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -35,8 +35,7 @@
         assert isinstance(ellipsis, bool)
         extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi)
         size = rffi.sizeof(rffi.VOIDP)
-        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult,
-                                could_cast_anything=False)
+        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult)
         self.fargs = fargs
         self.ellipsis = ellipsis
         self.abi = abi
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -19,7 +19,6 @@
     # XXX this could be improved with an elidable method get_size()
     # that raises in case it's still -1...
 
-    cast_anything = False
     is_primitive_integer = False
     is_nonfunc_pointer_or_array = False
     is_indirect_arg_for_call_python = False
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -120,7 +120,6 @@
 
 class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar):
     _attrs_ = []
-    cast_anything = True
 
     def cast_to_int(self, cdata):
         return self.space.wrap(ord(cdata[0]))
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,12 +14,11 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _attrs_            = ['ctitem', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'accept_str', 'length']
     length = -1
 
-    def __init__(self, space, size, extra, extra_position, ctitem,
-                 could_cast_anything=True):
+    def __init__(self, space, size, extra, extra_position, ctitem):
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
@@ -27,10 +26,11 @@
         #  - for arrays, it is the array item type
         #  - for functions, it is the return type
         self.ctitem = ctitem
-        self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.accept_str = (self.can_cast_anything or
-                            (ctitem.is_primitive_integer and
-                             ctitem.size == rffi.sizeof(lltype.Char)))
+        self.accept_str = (self.is_nonfunc_pointer_or_array and
+                (isinstance(ctitem, ctypevoid.W_CTypeVoid) or
+                 isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or
+                 (ctitem.is_primitive_integer and
+                  ctitem.size == rffi.sizeof(lltype.Char))))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -137,7 +137,10 @@
 
 class W_CTypePtrBase(W_CTypePtrOrArray):
     # base class for both pointers and pointers-to-functions
-    _attrs_ = []
+    _attrs_ = ['is_void_ptr', 'is_voidchar_ptr']
+    _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr']
+    is_void_ptr = False
+    is_voidchar_ptr = False
 
     def convert_to_object(self, cdata):
         ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0]
@@ -154,7 +157,16 @@
             else:
                 raise self._convert_error("compatible pointer", w_ob)
         if self is not other:
-            if not (self.can_cast_anything or other.can_cast_anything):
+            if self.is_void_ptr or other.is_void_ptr:
+                pass     # cast from or to 'void *'
+            elif self.is_voidchar_ptr or other.is_voidchar_ptr:
+                space = self.space
+                msg = ("implicit cast from '%s' to '%s' "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" % (other.name, self.name))
+                space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1)
+            else:
                 raise self._convert_error("compatible pointer", w_ob)
 
         rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
@@ -165,8 +177,8 @@
 
 
 class W_CTypePointer(W_CTypePtrBase):
-    _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types']
-    _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr']
+    _attrs_ = ['is_file', 'cache_array_type', '_array_types']
+    _immutable_fields_ = ['is_file', 'cache_array_type?']
     kind = "pointer"
     cache_array_type = None
     is_nonfunc_pointer_or_array = True
@@ -181,6 +193,8 @@
         self.is_file = (ctitem.name == "struct _IO_FILE" or
                         ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
+        self.is_voidchar_ptr = (self.is_void_ptr or
+                           isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar))
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init, allocator):
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -7,7 +7,6 @@
 
 class W_CTypeVoid(W_CType):
     _attrs_ = []
-    cast_anything = True
     kind = "void"
 
     def __init__(self, space):
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -32,8 +32,8 @@
 @unwrap_spec(w_cdata=cdataobj.W_CData)
 def from_handle(space, w_cdata):
     ctype = w_cdata.ctype
-    if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or
-        not ctype.can_cast_anything):
+    if (not isinstance(ctype, ctypeptr.W_CTypePointer) or
+        not ctype.is_voidchar_ptr):
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3665,3 +3665,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2

From pypy.commits at gmail.com  Thu Sep 15 05:19:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 15 Sep 2016 02:19:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: For now,
 forbid --profopt if we're building a shared pypy
Message-ID: <57da67ad.0145c20a.d1f91.c54e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87121:7058ed1d7d93
Date: 2016-09-15 11:19 +0200
http://bitbucket.org/pypy/pypy/changeset/7058ed1d7d93/

Log:	For now, forbid --profopt if we're building a shared pypy

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -239,6 +239,10 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+            if config.translation.profopt is not None:
+                raise Exception("Cannot use the --profopt option "
+                                "when --shared is on (it is by default). "
+                                "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)

From pypy.commits at gmail.com  Thu Sep 15 06:10:10 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 15 Sep 2016 03:10:10 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: more vec op backend impl.
 missing, pack/unpack for i/f
Message-ID: <57da7382.6773c20a.b2ece.dcdc@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87122:8022d6fb623c
Date: 2016-09-15 12:09 +0200
http://bitbucket.org/pypy/pypy/changeset/8022d6fb623c/

Log:	more vec op backend impl. missing, pack/unpack for i/f

diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py
--- a/rpython/jit/backend/zarch/assembler.py
+++ b/rpython/jit/backend/zarch/assembler.py
@@ -1542,8 +1542,8 @@
         pmc.overwrite()
 
 def notimplemented_op(asm, op, arglocs, regalloc):
+    msg = "[zarch/asm] %s not implemented\n" % op.getopname()
     if we_are_translated():
-        msg = "[ZARCH/asm] %s not implemented\n" % op.getopname()
         llop.debug_print(lltype.Void, msg)
     raise NotImplementedError(msg)
 
diff --git a/rpython/jit/backend/zarch/instruction_builder.py b/rpython/jit/backend/zarch/instruction_builder.py
--- a/rpython/jit/backend/zarch/instruction_builder.py
+++ b/rpython/jit/backend/zarch/instruction_builder.py
@@ -486,6 +486,21 @@
         self.writechar(opcode2)
     return encode_vrr_a
 
+def build_vrr_b(mnemonic, (opcode1,opcode2), argtypes='v,v,v,m,m'):
+    @builder.arguments(argtypes)
+    def encode_vrr_b(self, v1, v2, v3, mask4, mask5):
+        self.writechar(opcode1)
+        rbx =  (v1 >= 16) << 3
+        rbx |= (v2 >= 16) << 2
+        rbx |= (v3 >= 16) << 1
+        byte = (v1 & BIT_MASK_4) << 4 | (v2 & BIT_MASK_4)
+        self.writechar(chr(byte))
+        self.writechar(chr((v3 & BIT_MASK_4) << 4))
+        self.writechar(chr((mask5 & BIT_MASK_4) << 4))
+        self.writechar(chr((mask4 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
+        self.writechar(opcode2)
+    return encode_vrr_b
+
 def build_vrr_c(mnemonic, (opcode1,opcode2), argtypes='v,v,v,m,m,m'):
     @builder.arguments(argtypes)
     def encode_vrr_c(self, v1, v2, v3, mask4=0, mask5=0, mask6=0):
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -295,6 +295,7 @@
 
 vector_mnemonic_codes = {
     'VL':     ('vrx', ['\xE7','\x06'], 'v,bid'),
+    'VLREP':  ('vrx', ['\xE7','\x05']),
     'VLR':    ('vrr_a', ['\xE7','\x56'], 'v,v'),
 
     'VST':    ('vrx', ['\xE7','\x0E'], 'v,bid'),
@@ -318,6 +319,7 @@
 
     # conversion
     'VCDG':  ('vrr_a', ['\xE7','\xC3']),
+    'VCGD':  ('vrr_a', ['\xE7','\xC2']),
 
     # compare, sign, ...
     'VFPSO': ('vrr_a', ['\xE7','\xCC']),
@@ -326,6 +328,12 @@
     'VPERM': ('vrr_e', ['\xE7','\x8C'], 'v,v,v,v'),
     'VREPI': ('vri_a', ['\xE7','\x45']),
 
+    'VCEQ':  ('vrr_b', ['\xE7','\xF8']),
+
+    # pack, merge, shift, ...
+    'VMRL':  ('vrr_c', ['\xE7','\x60'], 'v,v,v,m'),
+    'VMRH':  ('vrr_c', ['\xE7','\x61'], 'v,v,v,m'),
+
     # '': ('', ['','']),
 }
 
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -220,85 +220,44 @@
         flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
 
     def emit_vec_float_xor(self, op, arglocs, regalloc):
-        resloc, l0, l1, sizeloc = arglocs
-        res = resloc.value
-        r0 = l0.value
-        r1 = l1.value
-        self.mc.xxlxor(res, r0, r1)
+        resloc, loc0, loc1, sizeloc = arglocs
+        self.mc.VX(resloc, loc0, loc1)
 
     def emit_vec_float_ne(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        resloc, loc1, loc2, sizeloc = arglocs
+        resloc, loc0, loc1, sizeloc = arglocs
         size = sizeloc.value
-        tmp = regalloc.vrm.get_scratch_reg().value
-        offloc = regalloc.rm.get_scratch_reg()
-        off = offloc.value
-        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
-        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
-        if size == 4:
-            self.mc.xvcmpeqspx(tmp, loc1.value, loc2.value)
-            self.mc.stxvw4x(tmp, off, r.SP.value)
-        elif size == 8:
-            self.mc.xvcmpeqdpx(tmp, loc1.value, loc2.value)
-            self.mc.stxvd2x(tmp, off, r.SP.value)
+        if size == 8:
+            # bit 3 in last argument sets the condition code
+            self.mc.VFCE(resloc, loc0, loc1, 3, 0, 1)
+            self.mc.VNO(resloc, resloc, resloc)
         else:
-            not_implemented("float == for size %d" % size)
-        res = resloc.value
-        self.mc.lvx(res, off, r.SP.value)
-        self.mc.vnor(res, res, res) # complement
+            not_implemented("[zarch/assembler] float != for size %d" % size)
         flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc)
 
     def emit_vec_cast_int_to_float(self, op, arglocs, regalloc):
         resloc, loc0 = arglocs
-        offloc = regalloc.rm.get_scratch_reg()
-        off = offloc.value
-        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
-        # bit 1 on mask4 -> supresses inexact exception
         self.mc.VCDG(resloc, loc0, 3, 4, m.RND_TOZERO.value)
-        #self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
-        #self.mc.stvx(l0.value, off, r.SP.value)
-        #self.mc.lxvd2x(res.value, off, r.SP.value)
-        #self.mc.xvcvsxddp(res.value, res.value)
 
     def emit_vec_int_eq(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        res, l0, l1, sizeloc = arglocs
+        resloc, loc0, loc1, sizeloc = arglocs
         size = sizeloc.value
-        if size == 1:
-            self.mc.vcmpequbx(res.value, l0.value, l1.value)
-        elif size == 2:
-            self.mc.vcmpequhx(res.value, l0.value, l1.value)
-        elif size == 4:
-            self.mc.vcmpequwx(res.value, l0.value, l1.value)
-        elif size == 8:
-            self.mc.vcmpequdx(res.value, l0.value, l1.value)
-        flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res)
+        self.mc.VCEQ(resloc, loc0, loc1, l.itemsize_to_mask(size), 1)
+        flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
 
     def emit_vec_int_ne(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        res, l0, l1, sizeloc = arglocs
+        resloc, loc0, loc1, sizeloc = arglocs
         size = sizeloc.value
-        tmp = regalloc.vrm.get_scratch_reg(type=INT).value
-        self.mc.vxor(tmp, tmp, tmp)
-        if size == 1:
-            self.mc.vcmpequbx(res.value, res.value, tmp)
-        elif size == 2:
-            self.mc.vcmpequhx(res.value, res.value, tmp)
-        elif size == 4:
-            self.mc.vcmpequwx(res.value, res.value, tmp)
-        elif size == 8:
-            self.mc.vcmpequdx(res.value, res.value, tmp)
-        self.mc.vnor(res.value, res.value, res.value)
-        flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, res)
+        self.mc.VCEQ(resloc, loc0, loc1, l.itemsize_to_mask(size), 1)
+        self.mc.VNO(resloc, resloc, resloc)
+        flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, res)
 
     def emit_vec_cast_float_to_int(self, op, arglocs, regalloc):
-        res, l0 = arglocs
-        offloc = regalloc.rm.get_scratch_reg()
-        v0 = regalloc.vrm.get_scratch_reg(type=INT)
-        off = offloc.value
-        # SP is always 16 byte aligned, and PARAM_SAVE_AREA_OFFSET % 16 == 0
-        self.mc.load_imm(offloc, PARAM_SAVE_AREA_OFFSET)
-        self.mc.xvcvdpsxds(res.value, l0.value)
+        resloc, loc0 = arglocs
+        # 4 => bit 1 from the MSB: XxC
+        self.mc.VCGD(resloc, loc0, 3, 4, mask.RND_TOZERO.value)
 
     def emit_vec_expand_f(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
@@ -320,30 +279,37 @@
 
     def emit_vec_expand_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        res, l0, off = arglocs
+        resloc, loc0 = arglocs
         size = op.bytesize
+        self.mc.VLREP(resloc, loc0, l.itemsize_to_mask(size))
 
-        self.mc.load_imm(r.SCRATCH2, off.value)
-        self.mc.lvx(res.value, r.SCRATCH2.value, r.SP.value)
-        if size == 1:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltb(res.value, res.value, 0b0000)
-            else:
-                self.mc.vspltb(res.value, res.value, 0b1111)
-        elif size == 2:
-            if IS_BIG_ENDIAN:
-                self.mc.vsplth(res.value, res.value, 0b000)
-            else:
-                self.mc.vsplth(res.value, res.value, 0b111)
-        elif size == 4:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltw(res.value, res.value, 0b00)
-            else:
-                self.mc.vspltw(res.value, res.value, 0b11)
-        elif size == 8:
-            pass
+    emit_vec_expand_f = emit_vec_expand_i
+
+    def _accum_reduce(self, op, arg, accumloc, targetloc):
+        # Currently the accumulator can ONLY be 64 bit float/int
+        if arg.type == FLOAT:
+            # r = (r[0]+r[1],r[0]+r[1])
+            self.mc.VMRL(targetloc, accumloc, accumloc)
+            if op == '+':
+                self.mc.VFA(targetloc, targetloc, accumloc)
+                return
+            elif op == '*':
+                self.mc.VFM(targetloc, targetloc, accumloc)
+                return
         else:
-            not_implemented("expand int size not impl")
+            assert arg.type == INT
+            # store the vector onto the stack, just below the stack pointer
+            self.mc.VST(accumloc, l.addr(0, r.SP))
+            self.mc.LG(r.SCRATCH, l.addr(0, r.SP))
+            self.mc.LG(targetloc, l.addr(8, r.SP))
+            if op == '+':
+                self.mc.AGR(targetloc, r.SCRATCH)
+                return
+            elif op == '*':
+                self.mc.MSGR(targetloc, r.SCRATCH)
+                return
+        not_implemented("reduce sum for %s not impl." % arg)
+
 
     def emit_vec_pack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
@@ -467,6 +433,7 @@
                 # r = (v[0], s[1])
                 self.mc.xxpermdi(res, vec, src, permi(0,1))
 
+
     def emit_vec_unpack_f(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, srcloc, srcidxloc, countloc = arglocs
@@ -486,41 +453,6 @@
             return
         not_implemented("unpack for combination src %d -> res %d" % (srcidx, residx))
 
-    def _accum_reduce(self, op, arg, accumloc, targetloc):
-        # Currently the accumulator can ONLY be the biggest
-        # 64 bit float/int
-        # TODO
-        tgt = targetloc.value
-        acc = accumloc.value
-        if arg.type == FLOAT:
-            # r = (r[0]+r[1],r[0]+r[1])
-            if IS_BIG_ENDIAN:
-                self.mc.xxpermdi(tgt, acc, acc, 0b00)
-            else:
-                self.mc.xxpermdi(tgt, acc, acc, 0b10)
-            if op == '+':
-                self.mc.xsadddp(tgt, tgt, acc)
-            elif op == '*':
-                self.mc.xsmuldp(tgt, tgt, acc)
-            else:
-                not_implemented("sum not implemented")
-            return
-        else:
-            assert arg.type == INT
-            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
-            self.mc.stvx(acc, r.SCRATCH2.value, r.SP.value)
-            self.mc.load(tgt, r.SP.value, PARAM_SAVE_AREA_OFFSET)
-            self.mc.load(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
-            if op == '+':
-                self.mc.add(tgt, tgt, acc)
-            elif op == '*':
-                self.mc.mulld(tgt, tgt, acc)
-            else:
-                not_implemented("sum not implemented")
-            return
-
-        not_implemented("reduce sum for %s not impl." % arg)
-
     def emit_vec_f(self, op, arglocs, regalloc):
         pass
     emit_vec_i = emit_vec_f
@@ -729,18 +661,10 @@
         assert isinstance(op, VectorOp)
         arg = op.getarg(0)
         mc = self.assembler.mc
-        if arg.is_constant():
-            assert isinstance(arg, ConstInt)
-            l0 = self.rm.get_scratch_reg()
-            mc.load_imm(l0, arg.value)
-        else:
-            l0 = self.ensure_reg(arg)
-        mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET)
+        l0 = self.ensure_reg_or_pool(arg)
         size = op.bytesize
-        if size == 8:
-            mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
         res = self.force_allocate_vector_reg(op)
-        return [res, l0, imm(PARAM_SAVE_AREA_OFFSET)]
+        return [res, l0]
 
     def prepare_vec_int_is_true(self, op):
         assert isinstance(op, VectorOp)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -273,9 +273,9 @@
     test_vec_xor_short = \
         vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SHORT)
 
-    test_vec_int_eq = \
+    test_vec_int_cmp_eq = \
         vec_int_arith(lambda a,b: a == b, rffi.SIGNED)
-    test_vec_int_ne = \
+    test_vec_int_cmp_ne = \
         vec_int_arith(lambda a,b: a == b, rffi.SIGNED)
 
     @py.test.mark.parametrize('i',[1,2,3,4,9])

From pypy.commits at gmail.com  Thu Sep 15 17:05:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 15 Sep 2016 14:05:23 -0700 (PDT)
Subject: [pypy-commit] cffi default: Docs: remove "char *" from the special
 casing in the reference
Message-ID: <57db0d13.eb01c30a.ef930.dac4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2771:c5a1ab91b002
Date: 2016-09-15 23:05 +0200
http://bitbucket.org/cffi/cffi/changeset/c5a1ab91b002/

Log:	Docs: remove "char *" from the special casing in the reference

diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -575,12 +575,12 @@
 +---------------+------------------------+------------------+----------------+
 |  pointers     | another <cdata> with   | a <cdata>        |``[]`` `(****)`,|
 |               | a compatible type (i.e.|                  |``+``, ``-``,   |
-|               | same type or ``char*`` |                  |bool()          |
+|               | same type              |                  |bool()          |
 |               | or ``void*``, or as an |                  |                |
 |               | array instead) `(*)`   |                  |                |
 +---------------+------------------------+                  |                |
-|  ``void *``,  | another <cdata> with   |                  |                |
-|  ``char *``   | any pointer or array   |                  |                |
+|  ``void *``   | another <cdata> with   |                  |                |
+|               | any pointer or array   |                  |                |
 |               | type                   |                  |                |
 +---------------+------------------------+                  +----------------+
 |  pointers to  | same as pointers       |                  | ``[]``, ``+``, |

From pypy.commits at gmail.com  Thu Sep 15 17:10:49 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Thu, 15 Sep 2016 14:10:49 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: QCGC codebase update
Message-ID: <57db0e59.4cc51c0a.c72f1.8df3@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87124:0710ed582ef1
Date: 2016-09-13 21:20 +0200
http://bitbucket.org/pypy/pypy/changeset/0710ed582ef1/

Log:	QCGC codebase update

diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -70,7 +70,7 @@
 		qcgc_collect();
 	}
 	if (qcgc_state.bytes_since_incmark > qcgc_state.incmark_threshold) {
-		qcgc_mark(true);
+		qcgc_incmark();
 	}
 
 	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
@@ -104,7 +104,7 @@
 }
 
 void qcgc_collect(void) {
-	qcgc_mark(false);
+	qcgc_mark();
 	qcgc_sweep();
 	qcgc_state.bytes_since_collection = 0;
 }
diff --git a/rpython/translator/c/src/qcgc/src/collector.c b/rpython/translator/c/src/qcgc/src/collector.c
--- a/rpython/translator/c/src/qcgc/src/collector.c
+++ b/rpython/translator/c/src/qcgc/src/collector.c
@@ -9,8 +9,77 @@
 
 QCGC_STATIC QCGC_INLINE void qcgc_pop_object(object_t *object);
 QCGC_STATIC QCGC_INLINE void qcgc_push_object(object_t *object);
+QCGC_STATIC void mark_setup(bool incremental);
+QCGC_STATIC void mark_cleanup(bool incremental);
 
-void qcgc_mark(bool incremental) {
+void qcgc_mark(void) {
+	mark_setup(false);
+	
+	while (qcgc_state.gray_stack_size > 0) {
+		// General purpose gray stack (prebuilt objects and huge blocks)
+
+		while (qcgc_state.gp_gray_stack->index > 0) {
+			object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
+			qcgc_state.gp_gray_stack = qcgc_gray_stack_pop(
+					qcgc_state.gp_gray_stack);
+			qcgc_pop_object(top);
+		}
+
+		// Arena gray stacks
+		for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
+			arena_t *arena = qcgc_allocator_state.arenas->items[i];
+
+			while (arena->gray_stack->index > 0) {
+				object_t *top = qcgc_gray_stack_top(arena->gray_stack);
+				arena->gray_stack = qcgc_gray_stack_pop(arena->gray_stack);
+				qcgc_pop_object(top);
+			}
+		}
+	}
+
+	mark_cleanup(false);
+
+#if CHECKED
+	assert(qcgc_state.phase == GC_COLLECT);
+#endif
+}
+
+void qcgc_incmark(void) {
+	mark_setup(true);
+
+	// General purpose gray stack (prebuilt objects and huge blocks)
+	size_t to_process = MAX(qcgc_state.gp_gray_stack->index / 2,
+			QCGC_INC_MARK_MIN);
+
+	while (to_process > 0 && qcgc_state.gp_gray_stack->index > 0) {
+		object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
+		qcgc_state.gp_gray_stack = qcgc_gray_stack_pop(
+				qcgc_state.gp_gray_stack);
+		qcgc_pop_object(top);
+		to_process--;
+	}
+
+	// Arena gray stacks
+	for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
+		arena_t *arena = qcgc_allocator_state.arenas->items[i];
+		to_process = MAX(arena->gray_stack->index / 2, QCGC_INC_MARK_MIN);
+
+		while (to_process > 0 && arena->gray_stack->index > 0) {
+			object_t *top = qcgc_gray_stack_top(arena->gray_stack);
+			arena->gray_stack = qcgc_gray_stack_pop(arena->gray_stack);
+			qcgc_pop_object(top);
+			to_process--;
+		}
+	}
+
+
+	mark_cleanup(true);
+#if CHECKED
+	assert(qcgc_state.phase != GC_PAUSE);
+#endif
+}
+
+QCGC_STATIC void mark_setup(bool incremental) {
 	{
 		struct log_info_s {
 			bool incremental;
@@ -45,42 +114,9 @@
 		qcgc_push_object(*it);
 	}
 
-	while (qcgc_state.gray_stack_size > 0) {
-		// General purpose gray stack (prebuilt objects and huge blocks)
-		size_t to_process = (incremental ?
-			MIN(qcgc_state.gp_gray_stack->index,
-					MAX(qcgc_state.gp_gray_stack->index / 2, QCGC_INC_MARK_MIN)) :
-			(qcgc_state.gp_gray_stack->index));
+}
 
-		while (to_process > 0) {
-			object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
-			qcgc_state.gp_gray_stack = qcgc_gray_stack_pop(
-					qcgc_state.gp_gray_stack);
-			qcgc_pop_object(top);
-			to_process--;
-		}
-
-		// Arena gray stacks
-		for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
-			arena_t *arena = qcgc_allocator_state.arenas->items[i];
-			to_process = (incremental ?
-					MIN(arena->gray_stack->index,
-						MAX(arena->gray_stack->index / 2, QCGC_INC_MARK_MIN)) :
-					(arena->gray_stack->index));
-
-			while (to_process > 0) {
-				object_t *top = qcgc_gray_stack_top(arena->gray_stack);
-				arena->gray_stack = qcgc_gray_stack_pop(arena->gray_stack);
-				qcgc_pop_object(top);
-				to_process--;
-			}
-		}
-
-		if (incremental) {
-			break; // Execute loop once for incremental collection
-		}
-	}
-
+QCGC_STATIC void mark_cleanup(bool incremental) {
 	if (qcgc_state.gray_stack_size == 0) {
 		qcgc_state.phase = GC_COLLECT;
 	}
@@ -94,13 +130,9 @@
 		qcgc_event_logger_log(EVENT_MARK_DONE, sizeof(struct log_info_s),
 				(uint8_t *) &log_info);
 	}
-#if CHECKED
-	assert(incremental || (qcgc_state.phase = GC_COLLECT));
-	assert(qcgc_state.phase != GC_PAUSE);
-#endif
 }
 
-void qcgc_pop_object(object_t *object) {
+QCGC_STATIC QCGC_INLINE void qcgc_pop_object(object_t *object) {
 #if CHECKED
 	assert(object != NULL);
 	assert((object->flags & QCGC_PREBUILT_OBJECT) == QCGC_PREBUILT_OBJECT ||
diff --git a/rpython/translator/c/src/qcgc/src/collector.h b/rpython/translator/c/src/qcgc/src/collector.h
--- a/rpython/translator/c/src/qcgc/src/collector.h
+++ b/rpython/translator/c/src/qcgc/src/collector.h
@@ -4,5 +4,6 @@
 
 #include <stdbool.h>
 
-void qcgc_mark(bool incremental);
+void qcgc_incmark(void);
+void qcgc_mark(void);
 void qcgc_sweep(void);

From pypy.commits at gmail.com  Thu Sep 15 17:10:51 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Thu, 15 Sep 2016 14:10:51 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc codebase
Message-ID: <57db0e5b.eb01c30a.ef930.dc6a@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87125:a4672aa11b87
Date: 2016-09-13 22:20 +0200
http://bitbucket.org/pypy/pypy/changeset/a4672aa11b87/

Log:	Update qcgc codebase

diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -31,8 +31,8 @@
 /**
  * Auto Mark/Collect
  */
-#define QCGC_MAJOR_COLLECTION_THRESHOLD (5 * (1<<QCGC_ARENA_SIZE_EXP))
 #define QCGC_INCMARK_THRESHOLD (1<<QCGC_ARENA_SIZE_EXP)
+#define QCGC_INCMARK_TO_SWEEP 5
 
 /**
  * DO NOT MODIFY BELOW HERE
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -32,18 +32,18 @@
 	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
 	qcgc_state.gray_stack_size = 0;
 	qcgc_state.phase = GC_PAUSE;
-	qcgc_state.bytes_since_collection = 0;
 	qcgc_state.bytes_since_incmark = 0;
+	qcgc_state.incmark_since_sweep = 0;
 	qcgc_state.free_cells = 0;
 	qcgc_state.largest_free_block = 0;
 	qcgc_allocator_initialize();
 	qcgc_hbtable_initialize();
 	qcgc_event_logger_initialize();
 
-	env_or_fallback(qcgc_state.major_collection_threshold,
-			"QCGC_MAJOR_COLLECTION", QCGC_MAJOR_COLLECTION_THRESHOLD);
 	env_or_fallback(qcgc_state.incmark_threshold,
 			"QCGC_INCMARK", QCGC_INCMARK_THRESHOLD);
+	env_or_fallback(qcgc_state.incmark_to_sweep,
+			"QCGC_INCMARK_TO_SWEEP", QCGC_INCMARK_TO_SWEEP);
 
 	setup_signal_handler();
 }
@@ -65,15 +65,18 @@
 #endif
 	object_t *result;
 
-	if (qcgc_state.bytes_since_collection >
-			qcgc_state.major_collection_threshold) {
-		qcgc_collect();
-	}
-	if (qcgc_state.bytes_since_incmark > qcgc_state.incmark_threshold) {
-		qcgc_incmark();
+	if (UNLIKELY(qcgc_state.bytes_since_incmark >
+				qcgc_state.incmark_threshold)) {
+		if (qcgc_state.incmark_since_sweep == qcgc_state.incmark_to_sweep) {
+			qcgc_collect();
+		} else {
+			qcgc_incmark();
+			qcgc_state.incmark_since_sweep++;
+		}
+		
 	}
 
-	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
+	if (LIKELY(size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP)) {
 		// Use bump / fit allocator
 		//if (qcgc_allocator_state.use_bump_allocator) {
 		if (true) {
@@ -92,7 +95,6 @@
 	}
 
 	// XXX: Should we use cells instead of bytes?
-	qcgc_state.bytes_since_collection += size;
 	qcgc_state.bytes_since_incmark += size;
 
 
@@ -106,7 +108,7 @@
 void qcgc_collect(void) {
 	qcgc_mark();
 	qcgc_sweep();
-	qcgc_state.bytes_since_collection = 0;
+	qcgc_state.incmark_since_sweep = 0;
 }
 
 void qcgc_write(object_t *object) {
diff --git a/rpython/translator/c/src/qcgc/src/gc_state.h b/rpython/translator/c/src/qcgc/src/gc_state.h
--- a/rpython/translator/c/src/qcgc/src/gc_state.h
+++ b/rpython/translator/c/src/qcgc/src/gc_state.h
@@ -33,10 +33,10 @@
 	size_t gray_stack_size;
 	gc_phase_t phase;
 
-	size_t bytes_since_collection;
 	size_t bytes_since_incmark;
-	size_t major_collection_threshold;
+	size_t incmark_since_sweep;
 	size_t incmark_threshold;
+	size_t incmark_to_sweep;
 
 	size_t free_cells;			// Overall amount of free cells without huge
 								// blocks and free areans. Valid right after sweep

From pypy.commits at gmail.com  Thu Sep 15 17:10:53 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Thu, 15 Sep 2016 14:10:53 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: QCGC update (performance)
Message-ID: <57db0e5d.81a2c20a.f812b.dde4@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87126:86dfb2ca9890
Date: 2016-09-14 08:11 +0200
http://bitbucket.org/pypy/pypy/changeset/86dfb2ca9890/

Log:	QCGC update (performance)

diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -25,6 +25,8 @@
 QCGC_STATIC QCGC_INLINE void initialize_shadowstack(void);
 QCGC_STATIC QCGC_INLINE void destroy_shadowstack(void);
 
+QCGC_STATIC object_t *bump_allocate(size_t size);
+
 void qcgc_initialize(void) {
 	initialize_shadowstack();
 	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); // XXX
@@ -79,14 +81,14 @@
 	if (LIKELY(size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP)) {
 		// Use bump / fit allocator
 		//if (qcgc_allocator_state.use_bump_allocator) {
-		if (true) {
-			result = qcgc_bump_allocate(size);
+		if (false) {
+			result = bump_allocate(size);
 		} else {
 			result = qcgc_fit_allocate(size);
 
 			// Fallback to bump allocator
 			if (result == NULL) {
-				result = qcgc_bump_allocate(size);
+				result = bump_allocate(size);
 			}
 		}
 	} else {
@@ -140,11 +142,13 @@
 	if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
 		// NOTE: No mark test here, as prebuilt objects are always reachable
 		// Push prebuilt object to general purpose gray stack
+		qcgc_state.gray_stack_size++;
 		qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
 				qcgc_state.gp_gray_stack, object);
 	} else if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
 		if (qcgc_hbtable_is_marked(object)) {
 			// Push huge block to general purpose gray stack
+			qcgc_state.gray_stack_size++;
 			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
 					qcgc_state.gp_gray_stack, object);
 		}
@@ -153,6 +157,7 @@
 					qcgc_arena_cell_index((cell_t *) object)) == BLOCK_BLACK) {
 			// This was black before, push it to gray stack again
 			arena_t *arena = qcgc_arena_addr((cell_t *) object);
+			qcgc_state.gray_stack_size++;
 			arena->gray_stack = qcgc_gray_stack_push(
 					arena->gray_stack, object);
 		}
@@ -199,3 +204,11 @@
 
 	free(qcgc_shadowstack.base);
 }
+
+QCGC_STATIC object_t *bump_allocate(size_t size) {
+	if (UNLIKELY(qcgc_allocator_state.bump_state.remaining_cells <
+			bytes_to_cells(size))) {
+		qcgc_bump_allocator_renew_block();
+	}
+	return qcgc_bump_allocate(size);
+}
diff --git a/rpython/translator/c/src/qcgc/src/allocator.c b/rpython/translator/c/src/qcgc/src/allocator.c
--- a/rpython/translator/c/src/qcgc/src/allocator.c
+++ b/rpython/translator/c/src/qcgc/src/allocator.c
@@ -3,15 +3,10 @@
 #include <assert.h>
 #include <stdbool.h>
 #include <stdlib.h>
-#include <string.h>
 
 #include "hugeblocktable.h"
 
-QCGC_STATIC size_t bytes_to_cells(size_t bytes);
-
 QCGC_STATIC QCGC_INLINE void bump_allocator_assign(cell_t *ptr, size_t cells);
-QCGC_STATIC QCGC_INLINE void bump_allocator_advance(size_t cells);
-QCGC_STATIC void bump_allocator_renew_block(void);
 
 QCGC_STATIC QCGC_INLINE bool is_small(size_t cells);
 QCGC_STATIC QCGC_INLINE size_t small_index(size_t cells);
@@ -71,20 +66,21 @@
 }
 
 void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
-	if (cells > 0) {
-		if (is_small(cells)) {
-			size_t index = small_index(cells);
-			qcgc_allocator_state.fit_state.small_free_list[index] =
-				qcgc_linear_free_list_add(
-						qcgc_allocator_state.fit_state.small_free_list[index],
-						ptr);
-		} else {
-			size_t index = large_index(cells);
-			qcgc_allocator_state.fit_state.large_free_list[index] =
-				qcgc_exp_free_list_add(
-						qcgc_allocator_state.fit_state.large_free_list[index],
-						(struct exp_free_list_item_s) {ptr, cells});
-		}
+#if CHECKED
+	assert(cells > 0);
+#endif
+	if (is_small(cells)) {
+		size_t index = small_index(cells);
+		qcgc_allocator_state.fit_state.small_free_list[index] =
+			qcgc_linear_free_list_add(
+					qcgc_allocator_state.fit_state.small_free_list[index],
+					ptr);
+	} else {
+		size_t index = large_index(cells);
+		qcgc_allocator_state.fit_state.large_free_list[index] =
+			qcgc_exp_free_list_add(
+					qcgc_allocator_state.fit_state.large_free_list[index],
+					(struct exp_free_list_item_s) {ptr, cells});
 	}
 }
 
@@ -92,42 +88,8 @@
  * Bump Allocator                                                              *
  ******************************************************************************/
 
-object_t *qcgc_bump_allocate(size_t bytes) {
+void qcgc_bump_allocator_renew_block(void) {
 #if CHECKED
-	assert(bytes <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP);
-#endif
-	size_t cells = bytes_to_cells(bytes);
-	if (UNLIKELY(cells > qcgc_allocator_state.bump_state.remaining_cells)) {
-		if (LIKELY(qcgc_allocator_state.bump_state.remaining_cells > 0)) {
-			qcgc_arena_set_blocktype(
-					qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr),
-					qcgc_arena_cell_index(
-						qcgc_allocator_state.bump_state.bump_ptr),
-					BLOCK_FREE);
-		}
-		bump_allocator_renew_block();
-	}
-	cell_t *mem = qcgc_allocator_state.bump_state.bump_ptr;
-	bump_allocator_advance(cells);
-
-	qcgc_arena_set_blocktype(qcgc_arena_addr(mem), qcgc_arena_cell_index(mem),
-			BLOCK_WHITE);
-	/*
-	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
-		qcgc_arena_set_blocktype(qcgc_allocator_state.bump_state.bump_ptr,
-				BLOCK_FREE);
-	}
-	*/
-
-	object_t *result = (object_t *) mem;
-
-#if QCGC_INIT_ZERO
-	memset(result, 0, cells * sizeof(cell_t));
-#endif
-
-	result->flags = QCGC_GRAY_FLAG;
-#if CHECKED
-	assert(qcgc_arena_is_coalesced(qcgc_arena_addr((cell_t *)result)));
 	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
 		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
 				i++) {
@@ -139,30 +101,16 @@
 		}
 	}
 #endif
-	return result;
-}
-
-QCGC_STATIC void bump_allocator_renew_block(void) {
-#if CHECKED
+	// Add remaining memory to fit allocator
 	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
-		assert(qcgc_arena_get_blocktype(
-					qcgc_arena_addr( qcgc_allocator_state.bump_state.bump_ptr),
-					qcgc_arena_cell_index(
-						qcgc_allocator_state.bump_state.bump_ptr))
-				== BLOCK_FREE);
-		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
-				i++) {
-			assert(qcgc_arena_get_blocktype(qcgc_arena_addr(
-							qcgc_allocator_state.bump_state.bump_ptr + i),
-						qcgc_arena_cell_index(
-							qcgc_allocator_state.bump_state.bump_ptr + i))
-					== BLOCK_EXTENT);
-		}
+		qcgc_arena_set_blocktype(
+				qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr),
+				qcgc_arena_cell_index(
+					qcgc_allocator_state.bump_state.bump_ptr),
+				BLOCK_FREE);
+		qcgc_fit_allocator_add(qcgc_allocator_state.bump_state.bump_ptr,
+				qcgc_allocator_state.bump_state.remaining_cells);
 	}
-#endif
-	// Add remaining memory to fit allocator
-	qcgc_fit_allocator_add(qcgc_allocator_state.bump_state.bump_ptr,
-			qcgc_allocator_state.bump_state.remaining_cells);
 
 	// Try finding some huge block from fit allocator
 	exp_free_list_t *free_list = qcgc_allocator_state.fit_state.
@@ -215,11 +163,6 @@
 	qcgc_allocator_state.bump_state.remaining_cells = cells;
 }
 
-QCGC_STATIC void bump_allocator_advance(size_t cells) {
-	qcgc_allocator_state.bump_state.bump_ptr += cells;
-	qcgc_allocator_state.bump_state.remaining_cells -= cells;
-}
-
 object_t *qcgc_fit_allocate(size_t bytes) {
 #if CHECKED
 	//free_list_consistency_check();
@@ -365,7 +308,6 @@
 						qcgc_allocator_state.fit_state.large_free_list[index],
 						0);
 
-			qcgc_arena_mark_allocated(item.ptr, cells);
 			qcgc_arena_set_blocktype(qcgc_arena_addr(item.ptr),
 					qcgc_arena_cell_index(item.ptr), BLOCK_WHITE);
 			if (item.size - cells > 0) {
@@ -379,10 +321,6 @@
 	return NULL;
 }
 
-QCGC_STATIC size_t bytes_to_cells(size_t bytes) {
-	return (bytes + sizeof(cell_t) - 1) / sizeof(cell_t);
-}
-
 QCGC_STATIC bool is_small(size_t cells) {
 	return cells <= QCGC_SMALL_FREE_LISTS;
 }
diff --git a/rpython/translator/c/src/qcgc/src/allocator.h b/rpython/translator/c/src/qcgc/src/allocator.h
--- a/rpython/translator/c/src/qcgc/src/allocator.h
+++ b/rpython/translator/c/src/qcgc/src/allocator.h
@@ -2,6 +2,8 @@
 
 #include "../qcgc.h"
 
+#include <string.h>
+
 #include "arena.h"
 #include "bag.h"
 
@@ -69,15 +71,6 @@
 object_t *qcgc_fit_allocate(size_t bytes);
 
 /**
- * Allocate new memory region using bump allocator
- *
- * @param	bytes	Desired size of the memory region in bytes
- * @return	Pointer to memory large enough to hold size bytes, NULL in case of
- *			errors, already zero initialized if QCGC_INIT_ZERO is set
- */
-object_t *qcgc_bump_allocate(size_t bytes);
-
-/**
  * Allocate new memory region using huge block allocator
  *
  * @param	bytes	Desired size of the memory region in bytes
@@ -99,3 +92,42 @@
  * @param	cells	Size of memory region in cells
  */
 void qcgc_fit_allocator_add(cell_t *ptr, size_t cells);
+
+QCGC_STATIC QCGC_INLINE size_t bytes_to_cells(size_t bytes) {
+	return (bytes + sizeof(cell_t) - 1) / sizeof(cell_t);
+}
+
+/**
+ * Find a new block for the bump allocator
+ */
+void qcgc_bump_allocator_renew_block(void);
+
+/**
+ * Allocate new memory region using bump allocator.
+ * Bump allocator must have enough space for desired bytes
+ * (client is responsible, use qcgc_bump_allocator_renew_block)
+ *
+ * @param	bytes	Desired size of the memory region in bytes
+ * @return	Pointer to memory large enough to hold size bytes, NULL in case of
+ *			errors, already zero initialized if QCGC_INIT_ZERO is set
+ */
+QCGC_STATIC QCGC_INLINE object_t *qcgc_bump_allocate(size_t bytes) {
+	size_t cells = bytes_to_cells(bytes);
+
+	cell_t *mem = qcgc_allocator_state.bump_state.bump_ptr;
+
+	qcgc_arena_set_blocktype(qcgc_arena_addr(mem), qcgc_arena_cell_index(mem),
+			BLOCK_WHITE);
+
+	qcgc_allocator_state.bump_state.bump_ptr += cells;
+	qcgc_allocator_state.bump_state.remaining_cells -= cells;
+
+	object_t *result = (object_t *) mem;
+
+#if QCGC_INIT_ZERO
+	memset(result, 0, cells * sizeof(cell_t));
+#endif
+
+	result->flags = QCGC_GRAY_FLAG;
+	return result;
+}
diff --git a/rpython/translator/c/src/qcgc/src/collector.c b/rpython/translator/c/src/qcgc/src/collector.c
--- a/rpython/translator/c/src/qcgc/src/collector.c
+++ b/rpython/translator/c/src/qcgc/src/collector.c
@@ -20,6 +20,7 @@
 
 		while (qcgc_state.gp_gray_stack->index > 0) {
 			object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
+			qcgc_state.gray_stack_size--;
 			qcgc_state.gp_gray_stack = qcgc_gray_stack_pop(
 					qcgc_state.gp_gray_stack);
 			qcgc_pop_object(top);
@@ -31,6 +32,7 @@
 
 			while (arena->gray_stack->index > 0) {
 				object_t *top = qcgc_gray_stack_top(arena->gray_stack);
+				qcgc_state.gray_stack_size--;
 				arena->gray_stack = qcgc_gray_stack_pop(arena->gray_stack);
 				qcgc_pop_object(top);
 			}
@@ -53,6 +55,7 @@
 
 	while (to_process > 0 && qcgc_state.gp_gray_stack->index > 0) {
 		object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
+		qcgc_state.gray_stack_size--;
 		qcgc_state.gp_gray_stack = qcgc_gray_stack_pop(
 				qcgc_state.gp_gray_stack);
 		qcgc_pop_object(top);
@@ -66,6 +69,7 @@
 
 		while (to_process > 0 && arena->gray_stack->index > 0) {
 			object_t *top = qcgc_gray_stack_top(arena->gray_stack);
+			qcgc_state.gray_stack_size--;
 			arena->gray_stack = qcgc_gray_stack_pop(arena->gray_stack);
 			qcgc_pop_object(top);
 			to_process--;
@@ -99,6 +103,7 @@
 		// because of the write barrier
 		size_t count = qcgc_state.prebuilt_objects->count;
 		for (size_t i = 0; i < count; i++) {
+			qcgc_state.gray_stack_size++;
 			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
 					qcgc_state.gp_gray_stack,
 					qcgc_state.prebuilt_objects->items[i]);
@@ -157,6 +162,7 @@
 			if (qcgc_hbtable_mark(object)) {
 				// Did mark it / was white before
 				object->flags |= QCGC_GRAY_FLAG;
+				qcgc_state.gray_stack_size++;
 				qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
 						qcgc_state.gp_gray_stack, object);
 			}
@@ -169,6 +175,7 @@
 		if (qcgc_arena_get_blocktype(arena, index) == BLOCK_WHITE) {
 			object->flags |= QCGC_GRAY_FLAG;
 			qcgc_arena_set_blocktype(arena, index, BLOCK_BLACK);
+			qcgc_state.gray_stack_size++;
 			arena->gray_stack = qcgc_gray_stack_push(arena->gray_stack, object);
 		}
 	}
diff --git a/rpython/translator/c/src/qcgc/src/core.c b/rpython/translator/c/src/qcgc/src/core.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/core.c
+++ /dev/null
@@ -1,184 +0,0 @@
-#include "core.h"
-
-#include <assert.h>
-#include <stdio.h>
-#include <sys/mman.h>
-
-#include "allocator.h"
-#include "collector.h"
-#include "event_logger.h"
-#include "gc_state.h"
-#include "hugeblocktable.h"
-#include "signal_handler.h"
-
-#define env_or_fallback(var, env_name, fallback) do {			\
-	char *env_val = getenv(env_name);							\
-	if (env_val != NULL) {										\
-		if (1 != sscanf(env_val, "%zu", &var)) {				\
-			var = fallback;										\
-		}														\
-	} else {													\
-		var = fallback;											\
-	}															\
-} while(0)
-
-QCGC_STATIC QCGC_INLINE void _initialize_shadowstack(void);
-QCGC_STATIC QCGC_INLINE void _destroy_shadowstack(void);
-
-/******************************************************************************/
-
-void qcgc_initialize(void) {
-	_initialize_shadowstack();
-	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); // XXX
-	qcgc_state.weakrefs = qcgc_weakref_bag_create(16); // XXX
-	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
-	qcgc_state.gray_stack_size = 0;
-	qcgc_state.phase = GC_PAUSE;
-	qcgc_state.bytes_since_collection = 0;
-	qcgc_state.bytes_since_incmark = 0;
-	qcgc_state.free_cells = 0;
-	qcgc_state.largest_free_block = 0;
-	qcgc_allocator_initialize();
-	qcgc_hbtable_initialize();
-	qcgc_event_logger_initialize();
-
-	env_or_fallback(qcgc_state.major_collection_threshold, "QCGC_MAJOR_COLLECTION",
-			QCGC_MAJOR_COLLECTION_THRESHOLD);
-	env_or_fallback(qcgc_state.incmark_threshold, "QCGC_INCMARK", QCGC_INCMARK_THRESHOLD);
-
-	setup_signal_handler();
-}
-
-void qcgc_destroy(void) {
-	qcgc_event_logger_destroy();
-	qcgc_hbtable_destroy();
-	qcgc_allocator_destroy();
-	_destroy_shadowstack();
-	free(qcgc_state.prebuilt_objects);
-	free(qcgc_state.weakrefs);
-	free(qcgc_state.gp_gray_stack);
-}
-
-object_t *qcgc_allocate(size_t size) {
-#if LOG_ALLOCATION
-	qcgc_event_logger_log(EVENT_ALLOCATE_START, sizeof(size_t),
-			(uint8_t *) &size);
-#endif
-	object_t *result;
-
-	if (qcgc_state.bytes_since_collection >
-			qcgc_state.major_collection_threshold) {
-		qcgc_collect();
-	}
-	if (qcgc_state.bytes_since_incmark > qcgc_state.incmark_threshold) {
-		qcgc_mark(true);
-	}
-
-	if (size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP) {
-		// Use bump / fit allocator
-		if (qcgc_allocator_state.use_bump_allocator) {
-			result = qcgc_bump_allocate(size);
-		} else {
-			result = qcgc_fit_allocate(size);
-
-			// Fallback to bump allocator
-			if (result == NULL) {
-				result = qcgc_bump_allocate(size);
-			}
-		}
-	} else {
-		// Use huge block allocator
-		result = qcgc_large_allocate(size);
-	}
-
-	// XXX: Should we use cells instead of bytes?
-	qcgc_state.bytes_since_collection += size;
-	qcgc_state.bytes_since_incmark += size;
-
-
-#if LOG_ALLOCATION
-	qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
-			(uint8_t *) &result);
-#endif
-	return result;
-}
-
-void qcgc_collect(void) {
-	qcgc_mark(false);
-	qcgc_sweep();
-	qcgc_state.bytes_since_collection = 0;
-}
-
-void qcgc_write(object_t *object) {
-#if CHECKED
-	assert(object != NULL);
-#endif
-	if ((object->flags & QCGC_GRAY_FLAG) != 0) {
-		// Already gray, skip
-		return;
-	}
-	object->flags |= QCGC_GRAY_FLAG;
-
-	// Register prebuilt object if necessary
-	if (((object->flags & QCGC_PREBUILT_OBJECT) != 0) &&
-			((object->flags & QCGC_PREBUILT_REGISTERED) == 0)) {
-		object->flags |= QCGC_PREBUILT_REGISTERED;
-		qcgc_state.prebuilt_objects = qcgc_shadow_stack_push(
-				qcgc_state.prebuilt_objects, object);
-	}
-
-	if (qcgc_state.phase == GC_PAUSE) {
-		return; // We are done
-	}
-
-	// Triggered barrier, we must not collect now
-	qcgc_state.phase = GC_MARK;
-
-	// Test reachability of object and push if neccessary
-	if ((object->flags & QCGC_PREBUILT_OBJECT) != 0) {
-		// NOTE: No mark test here, as prebuilt objects are always reachable
-		// Push prebuilt object to general purpose gray stack
-		qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
-				qcgc_state.gp_gray_stack, object);
-	} else if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
-		if (qcgc_hbtable_is_marked(object)) {
-			// Push huge block to general purpose gray stack
-			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
-					qcgc_state.gp_gray_stack, object);
-		}
-	} else {
-		if (qcgc_arena_get_blocktype((cell_t *) object) == BLOCK_BLACK) {
-			// This was black before, push it to gray stack again
-			arena_t *arena = qcgc_arena_addr((cell_t *) object);
-			arena->gray_stack = qcgc_gray_stack_push(
-					arena->gray_stack, object);
-		}
-	}
-}
-
-/******************************************************************************/
-
-QCGC_STATIC QCGC_INLINE void *_trap_page_addr(object_t **shadow_stack) {
-	object_t **shadow_stack_end = shadow_stack + QCGC_SHADOWSTACK_SIZE;
-	char *in_trap_page = (((char *)shadow_stack_end) + 4095);
-	void *rounded_trap_page = (void *)(((uintptr_t)in_trap_page) & (~4095));
-	return rounded_trap_page;
-}
-
-QCGC_STATIC QCGC_INLINE void _initialize_shadowstack(void) {
-	size_t stack_size = QCGC_SHADOWSTACK_SIZE * sizeof(object_t *);
-	// allocate stack + size for alignement + trap page
-	object_t **stack = (object_t **) malloc(stack_size + 8192);
-	assert(stack != NULL);
-	mprotect(_trap_page_addr(stack), 4096, PROT_NONE);
-
-	qcgc_state.shadow_stack = stack;
-	qcgc_state.shadow_stack_base = stack;
-}
-
-QCGC_STATIC void _destroy_shadowstack(void) {
-	mprotect(_trap_page_addr(qcgc_state.shadow_stack_base), 4096, PROT_READ |
-				PROT_WRITE);
-
-	free(qcgc_state.shadow_stack_base);
-}
diff --git a/rpython/translator/c/src/qcgc/src/core.h b/rpython/translator/c/src/qcgc/src/core.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/core.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#pragma once
-
-#include "config.h"
-
-#include <stddef.h>
-
-#include "object.h"
-
-
-void qcgc_initialize(void);
-void qcgc_destroy(void);
-object_t *qcgc_allocate(size_t size);
-void qcgc_collect(void);
-void qcgc_write(object_t *object);
diff --git a/rpython/translator/c/src/qcgc/src/gray_stack.c b/rpython/translator/c/src/qcgc/src/gray_stack.c
--- a/rpython/translator/c/src/qcgc/src/gray_stack.c
+++ b/rpython/translator/c/src/qcgc/src/gray_stack.c
@@ -1,13 +1,8 @@
 #include "gray_stack.h"
 
-#include <assert.h>
 #include <stdlib.h>
 
-#include "gc_state.h"
-
 QCGC_STATIC size_t gray_stack_size(size_t size);
-QCGC_STATIC gray_stack_t *gray_stack_grow(gray_stack_t *stack);
-QCGC_STATIC gray_stack_t *gray_stack_shrink(gray_stack_t *stack);
 
 gray_stack_t *qcgc_gray_stack_create(size_t size) {
 	gray_stack_t *result = (gray_stack_t *) malloc(gray_stack_size(size));
@@ -16,44 +11,17 @@
 	return result;
 }
 
-gray_stack_t *qcgc_gray_stack_push(gray_stack_t *stack, object_t *item) {
-	if (stack->size == stack->index) {
-		stack = gray_stack_grow(stack);
-	}
-	stack->items[stack->index] = item;
-	stack->index++;
-	qcgc_state.gray_stack_size++;
-	return stack;
-}
-
-object_t *qcgc_gray_stack_top(gray_stack_t *stack) {
-#if CHECKED
-	assert(stack->index != 0);
-#endif
-	return stack->items[stack->index - 1];
-}
-
-gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack) {
-	// TODO: Add lower bound for size (config.h)
-	if (stack->index < stack->size / 4) {
-		stack = gray_stack_shrink(stack);
-	}
-	stack->index--;
-	qcgc_state.gray_stack_size--;
-	return stack;
-}
-
 QCGC_STATIC size_t gray_stack_size(size_t size) {
 	return (sizeof(gray_stack_t) + size * sizeof(object_t *));
 }
 
-QCGC_STATIC gray_stack_t *gray_stack_grow(gray_stack_t *stack) {
+gray_stack_t *gray_stack_grow(gray_stack_t *stack) {
 	stack = (gray_stack_t *) realloc(stack, gray_stack_size(stack->size * 2));
 	stack->size *= 2;
 	return stack;
 }
 
-QCGC_STATIC gray_stack_t *gray_stack_shrink(gray_stack_t *stack) {
+gray_stack_t *gray_stack_shrink(gray_stack_t *stack) {
 	stack = (gray_stack_t *) realloc(stack, gray_stack_size(stack->size / 2));
 	stack->size /= 2;
 	return stack;
diff --git a/rpython/translator/c/src/qcgc/src/gray_stack.h b/rpython/translator/c/src/qcgc/src/gray_stack.h
--- a/rpython/translator/c/src/qcgc/src/gray_stack.h
+++ b/rpython/translator/c/src/qcgc/src/gray_stack.h
@@ -12,9 +12,35 @@
 gray_stack_t *qcgc_gray_stack_create(size_t size);
 
 __attribute__ ((warn_unused_result))
-gray_stack_t *qcgc_gray_stack_push(gray_stack_t *stack, object_t *item);
-
-object_t *qcgc_gray_stack_top(gray_stack_t *stack);
+gray_stack_t *gray_stack_grow(gray_stack_t *stack);
 
 __attribute__ ((warn_unused_result))
-gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack);
+gray_stack_t *gray_stack_shrink(gray_stack_t *stack);
+
+__attribute__ ((warn_unused_result))
+QCGC_STATIC QCGC_INLINE gray_stack_t *qcgc_gray_stack_push(
+		gray_stack_t *stack, object_t *item) {
+	if (stack->size == stack->index) {
+		stack = gray_stack_grow(stack);
+	}
+	stack->items[stack->index] = item;
+	stack->index++;
+	return stack;
+}
+
+QCGC_STATIC QCGC_INLINE object_t *qcgc_gray_stack_top(gray_stack_t *stack) {
+#if CHECKED
+	assert(stack->index != 0);
+#endif
+	return stack->items[stack->index - 1];
+}
+
+__attribute__ ((warn_unused_result))
+QCGC_STATIC QCGC_INLINE gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack) {
+	// TODO: Add lower bound for size (config.h)
+	if (stack->index < stack->size / 4) {
+		stack = gray_stack_shrink(stack);
+	}
+	stack->index--;
+	return stack;
+}

From pypy.commits at gmail.com  Thu Sep 15 17:10:55 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Thu, 15 Sep 2016 14:10:55 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: QCGC update
Message-ID: <57db0e5f.c5cb1c0a.eec2c.8525@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87127:a870d33012f1
Date: 2016-09-15 18:45 +0200
http://bitbucket.org/pypy/pypy/changeset/a870d33012f1/

Log:	QCGC update

diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -10,8 +10,8 @@
  */
 #define EVENT_LOG 1							// Enable event log
 #define LOGFILE "./qcgc_events.log"			// Default logfile
-#define LOG_ALLOCATION 0					// Enable allocation log (warning:
-											// significant performance impact)
+#define LOG_ALLOCATION 1					// Enable allocation log
+#define LOG_DUMP_FREELIST_STATS 1			// Dump freelist stats
 
 #define QCGC_SHADOWSTACK_SIZE 163840		// Total shadowstack size
 #define QCGC_ARENA_BAG_INIT_SIZE 16			// Initial size of the arena bag
@@ -31,7 +31,7 @@
 /**
  * Auto Mark/Collect
  */
-#define QCGC_INCMARK_THRESHOLD (1<<QCGC_ARENA_SIZE_EXP)
+#define QCGC_INCMARK_THRESHOLD (1<<(QCGC_ARENA_SIZE_EXP-4))
 #define QCGC_INCMARK_TO_SWEEP 5
 
 /**
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -34,7 +34,8 @@
 	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
 	qcgc_state.gray_stack_size = 0;
 	qcgc_state.phase = GC_PAUSE;
-	qcgc_state.bytes_since_incmark = 0;
+	qcgc_state.cells_since_incmark = 0;
+	qcgc_state.cells_since_collect = 0;
 	qcgc_state.incmark_since_sweep = 0;
 	qcgc_state.free_cells = 0;
 	qcgc_state.largest_free_block = 0;
@@ -62,12 +63,13 @@
 
 object_t *qcgc_allocate(size_t size) {
 #if LOG_ALLOCATION
-	qcgc_event_logger_log(EVENT_ALLOCATE_START, sizeof(size_t),
-			(uint8_t *) &size);
+	size_t cells = bytes_to_cells(size);
+	qcgc_event_logger_log(EVENT_ALLOCATE, sizeof(size_t),
+			(uint8_t *) &cells);
 #endif
 	object_t *result;
 
-	if (UNLIKELY(qcgc_state.bytes_since_incmark >
+	if (UNLIKELY(qcgc_state.cells_since_incmark >
 				qcgc_state.incmark_threshold)) {
 		if (qcgc_state.incmark_since_sweep == qcgc_state.incmark_to_sweep) {
 			qcgc_collect();
@@ -75,13 +77,11 @@
 			qcgc_incmark();
 			qcgc_state.incmark_since_sweep++;
 		}
-		
 	}
 
 	if (LIKELY(size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP)) {
 		// Use bump / fit allocator
-		//if (qcgc_allocator_state.use_bump_allocator) {
-		if (false) {
+		if (qcgc_allocator_state.use_bump_allocator) {
 			result = bump_allocate(size);
 		} else {
 			result = qcgc_fit_allocate(size);
@@ -91,19 +91,13 @@
 				result = bump_allocate(size);
 			}
 		}
+		qcgc_state.free_cells -= bytes_to_cells(size);
 	} else {
 		// Use huge block allocator
 		result = qcgc_large_allocate(size);
 	}
-
-	// XXX: Should we use cells instead of bytes?
-	qcgc_state.bytes_since_incmark += size;
-
-
-#if LOG_ALLOCATION
-	qcgc_event_logger_log(EVENT_ALLOCATE_DONE, sizeof(object_t *),
-			(uint8_t *) &result);
-#endif
+	qcgc_state.cells_since_incmark += bytes_to_cells(size);
+	qcgc_state.cells_since_collect += bytes_to_cells(size);
 	return result;
 }
 
diff --git a/rpython/translator/c/src/qcgc/src/allocator.c b/rpython/translator/c/src/qcgc/src/allocator.c
--- a/rpython/translator/c/src/qcgc/src/allocator.c
+++ b/rpython/translator/c/src/qcgc/src/allocator.c
@@ -2,9 +2,7 @@
 
 #include <assert.h>
 #include <stdbool.h>
-#include <stdlib.h>
-
-#include "hugeblocktable.h"
+#include "gc_state.h"
 
 QCGC_STATIC QCGC_INLINE void bump_allocator_assign(cell_t *ptr, size_t cells);
 
@@ -65,25 +63,6 @@
 	free(qcgc_allocator_state.free_arenas);
 }
 
-void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
-#if CHECKED
-	assert(cells > 0);
-#endif
-	if (is_small(cells)) {
-		size_t index = small_index(cells);
-		qcgc_allocator_state.fit_state.small_free_list[index] =
-			qcgc_linear_free_list_add(
-					qcgc_allocator_state.fit_state.small_free_list[index],
-					ptr);
-	} else {
-		size_t index = large_index(cells);
-		qcgc_allocator_state.fit_state.large_free_list[index] =
-			qcgc_exp_free_list_add(
-					qcgc_allocator_state.fit_state.large_free_list[index],
-					(struct exp_free_list_item_s) {ptr, cells});
-	}
-}
-
 /*******************************************************************************
  * Bump Allocator                                                              *
  ******************************************************************************/
@@ -124,6 +103,8 @@
 	} else {
 		// Grab a new arena
 		arena_t *arena = qcgc_arena_create();
+		qcgc_state.free_cells += QCGC_ARENA_CELLS_COUNT -
+			QCGC_ARENA_FIRST_CELL_INDEX;
 		bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
 				QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
 		qcgc_allocator_state.arenas =
@@ -163,6 +144,10 @@
 	qcgc_allocator_state.bump_state.remaining_cells = cells;
 }
 
+/*******************************************************************************
+ * Fit Allocator                                                               *
+ ******************************************************************************/
+
 object_t *qcgc_fit_allocate(size_t bytes) {
 #if CHECKED
 	//free_list_consistency_check();
@@ -192,23 +177,6 @@
 	return result;
 }
 
-/**
- * Constraints:
- * - Zero initialized
- * - Aligned to arena size
- * - Multiple of arena size
- * - No header, metadata stored in hash-map
- */
-object_t *qcgc_large_allocate(size_t bytes) {
-	object_t *result = aligned_alloc(QCGC_ARENA_SIZE, bytes);
-#if QCGC_INIT_ZERO
-	memset(result, 0, bytes);
-#endif
-	qcgc_hbtable_insert(result);
-	result->flags = QCGC_GRAY_FLAG;
-	return result;
-}
-
 void qcgc_fit_allocator_empty_lists(void) {
 	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
 		qcgc_allocator_state.fit_state.small_free_list[i]->count = 0;
@@ -219,6 +187,25 @@
 	}
 }
 
+void qcgc_fit_allocator_add(cell_t *ptr, size_t cells) {
+#if CHECKED
+	assert(cells > 0);
+#endif
+	if (is_small(cells)) {
+		size_t index = small_index(cells);
+		qcgc_allocator_state.fit_state.small_free_list[index] =
+			qcgc_linear_free_list_add(
+					qcgc_allocator_state.fit_state.small_free_list[index],
+					ptr);
+	} else {
+		size_t index = large_index(cells);
+		qcgc_allocator_state.fit_state.large_free_list[index] =
+			qcgc_exp_free_list_add(
+					qcgc_allocator_state.fit_state.large_free_list[index],
+					(struct exp_free_list_item_s) {ptr, cells});
+	}
+}
+
 QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells) {
 #if CHECKED
 	assert(small_index_to_cells(index) >= cells);
@@ -321,18 +308,18 @@
 	return NULL;
 }
 
-QCGC_STATIC bool is_small(size_t cells) {
+QCGC_STATIC QCGC_INLINE bool is_small(size_t cells) {
 	return cells <= QCGC_SMALL_FREE_LISTS;
 }
 
-QCGC_STATIC size_t small_index(size_t cells) {
+QCGC_STATIC QCGC_INLINE size_t small_index(size_t cells) {
 #if CHECKED
 	assert(is_small(cells));
 #endif
 	return cells - 1;
 }
 
-QCGC_STATIC size_t large_index(size_t cells) {
+QCGC_STATIC QCGC_INLINE size_t large_index(size_t cells) {
 #if CHECKED
 	assert(!is_small(cells));
 #endif
@@ -341,10 +328,11 @@
 	cells = cells >> QCGC_LARGE_FREE_LIST_FIRST_EXP;
 
 	// calculates floor(log(cells))
-	return MIN((8 * sizeof(unsigned long)) - __builtin_clzl(cells) - 1, QCGC_LARGE_FREE_LISTS - 1);
+	return MIN((8 * sizeof(unsigned long)) - __builtin_clzl(cells) - 1,
+			QCGC_LARGE_FREE_LISTS - 1);
 }
 
-QCGC_STATIC size_t small_index_to_cells(size_t index) {
+QCGC_STATIC QCGC_INLINE size_t small_index_to_cells(size_t index) {
 #if CHECKED
 	assert(index < QCGC_SMALL_FREE_LISTS);
 #endif
diff --git a/rpython/translator/c/src/qcgc/src/allocator.h b/rpython/translator/c/src/qcgc/src/allocator.h
--- a/rpython/translator/c/src/qcgc/src/allocator.h
+++ b/rpython/translator/c/src/qcgc/src/allocator.h
@@ -3,9 +3,11 @@
 #include "../qcgc.h"
 
 #include <string.h>
+#include <stdlib.h>
 
 #include "arena.h"
 #include "bag.h"
+#include "hugeblocktable.h"
 
 /**
  * Free lists:
@@ -71,15 +73,6 @@
 object_t *qcgc_fit_allocate(size_t bytes);
 
 /**
- * Allocate new memory region using huge block allocator
- *
- * @param	bytes	Desired size of the memory region in bytes
- * @return	Pointer to memory large enough to hold size bytes, NULL in case of
- *			errors, already zero initialized if QCGC_INIT_ZERO is set
- */
-object_t *qcgc_large_allocate(size_t bytes);
-
-/**
  * Empty all free lists (used before sweep)
  */
 void qcgc_fit_allocator_empty_lists(void);
@@ -131,3 +124,20 @@
 	result->flags = QCGC_GRAY_FLAG;
 	return result;
 }
+
+/**
+ * Allocate new memory region using huge block allocator
+ *
+ * @param	bytes	Desired size of the memory region in bytes
+ * @return	Pointer to memory large enough to hold size bytes, NULL in case of
+ *			errors, already zero initialized if QCGC_INIT_ZERO is set
+ */
+QCGC_STATIC QCGC_INLINE object_t *qcgc_large_allocate(size_t bytes) {
+	object_t *result = aligned_alloc(QCGC_ARENA_SIZE, bytes);
+#if QCGC_INIT_ZERO
+	memset(result, 0, bytes);
+#endif
+	qcgc_hbtable_insert(result);
+	result->flags = QCGC_GRAY_FLAG;
+	return result;
+}
diff --git a/rpython/translator/c/src/qcgc/src/collector.c b/rpython/translator/c/src/qcgc/src/collector.c
--- a/rpython/translator/c/src/qcgc/src/collector.c
+++ b/rpython/translator/c/src/qcgc/src/collector.c
@@ -14,7 +14,7 @@
 
 void qcgc_mark(void) {
 	mark_setup(false);
-	
+
 	while (qcgc_state.gray_stack_size > 0) {
 		// General purpose gray stack (prebuilt objects and huge blocks)
 
@@ -94,7 +94,7 @@
 				(uint8_t *) &log_info);
 	}
 
-	qcgc_state.bytes_since_incmark = 0;
+	qcgc_state.cells_since_incmark = 0;
 
 	if (qcgc_state.phase == GC_PAUSE) {
 
@@ -186,10 +186,16 @@
 	assert(qcgc_state.phase == GC_COLLECT);
 #endif
 	{
-		unsigned long arena_count;
-		arena_count = qcgc_allocator_state.arenas->count;
-		qcgc_event_logger_log(EVENT_SWEEP_START, sizeof(arena_count),
-				(uint8_t *) &arena_count);
+		struct log_info_s {
+			size_t arenas;
+			size_t free_cells;
+		};
+		struct log_info_s log_info = {
+			qcgc_allocator_state.arenas->count,
+			qcgc_state.free_cells,
+		};
+		qcgc_event_logger_log(EVENT_SWEEP_START, sizeof(struct log_info_s),
+				(uint8_t *) &log_info);
 	}
 
 	qcgc_hbtable_sweep();
@@ -218,22 +224,75 @@
 	// Determine whether fragmentation is too high
 	// Fragmenation = 1 - (largest block / total free space)
 	// Use bump allocator when fragmentation < 50%
+#if CHECKED
+	size_t free_cells = 0;
+	size_t largest_free_block = 0;
+		for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
+			if (qcgc_allocator_state.fit_state.small_free_list[i]->count > 0) {
+				largest_free_block = i + 1;
+			}
+			free_cells += qcgc_allocator_state.fit_state.small_free_list[i]
+				->count * (i + 1);
+		}
+		for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
+			for (size_t j = 0; j < qcgc_allocator_state.fit_state.
+					large_free_list[i]->count; j++) {
+				free_cells += qcgc_allocator_state.fit_state.large_free_list[i]
+					->items[j].size;
+				largest_free_block = MAX(largest_free_block,
+					qcgc_allocator_state.fit_state.large_free_list[i]
+					->items[j].size);
+			}
+		}
+	assert(free_cells == qcgc_state.free_cells);
+	assert(largest_free_block == qcgc_state.largest_free_block);
+	assert(free_cells <= qcgc_allocator_state.arenas->count * (QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX));
+	assert(qcgc_allocator_state.arenas->count *
+			(QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX) >=
+			free_cells);
+#endif
 	qcgc_allocator_state.use_bump_allocator = qcgc_state.free_cells <
 		2 * qcgc_state.largest_free_block;
 
 	update_weakrefs();
 
+	qcgc_state.free_cells += qcgc_allocator_state.bump_state.remaining_cells;
 	{
 		struct log_info_s {
+			size_t arenas;
 			size_t free_cells;
 			size_t largest_free_block;
 		};
 		struct log_info_s log_info = {
+			qcgc_allocator_state.arenas->count,
 			qcgc_state.free_cells,
 			qcgc_state.largest_free_block
 		};
 		qcgc_event_logger_log(EVENT_SWEEP_DONE, sizeof(struct log_info_s),
 				(uint8_t *) &log_info);
 	}
+#if LOG_DUMP_FREELIST_STATS
+	{
+		struct log_info_s {
+			size_t class;
+			size_t items;
+		};
+		struct log_info_s log_info;
+
+		for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
+			log_info = (struct log_info_s){ i + 1,
+				qcgc_allocator_state.fit_state.small_free_list[i]->count};
+			qcgc_event_logger_log(EVENT_FREELIST_DUMP,
+					sizeof(struct log_info_s), (uint8_t *) &log_info);
+		}
+		for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
+			log_info = (struct log_info_s){
+				1<<(QCGC_LARGE_FREE_LIST_FIRST_EXP + i),
+				qcgc_allocator_state.fit_state.large_free_list[i]->count};
+			qcgc_event_logger_log(EVENT_FREELIST_DUMP,
+					sizeof(struct log_info_s), (uint8_t *) &log_info);
+		}
+	}
+#endif
 }
 
diff --git a/rpython/translator/c/src/qcgc/src/event_logger.h b/rpython/translator/c/src/qcgc/src/event_logger.h
--- a/rpython/translator/c/src/qcgc/src/event_logger.h
+++ b/rpython/translator/c/src/qcgc/src/event_logger.h
@@ -15,13 +15,14 @@
 	EVENT_SWEEP_START,
 	EVENT_SWEEP_DONE,
 
-	EVENT_ALLOCATE_START,
-	EVENT_ALLOCATE_DONE,		// = 5
+	EVENT_ALLOCATE,
 
-	EVENT_NEW_ARENA,
+	EVENT_NEW_ARENA,			// = 5
 
 	EVENT_MARK_START,
 	EVENT_MARK_DONE,
+
+	EVENT_FREELIST_DUMP,
 };
 
 /**
diff --git a/rpython/translator/c/src/qcgc/src/gc_state.h b/rpython/translator/c/src/qcgc/src/gc_state.h
--- a/rpython/translator/c/src/qcgc/src/gc_state.h
+++ b/rpython/translator/c/src/qcgc/src/gc_state.h
@@ -33,7 +33,8 @@
 	size_t gray_stack_size;
 	gc_phase_t phase;
 
-	size_t bytes_since_incmark;
+	size_t cells_since_incmark;
+	size_t cells_since_collect;
 	size_t incmark_since_sweep;
 	size_t incmark_threshold;
 	size_t incmark_to_sweep;

From pypy.commits at gmail.com  Thu Sep 15 17:10:58 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Thu, 15 Sep 2016 14:10:58 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Probably last optimization import
 from qcgc repo
Message-ID: <57db0e62.43e61c0a.64418.9c20@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87128:4fa6b4f42a48
Date: 2016-09-15 23:10 +0200
http://bitbucket.org/pypy/pypy/changeset/4fa6b4f42a48/

Log:	Probably last optimization import from qcgc repo

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -899,8 +899,8 @@
             separate_module_sources = [separate_source],  # XXX
             separate_module_files = [os.path.join(library_dir, f) for f in
                 ['qcgc.c', 'src/allocator.c', 'src/arena.c', 'src/bag.c',
-                 'src/collector.c', 'src/event_logger.c', 'src/gray_stack.c',
-                 'src/hugeblocktable.c', 'src/shadow_stack.c',
+                 'src/collector.c', 'src/event_logger.c',
+                 'src/hugeblocktable.c', 'src/object_stack.c',
                  'src/signal_handler.c', 'src/weakref.c']],
             )
     return eci
diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -10,7 +10,7 @@
  */
 #define EVENT_LOG 1							// Enable event log
 #define LOGFILE "./qcgc_events.log"			// Default logfile
-#define LOG_ALLOCATION 1					// Enable allocation log
+#define LOG_ALLOCATION 0					// Enable allocation log
 #define LOG_DUMP_FREELIST_STATS 1			// Dump freelist stats
 
 #define QCGC_SHADOWSTACK_SIZE 163840		// Total shadowstack size
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -25,17 +25,14 @@
 QCGC_STATIC QCGC_INLINE void initialize_shadowstack(void);
 QCGC_STATIC QCGC_INLINE void destroy_shadowstack(void);
 
-QCGC_STATIC object_t *bump_allocate(size_t size);
-
 void qcgc_initialize(void) {
 	initialize_shadowstack();
-	qcgc_state.prebuilt_objects = qcgc_shadow_stack_create(16); // XXX
+	qcgc_state.prebuilt_objects = qcgc_object_stack_create(16); // XXX
 	qcgc_state.weakrefs = qcgc_weakref_bag_create(16); // XXX
-	qcgc_state.gp_gray_stack = qcgc_gray_stack_create(16); // XXX
+	qcgc_state.gp_gray_stack = qcgc_object_stack_create(16); // XXX
 	qcgc_state.gray_stack_size = 0;
 	qcgc_state.phase = GC_PAUSE;
 	qcgc_state.cells_since_incmark = 0;
-	qcgc_state.cells_since_collect = 0;
 	qcgc_state.incmark_since_sweep = 0;
 	qcgc_state.free_cells = 0;
 	qcgc_state.largest_free_block = 0;
@@ -61,12 +58,103 @@
 	free(qcgc_state.gp_gray_stack);
 }
 
-object_t *qcgc_allocate(size_t size) {
-#if LOG_ALLOCATION
+object_t *_qcgc_allocate_large(size_t size) {
+#if CHECKED
+	assert(size >= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP);
+#endif
+	if (UNLIKELY(qcgc_state.cells_since_incmark >
+				qcgc_state.incmark_threshold)) {
+		if (qcgc_state.incmark_since_sweep == qcgc_state.incmark_to_sweep) {
+			qcgc_collect();
+		} else {
+			qcgc_incmark();
+			qcgc_state.incmark_since_sweep++;
+		}
+	}
+
+	// FIXME: alligned_alloc requires size to be a multiple of the alignment
+	object_t *result = aligned_alloc(QCGC_ARENA_SIZE, size);
+#if QCGC_INIT_ZERO
+	memset(result, 0, size);
+#endif
+	qcgc_hbtable_insert(result);
+	result->flags = QCGC_GRAY_FLAG;
+
+	qcgc_state.cells_since_incmark += bytes_to_cells(size);
+
+	return result;
+}
+
+object_t *_qcgc_allocate_slowpath(size_t size) {
+	bool use_fit_allocator = _qcgc_bump_allocator.ptr == NULL;
 	size_t cells = bytes_to_cells(size);
-	qcgc_event_logger_log(EVENT_ALLOCATE, sizeof(size_t),
-			(uint8_t *) &cells);
+
+	if (UNLIKELY(qcgc_state.cells_since_incmark >
+				qcgc_state.incmark_threshold)) {
+		if (qcgc_state.incmark_since_sweep == qcgc_state.incmark_to_sweep) {
+			qcgc_reset_bump_ptr();
+			qcgc_collect();
+			use_fit_allocator = false; // Try using bump allocator again
+		} else {
+			qcgc_incmark();
+			qcgc_state.incmark_since_sweep++;
+		}
+	}
+
+	object_t *result = NULL;
+	if (!use_fit_allocator) {
+		qcgc_bump_allocator_renew_block(false);
+
+		qcgc_state.cells_since_incmark += _qcgc_bump_allocator.end -
+			_qcgc_bump_allocator.ptr;
+
+		cell_t *new_bump_ptr = _qcgc_bump_allocator.ptr + cells;
+		if (_qcgc_bump_allocator.ptr != NULL &&
+				new_bump_ptr <= _qcgc_bump_allocator.end) {
+			// Bump allocate
+			qcgc_arena_set_blocktype(qcgc_arena_addr(_qcgc_bump_allocator.ptr),
+					qcgc_arena_cell_index(_qcgc_bump_allocator.ptr),
+					BLOCK_WHITE);
+
+			result = (object_t *) _qcgc_bump_allocator.ptr;
+			_qcgc_bump_allocator.ptr = new_bump_ptr;
+
+#if QCGC_INIT_ZERO
+			memset(result, 0, cells * sizeof(cell_t));
 #endif
+
+			result->flags = QCGC_GRAY_FLAG;
+			return result;
+		}
+	}
+
+	// Fit allocate
+	if (result != NULL) {
+		qcgc_state.cells_since_incmark += bytes_to_cells(size);
+		return result;
+	}
+	qcgc_bump_allocator_renew_block(true);
+	qcgc_state.cells_since_incmark +=
+		_qcgc_bump_allocator.end - _qcgc_bump_allocator.ptr;
+
+	cell_t *new_bump_ptr = _qcgc_bump_allocator.ptr + cells;
+	qcgc_arena_set_blocktype(qcgc_arena_addr(_qcgc_bump_allocator.ptr),
+			qcgc_arena_cell_index(_qcgc_bump_allocator.ptr),
+			BLOCK_WHITE);
+
+	result = (object_t *) _qcgc_bump_allocator.ptr;
+	_qcgc_bump_allocator.ptr = new_bump_ptr;
+
+#if QCGC_INIT_ZERO
+	memset(result, 0, cells * sizeof(cell_t));
+#endif
+
+	result->flags = QCGC_GRAY_FLAG;
+	return result;
+}
+
+/*
+object_t *_qcgc_allocate_slowpath(size_t size) {
 	object_t *result;
 
 	if (UNLIKELY(qcgc_state.cells_since_incmark >
@@ -79,27 +167,20 @@
 		}
 	}
 
-	if (LIKELY(size <= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP)) {
-		// Use bump / fit allocator
-		if (qcgc_allocator_state.use_bump_allocator) {
+	// Use bump / fit allocator
+	if (_qcgc_bump_allocator.ptr != NULL) {
+		result = bump_allocate(size);
+	} else {
+		result = qcgc_fit_allocate(size);
+
+		// Fallback to bump allocator
+		if (result == NULL) {
 			result = bump_allocate(size);
-		} else {
-			result = qcgc_fit_allocate(size);
-
-			// Fallback to bump allocator
-			if (result == NULL) {
-				result = bump_allocate(size);
-			}
 		}
-		qcgc_state.free_cells -= bytes_to_cells(size);
-	} else {
-		// Use huge block allocator
-		result = qcgc_large_allocate(size);
 	}
-	qcgc_state.cells_since_incmark += bytes_to_cells(size);
-	qcgc_state.cells_since_collect += bytes_to_cells(size);
 	return result;
 }
+*/
 
 void qcgc_collect(void) {
 	qcgc_mark();
@@ -121,7 +202,7 @@
 	if (((object->flags & QCGC_PREBUILT_OBJECT) != 0) &&
 			((object->flags & QCGC_PREBUILT_REGISTERED) == 0)) {
 		object->flags |= QCGC_PREBUILT_REGISTERED;
-		qcgc_state.prebuilt_objects = qcgc_shadow_stack_push(
+		qcgc_state.prebuilt_objects = qcgc_object_stack_push(
 				qcgc_state.prebuilt_objects, object);
 	}
 
@@ -137,13 +218,13 @@
 		// NOTE: No mark test here, as prebuilt objects are always reachable
 		// Push prebuilt object to general purpose gray stack
 		qcgc_state.gray_stack_size++;
-		qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+		qcgc_state.gp_gray_stack = qcgc_object_stack_push(
 				qcgc_state.gp_gray_stack, object);
 	} else if ((object_t *) qcgc_arena_addr((cell_t *) object) == object) {
 		if (qcgc_hbtable_is_marked(object)) {
 			// Push huge block to general purpose gray stack
 			qcgc_state.gray_stack_size++;
-			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+			qcgc_state.gp_gray_stack = qcgc_object_stack_push(
 					qcgc_state.gp_gray_stack, object);
 		}
 	} else {
@@ -152,7 +233,7 @@
 			// This was black before, push it to gray stack again
 			arena_t *arena = qcgc_arena_addr((cell_t *) object);
 			qcgc_state.gray_stack_size++;
-			arena->gray_stack = qcgc_gray_stack_push(
+			arena->gray_stack = qcgc_object_stack_push(
 					arena->gray_stack, object);
 		}
 	}
@@ -188,21 +269,13 @@
 	assert(stack != NULL);
 	mprotect(_trap_page_addr(stack), 4096, PROT_NONE);
 
-	qcgc_shadowstack.top = stack;
-	qcgc_shadowstack.base = stack;
+	_qcgc_shadowstack.top = stack;
+	_qcgc_shadowstack.base = stack;
 }
 
 QCGC_STATIC void destroy_shadowstack(void) {
-	mprotect(_trap_page_addr(qcgc_shadowstack.base), 4096, PROT_READ |
+	mprotect(_trap_page_addr(_qcgc_shadowstack.base), 4096, PROT_READ |
 				PROT_WRITE);
 
-	free(qcgc_shadowstack.base);
+	free(_qcgc_shadowstack.base);
 }
-
-QCGC_STATIC object_t *bump_allocate(size_t size) {
-	if (UNLIKELY(qcgc_allocator_state.bump_state.remaining_cells <
-			bytes_to_cells(size))) {
-		qcgc_bump_allocator_renew_block();
-	}
-	return qcgc_bump_allocate(size);
-}
diff --git a/rpython/translator/c/src/qcgc/qcgc.h b/rpython/translator/c/src/qcgc/qcgc.h
--- a/rpython/translator/c/src/qcgc/qcgc.h
+++ b/rpython/translator/c/src/qcgc/qcgc.h
@@ -2,32 +2,180 @@
  * @file	qcgc.h
  */
 
-#pragma once
+#ifndef __QCGC_H
+#define __QCGC_H
 
 #include "config.h"
 
 #include <assert.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
+
+#include "src/event_logger.h"
+
+/*******************************************************************************
+ * Types and global state                                                      *
+ ******************************************************************************/
 
 /**
  * Object Layout.
  */
+typedef struct object_s {
+	uint32_t flags;
+} object_t;
+
 #define QCGC_GRAY_FLAG (1<<0)
 #define QCGC_PREBUILT_OBJECT (1<<1)
 #define QCGC_PREBUILT_REGISTERED (1<<2)
 
-typedef struct object_s {
-	uint32_t flags;
-} object_t;
-
 /**
  * Shadow stack
  */
 struct qcgc_shadowstack {
 	object_t **top;
 	object_t **base;
-} qcgc_shadowstack;
+} _qcgc_shadowstack;
+
+/**
+ * The smallest unit of memory that can be addressed and allocated.
+ */
+typedef uint8_t cell_t[16];
+
+/**
+ * Bump allocator
+ */
+struct qcgc_bump_allocator {
+	cell_t *ptr;
+	cell_t *end;
+} _qcgc_bump_allocator;
+
+/**
+ * Object stack
+ */
+typedef struct object_stack_s {
+	size_t count;
+	size_t size;
+	object_t *items[];
+} object_stack_t;
+
+/**
+ * Arena
+ */
+
+#define QCGC_ARENA_SIZE (1<<QCGC_ARENA_SIZE_EXP)
+
+#define QCGC_ARENA_BITMAP_SIZE (1<<(QCGC_ARENA_SIZE_EXP - 7)) // 1 / 128
+#define QCGC_ARENA_CELLS_COUNT (1<<(QCGC_ARENA_SIZE_EXP - 4))
+
+#define QCGC_ARENA_FIRST_CELL_INDEX (1<<(QCGC_ARENA_SIZE_EXP - 10))
+
+typedef union {
+	struct {
+		union {
+			object_stack_t *gray_stack;
+			uint8_t block_bitmap[QCGC_ARENA_BITMAP_SIZE];
+		};
+		uint8_t mark_bitmap[QCGC_ARENA_BITMAP_SIZE];
+	};
+	cell_t cells[QCGC_ARENA_CELLS_COUNT];
+} arena_t;
+
+typedef enum blocktype {
+	BLOCK_EXTENT,
+	BLOCK_FREE,
+	BLOCK_WHITE,
+	BLOCK_BLACK,
+} blocktype_t;
+
+/*******************************************************************************
+ * Internal functions                                                          *
+ ******************************************************************************/
+
+/**
+ * Allocate large block. May trigger garbage collection.
+ *
+ * @param	size	Object size in bytes
+ * @return	Pointer to memory region large enough to hold size bytes or NULL in
+ *			case of errros
+ */
+object_t *_qcgc_allocate_large(size_t size);
+
+/**
+ * Allocator slowpath. May trigger garabge collection.
+ *
+ * @param	size	Object size in bytes
+ * @return	Pointer to memory region large enough to hold size bytes or NULL in
+ *			case of errros
+ */
+object_t *_qcgc_allocate_slowpath(size_t size);
+
+/**
+ * Turns bytes to cells.
+ */
+QCGC_STATIC QCGC_INLINE size_t bytes_to_cells(size_t bytes) {
+	return (bytes + sizeof(cell_t) - 1) / sizeof(cell_t);
+}
+
+/**
+ * Arena pointer for given cell.
+ *
+ * @param	ptr		Pointer to cell for which you want to know the corresponding
+ *					arena
+ * @return	The arena the pointer belongs to
+ */
+QCGC_STATIC QCGC_INLINE arena_t *qcgc_arena_addr(cell_t *ptr) {
+	return (arena_t *)((intptr_t) ptr & ~(QCGC_ARENA_SIZE - 1));
+}
+
+/**
+ * Index of cell in arena.
+ *
+ * @param	ptr		Pointer to cell for which you want to know the cell index
+ * @return	Index of the cell to which ptr points to
+ */
+QCGC_STATIC QCGC_INLINE size_t qcgc_arena_cell_index(cell_t *ptr) {
+	return (size_t)((intptr_t) ptr & (QCGC_ARENA_SIZE - 1)) >> 4;
+}
+
+/**
+ * Set blocktype.
+ *
+ * @param	ptr		Pointer to cell for which you want to set the blocktype
+ * @param	type	Blocktype that should be set
+ */
+QCGC_STATIC QCGC_INLINE void qcgc_arena_set_blocktype(arena_t *arena,
+		size_t index, blocktype_t type) {
+#if CHECKED
+	assert(arena != NULL);
+	assert(index >= QCGC_ARENA_FIRST_CELL_INDEX);
+	assert(index < QCGC_ARENA_CELLS_COUNT);
+#endif
+	size_t byte = index / 8;
+	uint8_t mask = 0x1 << (index % 8);
+	switch(type) {
+		case BLOCK_EXTENT:
+			arena->block_bitmap[byte] &= ~mask;
+			arena->mark_bitmap[byte] &= ~mask;
+			break;
+		case BLOCK_FREE:
+			arena->block_bitmap[byte] &= ~mask;
+			arena->mark_bitmap[byte] |= mask;
+			break;
+		case BLOCK_WHITE:
+			arena->block_bitmap[byte] |= mask;
+			arena->mark_bitmap[byte] &= ~mask;
+			break;
+		case BLOCK_BLACK:
+			arena->block_bitmap[byte] |= mask;
+			arena->mark_bitmap[byte] |= mask;
+			break;
+	}
+}
+
+/*******************************************************************************
+ * Public functions                                                            *
+ ******************************************************************************/
 
 /**
  * Initialize the garbage collector.
@@ -46,7 +194,41 @@
  * @return	Pointer to memory region large enough to hold size bytes or NULL in
  *			case of errros
  */
-object_t *qcgc_allocate(size_t size);
+QCGC_STATIC QCGC_INLINE object_t *qcgc_allocate(size_t size) {
+#if CHECKED
+	assert(size > 0);
+#endif
+	size_t cells = bytes_to_cells(size);
+
+#if LOG_ALLOCATION
+	qcgc_event_logger_log(EVENT_ALLOCATE, sizeof(size_t),
+			(uint8_t *) &cells);
+#endif
+	if (UNLIKELY(size >= 1<<QCGC_LARGE_ALLOC_THRESHOLD_EXP)) {
+		return _qcgc_allocate_large(size);
+	}
+
+	cell_t *new_bump_ptr = _qcgc_bump_allocator.ptr + cells;
+	// XXX: UNLIKELY?
+	if (new_bump_ptr > _qcgc_bump_allocator.end) {
+		return _qcgc_allocate_slowpath(size);
+	}
+
+	qcgc_arena_set_blocktype(qcgc_arena_addr(_qcgc_bump_allocator.ptr),
+			qcgc_arena_cell_index(_qcgc_bump_allocator.ptr),
+			BLOCK_WHITE);
+
+	object_t *result = (object_t *) _qcgc_bump_allocator.ptr;
+	_qcgc_bump_allocator.ptr = new_bump_ptr;
+
+
+#if QCGC_INIT_ZERO
+	memset(result, 0, cells * sizeof(cell_t));
+#endif
+
+	result->flags = QCGC_GRAY_FLAG;
+	return result;
+}
 
 /**
  * Push root object.
@@ -54,8 +236,8 @@
  * @param	object	The root object
  */
 QCGC_STATIC QCGC_INLINE void qcgc_push_root(object_t *object) {
-	*qcgc_shadowstack.top = object;
-	qcgc_shadowstack.top++;
+	*_qcgc_shadowstack.top = object;
+	_qcgc_shadowstack.top++;
 }
 
 /**
@@ -64,8 +246,8 @@
  * @param	count	Number of object to pop
  */
 QCGC_STATIC QCGC_INLINE void qcgc_pop_root(size_t count) {
-	qcgc_shadowstack.top -= count;
-	assert(qcgc_shadowstack.base <= qcgc_shadowstack.top);
+	_qcgc_shadowstack.top -= count;
+	assert(_qcgc_shadowstack.base <= _qcgc_shadowstack.top);
 }
 
 /**
@@ -100,3 +282,5 @@
  * @param	visit	The function to be called on the referenced objects
  */
 extern void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object));
+
+#endif
diff --git a/rpython/translator/c/src/qcgc/src/allocator.c b/rpython/translator/c/src/qcgc/src/allocator.c
--- a/rpython/translator/c/src/qcgc/src/allocator.c
+++ b/rpython/translator/c/src/qcgc/src/allocator.c
@@ -20,12 +20,6 @@
 		qcgc_arena_bag_create(QCGC_ARENA_BAG_INIT_SIZE);
 	qcgc_allocator_state.free_arenas = qcgc_arena_bag_create(4); // XXX
 
-	qcgc_allocator_state.use_bump_allocator = true;
-
-	// Bump Allocator
-	qcgc_allocator_state.bump_state.bump_ptr = NULL;
-	qcgc_allocator_state.bump_state.remaining_cells = 0;
-
 	// Fit Allocator
 	for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
 		qcgc_allocator_state.fit_state.small_free_list[i] =
@@ -36,6 +30,10 @@
 		qcgc_allocator_state.fit_state.large_free_list[i] =
 			qcgc_exp_free_list_create(QCGC_LARGE_FREE_LIST_INIT_SIZE);
 	}
+
+	_qcgc_bump_allocator.ptr = NULL;
+	_qcgc_bump_allocator.end = NULL;
+	qcgc_bump_allocator_renew_block(true);
 }
 
 void qcgc_allocator_destroy(void) {
@@ -67,31 +65,20 @@
  * Bump Allocator                                                              *
  ******************************************************************************/
 
-void qcgc_bump_allocator_renew_block(void) {
+void qcgc_bump_allocator_renew_block(bool force_arena) {
 #if CHECKED
-	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
-		for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
-				i++) {
-			assert(qcgc_arena_get_blocktype(qcgc_arena_addr(
-							qcgc_allocator_state.bump_state.bump_ptr + i),
-						qcgc_arena_cell_index(
-							qcgc_allocator_state.bump_state.bump_ptr + i))
+	if (_qcgc_bump_allocator.end > _qcgc_bump_allocator.ptr) {
+		for (cell_t *it = _qcgc_bump_allocator.ptr + 1;
+				it < _qcgc_bump_allocator.end; it++) {
+			assert(qcgc_arena_get_blocktype(qcgc_arena_addr(it),
+						qcgc_arena_cell_index(it))
 					== BLOCK_EXTENT);
 		}
 	}
 #endif
-	// Add remaining memory to fit allocator
-	if (qcgc_allocator_state.bump_state.remaining_cells > 0) {
-		qcgc_arena_set_blocktype(
-				qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr),
-				qcgc_arena_cell_index(
-					qcgc_allocator_state.bump_state.bump_ptr),
-				BLOCK_FREE);
-		qcgc_fit_allocator_add(qcgc_allocator_state.bump_state.bump_ptr,
-				qcgc_allocator_state.bump_state.remaining_cells);
-	}
+	qcgc_reset_bump_ptr();
 
-	// Try finding some huge block from fit allocator
+	// Always use a huge block if there is one
 	exp_free_list_t *free_list = qcgc_allocator_state.fit_state.
 		large_free_list[QCGC_LARGE_FREE_LISTS - 1];
 
@@ -100,33 +87,45 @@
 		bump_allocator_assign(free_list->items[0].ptr,
 				free_list->items[0].size);
 		free_list = qcgc_exp_free_list_remove_index(free_list, 0);
+		qcgc_allocator_state.fit_state.
+			large_free_list[QCGC_LARGE_FREE_LISTS - 1] = free_list;
+		qcgc_state.free_cells -=
+			_qcgc_bump_allocator.end - _qcgc_bump_allocator.ptr;
 	} else {
-		// Grab a new arena
-		arena_t *arena = qcgc_arena_create();
-		qcgc_state.free_cells += QCGC_ARENA_CELLS_COUNT -
-			QCGC_ARENA_FIRST_CELL_INDEX;
-		bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
-				QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
-		qcgc_allocator_state.arenas =
-			qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
+		if (qcgc_allocator_state.free_arenas->count > 0) {
+			// Reuse arena
+			arena_t *arena = qcgc_allocator_state.free_arenas->items[0];
+			qcgc_allocator_state.free_arenas = qcgc_arena_bag_remove_index(
+					qcgc_allocator_state.free_arenas, 0);
+			bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
+					QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
+			qcgc_allocator_state.arenas =
+				qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
+		} else {
+			// FIXME: Nifty decision making whether to allocate new arena
+			bool new_arena = false;
+			if (force_arena || new_arena) {
+				arena_t *arena = qcgc_arena_create();
+				bump_allocator_assign(&(arena->cells[QCGC_ARENA_FIRST_CELL_INDEX]),
+						QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX);
+				qcgc_allocator_state.arenas =
+					qcgc_arena_bag_add(qcgc_allocator_state.arenas, arena);
+			}
+		}
 	}
-
-	qcgc_allocator_state.fit_state.
-		large_free_list[QCGC_LARGE_FREE_LISTS - 1] = free_list;
 #if CHECKED
-	assert(qcgc_allocator_state.bump_state.bump_ptr != NULL);
-	assert(qcgc_arena_get_blocktype(
-				qcgc_arena_addr( qcgc_allocator_state.bump_state.bump_ptr),
-				qcgc_arena_cell_index(
-					qcgc_allocator_state.bump_state.bump_ptr))
-			== BLOCK_FREE);
-	for (size_t i = 1; i < qcgc_allocator_state.bump_state.remaining_cells;
-			i++) {
-		assert(qcgc_arena_get_blocktype(qcgc_arena_addr(
-						qcgc_allocator_state.bump_state.bump_ptr + i),
+	assert(!force_arena || _qcgc_bump_allocator.ptr != NULL);
+	if (_qcgc_bump_allocator.ptr != NULL) {
+		assert(qcgc_arena_get_blocktype(
+					qcgc_arena_addr( _qcgc_bump_allocator.ptr),
 					qcgc_arena_cell_index(
-						qcgc_allocator_state.bump_state.bump_ptr + i))
-				== BLOCK_EXTENT);
+						_qcgc_bump_allocator.ptr))
+				== BLOCK_FREE);
+		for (cell_t *it = _qcgc_bump_allocator.ptr + 1;
+				it < _qcgc_bump_allocator.end; it++) {
+			assert(qcgc_arena_get_blocktype(qcgc_arena_addr(it),
+						qcgc_arena_cell_index(it)) == BLOCK_EXTENT);
+		}
 	}
 #endif
 }
@@ -140,8 +139,8 @@
 					qcgc_arena_cell_index(ptr + i)) == BLOCK_EXTENT);
 	}
 #endif
-	qcgc_allocator_state.bump_state.bump_ptr = ptr;
-	qcgc_allocator_state.bump_state.remaining_cells = cells;
+	_qcgc_bump_allocator.ptr = ptr;
+	_qcgc_bump_allocator.end = ptr + cells;
 }
 
 /*******************************************************************************
@@ -173,6 +172,7 @@
 	memset(result, 0, cells * sizeof(cell_t));
 #endif
 
+	qcgc_state.free_cells -= cells;
 	result->flags = QCGC_GRAY_FLAG;
 	return result;
 }
@@ -204,6 +204,7 @@
 					qcgc_allocator_state.fit_state.large_free_list[index],
 					(struct exp_free_list_item_s) {ptr, cells});
 	}
+	qcgc_state.free_cells += cells;
 }
 
 QCGC_STATIC cell_t *fit_allocator_small_first_fit(size_t index, size_t cells) {
@@ -227,6 +228,7 @@
 				qcgc_arena_set_blocktype(qcgc_arena_addr(result + cells),
 						qcgc_arena_cell_index(result + cells), BLOCK_FREE);
 				qcgc_fit_allocator_add(result + cells, list_cell_size - cells);
+				qcgc_state.free_cells -= list_cell_size - cells;
 			}
 			return result;
 		}
@@ -274,6 +276,7 @@
 			qcgc_arena_set_blocktype(qcgc_arena_addr(result + cells),
 					qcgc_arena_cell_index(result + cells), BLOCK_FREE);
 			qcgc_fit_allocator_add(result + cells, best_fit_cells - cells);
+			qcgc_state.free_cells -= best_fit_cells - cells;
 		}
 	} else {
 		// No best fit, go for first fit
@@ -301,6 +304,7 @@
 				qcgc_arena_set_blocktype(qcgc_arena_addr(item.ptr + cells),
 						qcgc_arena_cell_index(item.ptr + cells), BLOCK_FREE);
 				qcgc_fit_allocator_add(item.ptr + cells, item.size - cells);
+				qcgc_state.free_cells -= item.size - cells;
 			}
 			return item.ptr;
 		}
diff --git a/rpython/translator/c/src/qcgc/src/allocator.h b/rpython/translator/c/src/qcgc/src/allocator.h
--- a/rpython/translator/c/src/qcgc/src/allocator.h
+++ b/rpython/translator/c/src/qcgc/src/allocator.h
@@ -41,15 +41,10 @@
 struct qcgc_allocator_state {
 	arena_bag_t *arenas;
 	arena_bag_t *free_arenas;
-	struct bump_state {
-		cell_t *bump_ptr;
-		size_t remaining_cells;
-	} bump_state;
 	struct fit_state {
 		linear_free_list_t *small_free_list[QCGC_SMALL_FREE_LISTS];
 		exp_free_list_t *large_free_list[QCGC_LARGE_FREE_LISTS];
 	} fit_state;
-	bool use_bump_allocator;
 } qcgc_allocator_state;
 
 /**
@@ -77,7 +72,6 @@
  */
 void qcgc_fit_allocator_empty_lists(void);
 
-
 /**
  * Add memory to free lists
  *
@@ -86,58 +80,26 @@
  */
 void qcgc_fit_allocator_add(cell_t *ptr, size_t cells);
 
-QCGC_STATIC QCGC_INLINE size_t bytes_to_cells(size_t bytes) {
-	return (bytes + sizeof(cell_t) - 1) / sizeof(cell_t);
+/**
+ * Reset bump pointer
+ */
+QCGC_STATIC QCGC_INLINE void qcgc_reset_bump_ptr(void) {
+	if (_qcgc_bump_allocator.end > _qcgc_bump_allocator.ptr) {
+		qcgc_arena_set_blocktype(
+				qcgc_arena_addr(_qcgc_bump_allocator.ptr),
+				qcgc_arena_cell_index(
+					_qcgc_bump_allocator.ptr),
+				BLOCK_FREE);
+		qcgc_fit_allocator_add(_qcgc_bump_allocator.ptr,
+				_qcgc_bump_allocator.end - _qcgc_bump_allocator.ptr);
+	}
+	_qcgc_bump_allocator.ptr = NULL;
+	_qcgc_bump_allocator.end = NULL;
 }
 
 /**
  * Find a new block for the bump allocator
+ *
+ * @param	force_arena	Force generation of new arena if no block is found
  */
-void qcgc_bump_allocator_renew_block(void);
-
-/**
- * Allocate new memory region using bump allocator.
- * Bump allocator must have enough space for desired bytes
- * (client is responsible, use qcgc_bump_allocator_renew_block)
- *
- * @param	bytes	Desired size of the memory region in bytes
- * @return	Pointer to memory large enough to hold size bytes, NULL in case of
- *			errors, already zero initialized if QCGC_INIT_ZERO is set
- */
-QCGC_STATIC QCGC_INLINE object_t *qcgc_bump_allocate(size_t bytes) {
-	size_t cells = bytes_to_cells(bytes);
-
-	cell_t *mem = qcgc_allocator_state.bump_state.bump_ptr;
-
-	qcgc_arena_set_blocktype(qcgc_arena_addr(mem), qcgc_arena_cell_index(mem),
-			BLOCK_WHITE);
-
-	qcgc_allocator_state.bump_state.bump_ptr += cells;
-	qcgc_allocator_state.bump_state.remaining_cells -= cells;
-
-	object_t *result = (object_t *) mem;
-
-#if QCGC_INIT_ZERO
-	memset(result, 0, cells * sizeof(cell_t));
-#endif
-
-	result->flags = QCGC_GRAY_FLAG;
-	return result;
-}
-
-/**
- * Allocate new memory region using huge block allocator
- *
- * @param	bytes	Desired size of the memory region in bytes
- * @return	Pointer to memory large enough to hold size bytes, NULL in case of
- *			errors, already zero initialized if QCGC_INIT_ZERO is set
- */
-QCGC_STATIC QCGC_INLINE object_t *qcgc_large_allocate(size_t bytes) {
-	object_t *result = aligned_alloc(QCGC_ARENA_SIZE, bytes);
-#if QCGC_INIT_ZERO
-	memset(result, 0, bytes);
-#endif
-	qcgc_hbtable_insert(result);
-	result->flags = QCGC_GRAY_FLAG;
-	return result;
-}
+void qcgc_bump_allocator_renew_block(bool force_arena);
diff --git a/rpython/translator/c/src/qcgc/src/arena.c b/rpython/translator/c/src/qcgc/src/arena.c
--- a/rpython/translator/c/src/qcgc/src/arena.c
+++ b/rpython/translator/c/src/qcgc/src/arena.c
@@ -12,6 +12,7 @@
 #include "allocator.h"
 #include "event_logger.h"
 #include "gc_state.h"
+#include "object_stack.h"
 
 arena_t *qcgc_arena_create(void) {
 	qcgc_event_logger_log(EVENT_NEW_ARENA, 0, NULL);
@@ -45,7 +46,7 @@
 	result->mark_bitmap[QCGC_ARENA_FIRST_CELL_INDEX / 8] = 1;
 
 	// Create gray stack
-	result->gray_stack = qcgc_gray_stack_create(QCGC_GRAY_STACK_INIT_SIZE);
+	result->gray_stack = qcgc_object_stack_create(QCGC_GRAY_STACK_INIT_SIZE);
 	return result;
 }
 
@@ -94,7 +95,7 @@
 #if CHECKED
 	assert(arena != NULL);
 	assert(qcgc_arena_is_coalesced(arena));
-	assert(qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) == arena);
+	assert(qcgc_arena_addr(_qcgc_bump_allocator.ptr) == arena);
 #endif
 	// Ignore free cell / largest block counting here, as blocks are not
 	// registerd in free lists as well
@@ -121,7 +122,7 @@
 	assert(arena != NULL);
 	assert(qcgc_arena_is_coalesced(arena));
 #endif
-	if (qcgc_arena_addr(qcgc_allocator_state.bump_state.bump_ptr) == arena) {
+	if (qcgc_arena_addr(_qcgc_bump_allocator.ptr) == arena) {
 		return qcgc_arena_pseudo_sweep(arena);
 	}
 
@@ -176,7 +177,6 @@
 						memset(arena->cells + last_free_cell, 0,
 								sizeof(cell_t) * (cell - last_free_cell));
 #endif
-						qcgc_state.free_cells += cell - last_free_cell;
 						qcgc_state.largest_free_block = MAX(
 								qcgc_state.largest_free_block,
 								cell - last_free_cell);
@@ -195,7 +195,6 @@
 		memset(arena->cells + last_free_cell, 0,
 				sizeof(cell_t) * (QCGC_ARENA_CELLS_COUNT - last_free_cell));
 #endif
-		qcgc_state.free_cells += QCGC_ARENA_CELLS_COUNT - last_free_cell;
 		qcgc_state.largest_free_block = MAX(
 				qcgc_state.largest_free_block,
 				QCGC_ARENA_CELLS_COUNT - last_free_cell);
diff --git a/rpython/translator/c/src/qcgc/src/arena.h b/rpython/translator/c/src/qcgc/src/arena.h
--- a/rpython/translator/c/src/qcgc/src/arena.h
+++ b/rpython/translator/c/src/qcgc/src/arena.h
@@ -9,51 +9,6 @@
 #include <stdbool.h>
 #include <sys/types.h>
 
-#include "gray_stack.h"
-
-#define QCGC_ARENA_SIZE (1<<QCGC_ARENA_SIZE_EXP)
-
-#define QCGC_ARENA_BITMAP_SIZE (1<<(QCGC_ARENA_SIZE_EXP - 7)) // 1 / 128
-#define QCGC_ARENA_CELLS_COUNT (1<<(QCGC_ARENA_SIZE_EXP - 4))
-
-#define QCGC_ARENA_FIRST_CELL_INDEX (1<<(QCGC_ARENA_SIZE_EXP - 10))
-
-/**
- * @typedef cell_t
- * The smallest unit of memory that can be addressed and allocated in arenas.
- */
-typedef uint8_t cell_t[16];
-
-/**
- * @typedef arena_t
- * Arena object
- */
-typedef union {
-	struct {
-		union {
-			gray_stack_t *gray_stack;
-			uint8_t block_bitmap[QCGC_ARENA_BITMAP_SIZE];
-		};
-		uint8_t mark_bitmap[QCGC_ARENA_BITMAP_SIZE];
-	};
-	cell_t cells[QCGC_ARENA_CELLS_COUNT];
-} arena_t;
-
-/**
- * @typedef blocktype_t
- * Blocktypes:
- * - BLOCK_EXTENT	Extension of previous block
- * - BLOCK_FREE		Free block
- * - BLOCK_WHITE	Allocated block, marked white
- * - BLOCK_BLACK	Allocated block, marked black
- */
-typedef enum blocktype {
-	BLOCK_EXTENT,
-	BLOCK_FREE,
-	BLOCK_WHITE,
-	BLOCK_BLACK,
-} blocktype_t;
-
 /**
  * Create a new arena.
  *
@@ -107,27 +62,6 @@
  ******************************************************************************/
 
 /**
- * Arena pointer for given cell.
- *
- * @param	ptr		Pointer to cell for which you want to know the corresponding
- *					arena
- * @return	The arena the pointer belongs to
- */
-QCGC_STATIC QCGC_INLINE arena_t *qcgc_arena_addr(cell_t *ptr) {
-	return (arena_t *)((intptr_t) ptr & ~(QCGC_ARENA_SIZE - 1));
-}
-
-/**
- * Index of cell in arena.
- *
- * @param	ptr		Pointer to cell for which you want to know the cell index
- * @return	Index of the cell to which ptr points to
- */
-QCGC_STATIC QCGC_INLINE size_t qcgc_arena_cell_index(cell_t *ptr) {
-	return (size_t)((intptr_t) ptr & (QCGC_ARENA_SIZE - 1)) >> 4;
-}
-
-/**
  * Get blocktype.
  *
  * @param	arena		Arena in which to perform the lookup
@@ -162,41 +96,6 @@
 	}
 }
 
-/**
- * Set blocktype.
- *
- * @param	ptr		Pointer to cell for which you want to set the blocktype
- * @param	type	Blocktype that should be set
- */
-QCGC_STATIC QCGC_INLINE void qcgc_arena_set_blocktype(arena_t *arena,
-		size_t index, blocktype_t type) {
-#if CHECKED
-	assert(arena != NULL);
-	assert(index >= QCGC_ARENA_FIRST_CELL_INDEX);
-	assert(index < QCGC_ARENA_CELLS_COUNT);
-#endif
-	size_t byte = index / 8;
-	uint8_t mask = 0x1 << (index % 8);
-	switch(type) {
-		case BLOCK_EXTENT:
-			arena->block_bitmap[byte] &= ~mask;
-			arena->mark_bitmap[byte] &= ~mask;
-			break;
-		case BLOCK_FREE:
-			arena->block_bitmap[byte] &= ~mask;
-			arena->mark_bitmap[byte] |= mask;
-			break;
-		case BLOCK_WHITE:
-			arena->block_bitmap[byte] |= mask;
-			arena->mark_bitmap[byte] &= ~mask;
-			break;
-		case BLOCK_BLACK:
-			arena->block_bitmap[byte] |= mask;
-			arena->mark_bitmap[byte] |= mask;
-			break;
-	}
-}
-
 /*******************************************************************************
  * Debug functions                                                             *
  ******************************************************************************/
diff --git a/rpython/translator/c/src/qcgc/src/collector.c b/rpython/translator/c/src/qcgc/src/collector.c
--- a/rpython/translator/c/src/qcgc/src/collector.c
+++ b/rpython/translator/c/src/qcgc/src/collector.c
@@ -12,16 +12,19 @@
 QCGC_STATIC void mark_setup(bool incremental);
 QCGC_STATIC void mark_cleanup(bool incremental);
 
+QCGC_STATIC void check_free_cells(void);
+QCGC_STATIC void check_largest_free_block(void);
+
 void qcgc_mark(void) {
 	mark_setup(false);
 
 	while (qcgc_state.gray_stack_size > 0) {
 		// General purpose gray stack (prebuilt objects and huge blocks)
 
-		while (qcgc_state.gp_gray_stack->index > 0) {
-			object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
+		while (qcgc_state.gp_gray_stack->count > 0) {
+			object_t *top = qcgc_object_stack_top(qcgc_state.gp_gray_stack);
 			qcgc_state.gray_stack_size--;
-			qcgc_state.gp_gray_stack = qcgc_gray_stack_pop(
+			qcgc_state.gp_gray_stack = qcgc_object_stack_pop(
 					qcgc_state.gp_gray_stack);
 			qcgc_pop_object(top);
 		}
@@ -30,10 +33,10 @@
 		for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
 			arena_t *arena = qcgc_allocator_state.arenas->items[i];
 
-			while (arena->gray_stack->index > 0) {
-				object_t *top = qcgc_gray_stack_top(arena->gray_stack);
+			while (arena->gray_stack->count > 0) {
+				object_t *top = qcgc_object_stack_top(arena->gray_stack);
 				qcgc_state.gray_stack_size--;
-				arena->gray_stack = qcgc_gray_stack_pop(arena->gray_stack);
+				arena->gray_stack = qcgc_object_stack_pop(arena->gray_stack);
 				qcgc_pop_object(top);
 			}
 		}
@@ -50,13 +53,13 @@
 	mark_setup(true);
 
 	// General purpose gray stack (prebuilt objects and huge blocks)
-	size_t to_process = MAX(qcgc_state.gp_gray_stack->index / 2,
+	size_t to_process = MAX(qcgc_state.gp_gray_stack->count / 2,
 			QCGC_INC_MARK_MIN);
 
-	while (to_process > 0 && qcgc_state.gp_gray_stack->index > 0) {
-		object_t *top = qcgc_gray_stack_top(qcgc_state.gp_gray_stack);
+	while (to_process > 0 && qcgc_state.gp_gray_stack->count > 0) {
+		object_t *top = qcgc_object_stack_top(qcgc_state.gp_gray_stack);
 		qcgc_state.gray_stack_size--;
-		qcgc_state.gp_gray_stack = qcgc_gray_stack_pop(
+		qcgc_state.gp_gray_stack = qcgc_object_stack_pop(
 				qcgc_state.gp_gray_stack);
 		qcgc_pop_object(top);
 		to_process--;
@@ -65,12 +68,12 @@
 	// Arena gray stacks
 	for (size_t i = 0; i < qcgc_allocator_state.arenas->count; i++) {
 		arena_t *arena = qcgc_allocator_state.arenas->items[i];
-		to_process = MAX(arena->gray_stack->index / 2, QCGC_INC_MARK_MIN);
+		to_process = MAX(arena->gray_stack->count / 2, QCGC_INC_MARK_MIN);
 
-		while (to_process > 0 && arena->gray_stack->index > 0) {
-			object_t *top = qcgc_gray_stack_top(arena->gray_stack);
+		while (to_process > 0 && arena->gray_stack->count > 0) {
+			object_t *top = qcgc_object_stack_top(arena->gray_stack);
 			qcgc_state.gray_stack_size--;
-			arena->gray_stack = qcgc_gray_stack_pop(arena->gray_stack);
+			arena->gray_stack = qcgc_object_stack_pop(arena->gray_stack);
 			qcgc_pop_object(top);
 			to_process--;
 		}
@@ -104,7 +107,7 @@
 		size_t count = qcgc_state.prebuilt_objects->count;
 		for (size_t i = 0; i < count; i++) {
 			qcgc_state.gray_stack_size++;
-			qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+			qcgc_state.gp_gray_stack = qcgc_object_stack_push(
 					qcgc_state.gp_gray_stack,
 					qcgc_state.prebuilt_objects->items[i]);
 		}
@@ -113,8 +116,8 @@
 	qcgc_state.phase = GC_MARK;
 
 	// Always push all roots to make shadowstack pushes faster
-	for (object_t **it = qcgc_shadowstack.base;
-		it < qcgc_shadowstack.top;
+	for (object_t **it = _qcgc_shadowstack.base;
+		it < _qcgc_shadowstack.top;
 		it++) {
 		qcgc_push_object(*it);
 	}
@@ -163,7 +166,7 @@
 				// Did mark it / was white before
 				object->flags |= QCGC_GRAY_FLAG;
 				qcgc_state.gray_stack_size++;
-				qcgc_state.gp_gray_stack = qcgc_gray_stack_push(
+				qcgc_state.gp_gray_stack = qcgc_object_stack_push(
 						qcgc_state.gp_gray_stack, object);
 			}
 			return;
@@ -176,7 +179,7 @@
 			object->flags |= QCGC_GRAY_FLAG;
 			qcgc_arena_set_blocktype(arena, index, BLOCK_BLACK);
 			qcgc_state.gray_stack_size++;
-			arena->gray_stack = qcgc_gray_stack_push(arena->gray_stack, object);
+			arena->gray_stack = qcgc_object_stack_push(arena->gray_stack, object);
 		}
 	}
 }
@@ -184,6 +187,7 @@
 void qcgc_sweep(void) {
 #if CHECKED
 	assert(qcgc_state.phase == GC_COLLECT);
+	check_free_cells();
 #endif
 	{
 		struct log_info_s {
@@ -225,38 +229,11 @@
 	// Fragmenation = 1 - (largest block / total free space)
 	// Use bump allocator when fragmentation < 50%
 #if CHECKED
-	size_t free_cells = 0;
-	size_t largest_free_block = 0;
-		for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
-			if (qcgc_allocator_state.fit_state.small_free_list[i]->count > 0) {
-				largest_free_block = i + 1;
-			}
-			free_cells += qcgc_allocator_state.fit_state.small_free_list[i]
-				->count * (i + 1);
-		}
-		for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
-			for (size_t j = 0; j < qcgc_allocator_state.fit_state.
-					large_free_list[i]->count; j++) {
-				free_cells += qcgc_allocator_state.fit_state.large_free_list[i]
-					->items[j].size;
-				largest_free_block = MAX(largest_free_block,
-					qcgc_allocator_state.fit_state.large_free_list[i]
-					->items[j].size);
-			}
-		}
-	assert(free_cells == qcgc_state.free_cells);
-	assert(largest_free_block == qcgc_state.largest_free_block);
-	assert(free_cells <= qcgc_allocator_state.arenas->count * (QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX));
-	assert(qcgc_allocator_state.arenas->count *
-			(QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX) >=
-			free_cells);
+	check_free_cells();
+	check_largest_free_block();
 #endif
-	qcgc_allocator_state.use_bump_allocator = qcgc_state.free_cells <
-		2 * qcgc_state.largest_free_block;
-
 	update_weakrefs();
 
-	qcgc_state.free_cells += qcgc_allocator_state.bump_state.remaining_cells;
 	{
 		struct log_info_s {
 			size_t arenas;
@@ -296,3 +273,40 @@
 #endif
 }
 
+void check_free_cells(void) {
+	size_t free_cells = 0;
+		for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
+			free_cells += qcgc_allocator_state.fit_state.small_free_list[i]
+				->count * (i + 1);
+		}
+		for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
+			for (size_t j = 0; j < qcgc_allocator_state.fit_state.
+					large_free_list[i]->count; j++) {
+				free_cells += qcgc_allocator_state.fit_state.large_free_list[i]
+					->items[j].size;
+			}
+		}
+	assert(free_cells == qcgc_state.free_cells);
+	assert(free_cells <= qcgc_allocator_state.arenas->count * (QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX));
+	assert(qcgc_allocator_state.arenas->count *
+			(QCGC_ARENA_CELLS_COUNT - QCGC_ARENA_FIRST_CELL_INDEX) >=
+			free_cells);
+}
+
+void check_largest_free_block(void) {
+	size_t largest_free_block = 0;
+		for (size_t i = 0; i < QCGC_SMALL_FREE_LISTS; i++) {
+			if (qcgc_allocator_state.fit_state.small_free_list[i]->count > 0) {
+				largest_free_block = i + 1;
+			}
+		}
+		for (size_t i = 0; i < QCGC_LARGE_FREE_LISTS; i++) {
+			for (size_t j = 0; j < qcgc_allocator_state.fit_state.
+					large_free_list[i]->count; j++) {
+				largest_free_block = MAX(largest_free_block,
+					qcgc_allocator_state.fit_state.large_free_list[i]
+					->items[j].size);
+			}
+		}
+	assert(largest_free_block == qcgc_state.largest_free_block);
+}
diff --git a/rpython/translator/c/src/qcgc/src/gc_state.h b/rpython/translator/c/src/qcgc/src/gc_state.h
--- a/rpython/translator/c/src/qcgc/src/gc_state.h
+++ b/rpython/translator/c/src/qcgc/src/gc_state.h
@@ -5,8 +5,7 @@
 #include <stddef.h>
 
 #include "bag.h"
-#include "gray_stack.h"
-#include "shadow_stack.h"
+#include "object_stack.h"
 
 /**
  * @typedef gc_state_t
@@ -27,14 +26,13 @@
  * Global state of the garbage collector
  */
 struct qcgc_state {
-	shadow_stack_t *prebuilt_objects;
+	object_stack_t *prebuilt_objects;
 	weakref_bag_t *weakrefs;
-	gray_stack_t *gp_gray_stack;
+	object_stack_t *gp_gray_stack;
 	size_t gray_stack_size;
 	gc_phase_t phase;
 
 	size_t cells_since_incmark;
-	size_t cells_since_collect;
 	size_t incmark_since_sweep;
 	size_t incmark_threshold;
 	size_t incmark_to_sweep;
diff --git a/rpython/translator/c/src/qcgc/src/gray_stack.c b/rpython/translator/c/src/qcgc/src/gray_stack.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/gray_stack.c
+++ /dev/null
@@ -1,28 +0,0 @@
-#include "gray_stack.h"
-
-#include <stdlib.h>
-
-QCGC_STATIC size_t gray_stack_size(size_t size);
-
-gray_stack_t *qcgc_gray_stack_create(size_t size) {
-	gray_stack_t *result = (gray_stack_t *) malloc(gray_stack_size(size));
-	result->size = size;
-	result->index = 0;
-	return result;
-}
-
-QCGC_STATIC size_t gray_stack_size(size_t size) {
-	return (sizeof(gray_stack_t) + size * sizeof(object_t *));
-}
-
-gray_stack_t *gray_stack_grow(gray_stack_t *stack) {
-	stack = (gray_stack_t *) realloc(stack, gray_stack_size(stack->size * 2));
-	stack->size *= 2;
-	return stack;
-}
-
-gray_stack_t *gray_stack_shrink(gray_stack_t *stack) {
-	stack = (gray_stack_t *) realloc(stack, gray_stack_size(stack->size / 2));
-	stack->size /= 2;
-	return stack;
-}
diff --git a/rpython/translator/c/src/qcgc/src/gray_stack.h b/rpython/translator/c/src/qcgc/src/gray_stack.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/gray_stack.h
+++ /dev/null
@@ -1,46 +0,0 @@
-#pragma once
-
-#include "../qcgc.h"
-
-typedef struct gray_stack_s {
-	size_t index;
-	size_t size;
-	object_t *items[];
-} gray_stack_t;
-
-__attribute__ ((warn_unused_result))
-gray_stack_t *qcgc_gray_stack_create(size_t size);
-
-__attribute__ ((warn_unused_result))
-gray_stack_t *gray_stack_grow(gray_stack_t *stack);
-
-__attribute__ ((warn_unused_result))
-gray_stack_t *gray_stack_shrink(gray_stack_t *stack);
-
-__attribute__ ((warn_unused_result))
-QCGC_STATIC QCGC_INLINE gray_stack_t *qcgc_gray_stack_push(
-		gray_stack_t *stack, object_t *item) {
-	if (stack->size == stack->index) {
-		stack = gray_stack_grow(stack);
-	}
-	stack->items[stack->index] = item;
-	stack->index++;
-	return stack;
-}
-
-QCGC_STATIC QCGC_INLINE object_t *qcgc_gray_stack_top(gray_stack_t *stack) {
-#if CHECKED
-	assert(stack->index != 0);
-#endif
-	return stack->items[stack->index - 1];
-}
-
-__attribute__ ((warn_unused_result))
-QCGC_STATIC QCGC_INLINE gray_stack_t *qcgc_gray_stack_pop(gray_stack_t *stack) {
-	// TODO: Add lower bound for size (config.h)
-	if (stack->index < stack->size / 4) {
-		stack = gray_stack_shrink(stack);
-	}
-	stack->index--;
-	return stack;
-}
diff --git a/rpython/translator/c/src/qcgc/src/hugeblocktable.h b/rpython/translator/c/src/qcgc/src/hugeblocktable.h
--- a/rpython/translator/c/src/qcgc/src/hugeblocktable.h
+++ b/rpython/translator/c/src/qcgc/src/hugeblocktable.h
@@ -5,7 +5,6 @@
 #include <stdbool.h>
 
 #include "bag.h"
-#include "gray_stack.h"
 
 // Choosing a prime number, hoping for good results
 #define QCGC_HBTABLE_BUCKETS 61
diff --git a/rpython/translator/c/src/qcgc/src/object_stack.c b/rpython/translator/c/src/qcgc/src/object_stack.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/src/object_stack.c
@@ -0,0 +1,30 @@
+#include "object_stack.h"
+
+#include <stdlib.h>
+
+QCGC_STATIC QCGC_INLINE size_t object_stack_size(size_t size);
+
+object_stack_t *qcgc_object_stack_create(size_t size) {
+	object_stack_t *result = (object_stack_t *) malloc(object_stack_size(size));
+	result->size = size;
+	result->count = 0;
+	return result;
+}
+
+QCGC_STATIC size_t object_stack_size(size_t size) {
+	return (sizeof(object_stack_t) + size * sizeof(object_t *));
+}
+
+object_stack_t *object_stack_grow(object_stack_t *stack) {
+	stack = (object_stack_t *) realloc(stack,
+			object_stack_size(stack->size * 2));
+	stack->size *= 2;
+	return stack;
+}
+
+object_stack_t *object_stack_shrink(object_stack_t *stack) {
+	stack = (object_stack_t *) realloc(stack,
+			object_stack_size(stack->size / 2));
+	stack->size /= 2;
+	return stack;
+}
diff --git a/rpython/translator/c/src/qcgc/src/object_stack.h b/rpython/translator/c/src/qcgc/src/object_stack.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/src/qcgc/src/object_stack.h
@@ -0,0 +1,41 @@
+#pragma once
+
+#include "../qcgc.h"
+
+__attribute__ ((warn_unused_result))
+object_stack_t *qcgc_object_stack_create(size_t size);
+
+__attribute__ ((warn_unused_result))
+object_stack_t *object_stack_grow(object_stack_t *stack);
+
+__attribute__ ((warn_unused_result))
+object_stack_t *object_stack_shrink(object_stack_t *stack);
+
+__attribute__ ((warn_unused_result))
+QCGC_STATIC QCGC_INLINE object_stack_t *qcgc_object_stack_push(
+		object_stack_t *stack, object_t *item) {
+	if (stack->size == stack->count) {
+		stack = object_stack_grow(stack);
+	}
+	stack->items[stack->count] = item;
+	stack->count++;
+	return stack;
+}
+
+QCGC_STATIC QCGC_INLINE object_t *qcgc_object_stack_top(object_stack_t *stack) {
+#if CHECKED
+	assert(stack->count != 0);
+#endif
+	return stack->items[stack->count - 1];
+}
+
+__attribute__ ((warn_unused_result))
+QCGC_STATIC QCGC_INLINE object_stack_t *qcgc_object_stack_pop(
+		object_stack_t *stack) {
+	// TODO: Add lower bound for size (config.h)
+	if (stack->count < stack->size / 4) {
+		stack = object_stack_shrink(stack);
+	}
+	stack->count--;
+	return stack;
+}
diff --git a/rpython/translator/c/src/qcgc/src/shadow_stack.c b/rpython/translator/c/src/qcgc/src/shadow_stack.c
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/shadow_stack.c
+++ /dev/null
@@ -1,56 +0,0 @@
-#include "shadow_stack.h"
-
-#include <assert.h>
-#include <stdlib.h>
-
-QCGC_STATIC size_t shadow_stack_size(size_t size);
-QCGC_STATIC shadow_stack_t *shadow_stack_grow(shadow_stack_t *stack);
-QCGC_STATIC shadow_stack_t *shadow_stack_shrink(shadow_stack_t *stack);
-
-shadow_stack_t *qcgc_shadow_stack_create(size_t size) {
-	shadow_stack_t *result = (shadow_stack_t *) malloc(shadow_stack_size(size));
-	result->size = size;
-	result->count = 0;
-	return result;
-}
-
-shadow_stack_t *qcgc_shadow_stack_push(shadow_stack_t *stack, object_t *item) {
-	if (stack->size == stack->count) {
-		stack = shadow_stack_grow(stack);
-	}
-	stack->items[stack->count] = item;
-	stack->count++;
-	return stack;
-}
-
-object_t *qcgc_shadow_stack_top(shadow_stack_t *stack) {
-#if CHECKED
-	assert(stack->count != 0);
-#endif
-	return stack->items[stack->count - 1];
-}
-
-shadow_stack_t *qcgc_shadow_stack_pop(shadow_stack_t *stack) {
-	// TODO: Add lower bound for size (config.h)
-	if (stack->count < stack->size / 4) {
-		stack = shadow_stack_shrink(stack);
-	}
-	stack->count--;
-	return stack;
-}
-
-QCGC_STATIC size_t shadow_stack_size(size_t size) {
-	return (sizeof(shadow_stack_t) + size * sizeof(object_t *));
-}
-
-QCGC_STATIC shadow_stack_t *shadow_stack_grow(shadow_stack_t *stack) {
-	stack = (shadow_stack_t *) realloc(stack, shadow_stack_size(stack->size * 2));
-	stack->size *= 2;
-	return stack;
-}
-
-QCGC_STATIC shadow_stack_t *shadow_stack_shrink(shadow_stack_t *stack) {
-	stack = (shadow_stack_t *) realloc(stack, shadow_stack_size(stack->size / 2));
-	stack->size /= 2;
-	return stack;
-}
diff --git a/rpython/translator/c/src/qcgc/src/shadow_stack.h b/rpython/translator/c/src/qcgc/src/shadow_stack.h
deleted file mode 100644
--- a/rpython/translator/c/src/qcgc/src/shadow_stack.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-#include "../qcgc.h"
-
-typedef struct shadow_stack_s {
-	size_t count;
-	size_t size;
-	object_t *items[];
-} shadow_stack_t;
-
-__attribute__ ((warn_unused_result))
-shadow_stack_t *qcgc_shadow_stack_create(size_t size);
-
-__attribute__ ((warn_unused_result))
-shadow_stack_t *qcgc_shadow_stack_push(shadow_stack_t *stack, object_t *item);
-
-object_t *qcgc_shadow_stack_top(shadow_stack_t *stack);
-
-__attribute__ ((warn_unused_result))
-shadow_stack_t *qcgc_shadow_stack_pop(shadow_stack_t *stack);
diff --git a/rpython/translator/c/src/qcgc/src/signal_handler.c b/rpython/translator/c/src/qcgc/src/signal_handler.c
--- a/rpython/translator/c/src/qcgc/src/signal_handler.c
+++ b/rpython/translator/c/src/qcgc/src/signal_handler.c
@@ -43,7 +43,7 @@
 }
 
 QCGC_STATIC bool is_stack_overflow(void *addr) {
-	void *shadow_stack_end = (void *)(qcgc_shadowstack.base +
+	void *shadow_stack_end = (void *)(_qcgc_shadowstack.base +
 		QCGC_SHADOWSTACK_SIZE);
 	return (addr >= shadow_stack_end && addr < shadow_stack_end + 8192);
 }

From pypy.commits at gmail.com  Fri Sep 16 05:38:55 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 02:38:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix bug in codewriter about passing the
 exitswitch variable to a call
Message-ID: <57dbbdaf.e2cdc20a.3739e.9ac7@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r87129:9e6653e203d8
Date: 2016-09-16 11:38 +0200
http://bitbucket.org/pypy/pypy/changeset/9e6653e203d8/

Log:	fix bug in codewriter about passing the exitswitch variable to a
	call

	this kind of code broke stuff: v = x is None f(v) if v: ...

	because the "if v in op.args" condition didn't trigger for the call,
	since call args have an indirection

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -200,8 +200,12 @@
             or v.concretetype != lltype.Bool):
             return False
         for op in block.operations[::-1]:
-            if v in op.args:
-                return False   # variable is also used in cur block
+            # check if variable is used in block
+            for arg in op.args:
+                if arg == v:
+                    return False
+                if isinstance(arg, ListOfKind) and v in arg.content:
+                    return False
             if v is op.result:
                 if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne',
                                      'int_gt', 'int_ge',
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -243,6 +243,20 @@
         assert block.exitswitch == (opname, v1, '-live-before')
         assert block.exits == exits
 
+def test_optimize_goto_if_not__argument_to_call():
+    for opname in ['ptr_iszero', 'ptr_nonzero']:
+        v1 = Variable()
+        v3 = Variable(); v3.concretetype = lltype.Bool
+        v4 = Variable()
+        block = Block([v1])
+        callop = SpaceOperation('residual_call_r_i',
+                ["fake", ListOfKind('int', [v3])], v4)
+        block.operations = [SpaceOperation(opname, [v1], v3), callop]
+        block.exitswitch = v3
+        block.exits = exits = [FakeLink(False), FakeLink(True)]
+        res = Transformer().optimize_goto_if_not(block)
+        assert not res
+
 def test_symmetric():
     ops = {'int_add': 'int_add',
            'int_or': 'int_or',

From pypy.commits at gmail.com  Fri Sep 16 05:42:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 02:42:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: in-progress
Message-ID: <57dbbe93.04141c0a.25833.5208@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87130:d4e91edb572f
Date: 2016-09-16 11:42 +0200
http://bitbucket.org/pypy/pypy/changeset/d4e91edb572f/

Log:	in-progress

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -167,9 +167,10 @@
         return self.throw(w_type, w_val, w_tb)
 
     def _get_yield_from(self):
-        # Probably a hack (but CPython has the same):
+        # Probably a hack (but CPython has the same, _PyGen_yf()):
         # If the current frame is stopped in a "yield from",
         # return the paused generator.
+        # XXX this is probably very bad for the JIT.  Think again!
         if not self.frame:
             return None
         co_code = self.frame.pycode.co_code
@@ -185,26 +186,29 @@
             # Paused in a "yield from", pass the throw to the inner generator.
             return self._throw_delegate(space, w_yf, w_type, w_val, w_tb)
         else:
-            # Not paused in a "yield from", quit this generator
+            # Not paused in a "yield from", throw inside this generator
             return self._throw_here(space, w_type, w_val, w_tb)
 
     def _throw_delegate(self, space, w_yf, w_type, w_val, w_tb):
-        if space.is_w(w_type, space.w_GeneratorExit):
+        if space.exception_match(w_type, space.w_GeneratorExit):
             try:
-                w_close = space.getattr(w_yf, space.wrap("close"))
-            except OperationError as e:
-                if not e.match(space, space.w_AttributeError):
-                    e.write_unraisable(space, "generator.close()")
-            else:
                 self.running = True
                 try:
-                    space.call_function(w_close)
-                except OperationError as operr:
-                    self.running = False
-                    return self.send_ex(space.w_None, operr)
+                    gen_close_iter(space, w_yf)
                 finally:
                     self.running = False
+            except OperationError as e:
+                return self.send_ex(space.w_None, e)
             return self._throw_here(space, w_type, w_val, w_tb)
+        #
+        if isinstance(w_yf, GeneratorIterator):
+            self.running = True
+            try:
+                return w_yf.throw(space, w_type, w_val, w_tb)
+            except OperationError as e:
+                operr = e
+            finally:
+                self.running = False
         else:
             try:
                 w_throw = space.getattr(w_yf, space.wrap("throw"))
@@ -215,22 +219,22 @@
             self.running = True
             try:
                 return space.call_function(w_throw, w_type, w_val, w_tb)
-            except OperationError as operr:
-                self.running = False
-                # Pop subiterator from stack.
-                w_subiter = self.frame.popvalue()
-                assert space.is_w(w_subiter, w_yf)
-                # Termination repetition of YIELD_FROM
-                self.frame.last_instr += 1
-                if operr.match(space, space.w_StopIteration):
-                    operr.normalize_exception(space)
-                    w_val = space.getattr(operr.get_w_value(space),
-                                          space.wrap("value"))
-                    return self.send_ex(w_val)
-                else:
-                    return self.send_ex(space.w_None, operr)
+            except OperationError as e:
+                operr = e
             finally:
                 self.running = False
+        # Pop subiterator from stack.
+        w_subiter = self.frame.popvalue()
+        assert space.is_w(w_subiter, w_yf)
+        # Termination repetition of YIELD_FROM
+        self.frame.last_instr += 1
+        if operr.match(space, space.w_StopIteration):
+            operr.normalize_exception(space)
+            w_val = space.getattr(operr.get_w_value(space),
+                                  space.wrap("value"))
+            return self.send_ex(w_val)
+        else:
+            return self.send_ex(space.w_None, operr)
 
     def _throw_here(self, space, w_type, w_val, w_tb):
         from pypy.interpreter.pytraceback import check_traceback
@@ -252,12 +256,22 @@
 
     def descr_close(self):
         """close() -> raise GeneratorExit inside generator/coroutine."""
-        _PyGen_yf()....
         if self.frame is None:
             return     # nothing to do in this case
         space = self.space
+        w_yf = self._get_yield_from()
+        operr = OperationError(space.w_GeneratorExit,
+                               space.call_function(space.w_GeneratorExit))
+        if w_yf is not None:
+            self.running = True
+            try:
+                gen_close_iter(space, w_yf)
+            except OperationError as e:
+                operr = e
+            finally:
+                self.running = False
         try:
-            w_retval = self.send_ex..?..(space.w_GeneratorExit..)
+            w_retval = self.send_ex(space.w_None, operr)
         except OperationError as e:
             if e.match(space, space.w_StopIteration) or \
                     e.match(space, space.w_GeneratorExit):
@@ -397,6 +411,22 @@
         return self
 
 
+def gen_close_iter(space, w_yf):
+    # This helper function is used by close() and throw() to
+    # close a subiterator being delegated to by yield-from.
+    if isinstance(w_yf, GeneratorIterator):
+        w_yf.descr_close()
+    else:
+        try:
+            w_close = space.getattr(w_yf, space.wrap("close"))
+        except OperationError as e:
+            if not e.match(space, space.w_AttributeError):
+                # aaaaaaaah but that's what CPython does too
+                e.write_unraisable(space, "generator/coroutine.close()")
+        else:
+            space.call_function(w_close)
+
+
 def get_printable_location_genentry(bytecode):
     return '%s <generator>' % (bytecode.get_repr(),)
 generatorentry_driver = jit.JitDriver(greens=['pycode'],

From pypy.commits at gmail.com  Fri Sep 16 08:42:09 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 16 Sep 2016 05:42:09 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: a nearly complete vector
 backend for s390x
Message-ID: <57dbe8a1.442d1c0a.ae58.a0d3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87133:22a63dae82b3
Date: 2016-09-16 14:41 +0200
http://bitbucket.org/pypy/pypy/changeset/22a63dae82b3/

Log:	a nearly complete vector backend for s390x

diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -62,19 +62,6 @@
     # TODO VEC_DOUBLE_WORD_ONES = 0
 
     def setup_once_vector(self):
-        # TODO if IS_BIG_ENDIAN:
-        # TODO     # 2x 64 bit signed integer(1) BE
-        # TODO     data = (b'\x00' * 7 + b'\x01') * 2
-        # TODO else:
-        # TODO     # 2x 64 bit signed integer(1) LE
-        # TODO     data = (b'\x01' + b'\x00' * 7) * 2
-        # TODO datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, [])
-        # TODO mem = datablockwrapper.malloc_aligned(len(data), alignment=16)
-        # TODO datablockwrapper.done()
-        # TODO addr = rffi.cast(rffi.CArrayPtr(lltype.Char), mem)
-        # TODO for i in range(len(data)):
-        # TODO     addr[i] = data[i]
-        # TODO self.VEC_DOUBLE_WORD_ONES = mem
         pass
 
     def emit_vec_load_f(self, op, arglocs, regalloc):
@@ -195,21 +182,12 @@
 
     def emit_vec_int_is_true(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        # TODO
         resloc, argloc, sizeloc = arglocs
         size = sizeloc.value
-        tmp = regalloc.vrm.get_scratch_reg(type=INT).value
-        self.mc.vxor(tmp, tmp, tmp)
-        # argloc[i] > 0:
-        # For an unsigned integer that is equivalent to argloc[i] != 0
-        if size == 1:
-            self.mc.vcmpgtubx(resloc.value, argloc.value, tmp)
-        elif size == 2:
-            self.mc.vcmpgtuhx(resloc.value, argloc.value, tmp)
-        elif size == 4:
-            self.mc.vcmpgtuwx(resloc.value, argloc.value, tmp)
-        elif size == 8:
-            self.mc.vcmpgtudx(resloc.value, argloc.value, tmp)
+        tmploc = regalloc.vrm.get_scratch_reg()
+        self.mc.VX(tmploc, tmploc, tmploc) # all zero
+        self.mc.VNO(tmploc, tmploc, tmploc) # all one
+        self.mc.VCEQ(resloc, argloc, tmploc, l.itemsize_to_mask(size), 0b0001)
         flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc)
 
     def emit_vec_float_eq(self, op, arglocs, regalloc):
@@ -596,8 +574,8 @@
     prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int
 
     def prepare_vec_guard_true(self, op):
-        self.assembler.guard_success_cc = c.VEQ
+        self.assembler.guard_success_cc = c.VEQI
         return self._prepare_guard(op)
 
     def prepare_vec_guard_false(self, op):
-        self.assembler.guard_success_cc = c.VNE
+        self.assembler.guard_success_cc = c.VNEI

From pypy.commits at gmail.com  Fri Sep 16 08:42:05 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 16 Sep 2016 05:42:05 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: some more pack/unpack cases
 implemented
Message-ID: <57dbe89d.eb01c30a.ef930.f1af@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87131:2cd9f79ff7de
Date: 2016-09-16 11:59 +0200
http://bitbucket.org/pypy/pypy/changeset/2cd9f79ff7de/

Log:	some more pack/unpack cases implemented

diff --git a/rpython/jit/backend/zarch/instruction_builder.py b/rpython/jit/backend/zarch/instruction_builder.py
--- a/rpython/jit/backend/zarch/instruction_builder.py
+++ b/rpython/jit/backend/zarch/instruction_builder.py
@@ -546,6 +546,30 @@
         self.writechar(opcode2)
     return encode_vri_a
 
+def build_vrs_b(mnemonic, (opcode1,opcode2), argtypes='v,r,db,m'):
+    @builder.arguments(argtypes)
+    def encode_vrs_b(self, v1, r2, db3, m4):
+        self.writechar(opcode1)
+        rbx = (v1 >= 16) << 3
+        byte = (v1 & BIT_MASK_4) << 4 | (r2 & BIT_MASK_4)
+        self.writechar(chr(byte))
+        encode_base_displace(self, db3)
+        self.writechar(chr((m4 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
+        self.writechar(opcode2)
+    return encode_vrs_b
+
+def build_vrs_c(mnemonic, (opcode1,opcode2), argtypes='r,v,db,m'):
+    @builder.arguments(argtypes)
+    def encode_vrs_c(self, r1, v2, db3, m4):
+        self.writechar(opcode1)
+        rbx = (v2 >= 16) << 2
+        byte = (r1 & BIT_MASK_4) << 4 | (v2 & BIT_MASK_4)
+        self.writechar(chr(byte))
+        encode_base_displace(self, db3)
+        self.writechar(chr((m4 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
+        self.writechar(opcode2)
+    return encode_vrs_c
+
 
 def build_unpack_func(mnemonic, func):
     @always_inline
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -334,6 +334,8 @@
     'VMRL':  ('vrr_c', ['\xE7','\x60'], 'v,v,v,m'),
     'VMRH':  ('vrr_c', ['\xE7','\x61'], 'v,v,v,m'),
     'VPDI':  ('vrr_c', ['\xE7','\x84'], 'v,v,v,m'),
+    'VLVG':  ('vrs_b', ['\xE7','\x22']),
+    'VLGV':  ('vrs_c', ['\xE7','\x21']),
 
     # '': ('', ['','']),
 }
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -263,24 +263,6 @@
         # 4 => bit 1 from the MSB: XxC
         self.mc.VCGD(resloc, loc0, 3, 4, mask.RND_TOZERO.value)
 
-    def emit_vec_expand_f(self, op, arglocs, regalloc):
-        assert isinstance(op, VectorOp)
-        resloc, srcloc = arglocs
-        size = op.bytesize
-        res = resloc.value
-        if isinstance(srcloc, l.ConstFloatLoc):
-            # they are aligned!
-            assert size == 8
-            tloc = regalloc.rm.get_scratch_reg()
-            self.mc.load_imm(tloc, srcloc.value)
-            self.mc.lxvd2x(res, 0, tloc.value)
-        elif size == 8:
-            # splat the low of src to both slots in res
-            src = srcloc.value
-            self.mc.xxspltdl(res, src, src)
-        else:
-            not_implemented("vec expand in this combination not supported")
-
     def emit_vec_expand_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, loc0 = arglocs
@@ -292,13 +274,12 @@
     def _accum_reduce(self, op, arg, accumloc, targetloc):
         # Currently the accumulator can ONLY be 64 bit float/int
         if arg.type == FLOAT:
-            # r = (r[0]+r[1],r[0]+r[1])
-            self.mc.VMRL(targetloc, accumloc, accumloc, l.MASK_VEC_DWORD)
+            self.mc.VPDI(targetloc, accumloc, accumloc, permi(1,0))
             if op == '+':
-                self.mc.VFA(targetloc, targetloc, accumloc, 3, 0, 0)
+                self.mc.VFA(targetloc, targetloc, accumloc, 3, 0b1000, 0)
                 return
             elif op == '*':
-                self.mc.VFM(targetloc, targetloc, accumloc, 3, 0, 0)
+                self.mc.VFM(targetloc, targetloc, accumloc, 3, 0b1000, 0)
                 return
         else:
             assert arg.type == INT
@@ -317,41 +298,14 @@
 
     def emit_vec_pack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs
-        srcidx = srcidxloc.value
+        resloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs
         residx = residxloc.value
         count = countloc.value
-        res = resultloc.value
-        vector = vloc.value
-        src = sourceloc.value
         size = op.bytesize
         assert resultloc.is_vector_reg() # vector <- reg
-        self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
-        self.mc.stvx(vector, r.SCRATCH2.value, r.SP.value)
-        idx = residx
-        if size == 8:
-            if not IS_BIG_ENDIAN:
-                idx = (16 // size) - 1 - idx
-            self.mc.store(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+8*idx)
-        elif size == 4:
-            for j in range(count):
-                idx = j + residx
-                if not IS_BIG_ENDIAN:
-                    idx = (16 // size) - 1 - idx
-                self.mc.stw(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+4*idx)
-        elif size == 2:
-            for j in range(count):
-                idx = j + residx
-                if not IS_BIG_ENDIAN:
-                    idx = (16 // size) - 1 - idx
-                self.mc.sth(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+2*idx)
-        elif size == 1:
-            for j in range(count):
-                idx = j + residx
-                if not IS_BIG_ENDIAN:
-                    idx = (16 // size) - 1 - idx
-                self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+idx)
-        self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+        for j in range(count):
+            index = l.addr(j + residx)
+            self.mc.VLVG(resloc, sourceloc, index, l.itemsize_to_mask(size))
 
     def emit_vec_unpack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
@@ -364,44 +318,26 @@
         if count == 1:
             assert srcloc.is_vector_reg()
             assert not resloc.is_vector_reg()
-            off = PARAM_SAVE_AREA_OFFSET
-            self.mc.load_imm(r.SCRATCH2, off)
-            self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
-            if not IS_BIG_ENDIAN:
-                idx = (16 // size) - 1 - idx
-            off += size * idx
-            if size == 8:
-                self.mc.load(res, r.SP.value, off)
-                return
-            elif size == 4:
-                self.mc.lwa(res, r.SP.value, off)
-                return
-            elif size == 2:
-                self.mc.lha(res, r.SP.value, off)
-                return
-            elif size == 1:
-                self.mc.lbz(res, r.SP.value, off)
-                self.mc.extsb(res, res)
-                return
+            self.mc.VLGV(resloc, srcloc, index, l.itemsize_to_mask(size))
         else:
             # count is not 1, but only 2 is supported for i32
             # 4 for i16 and 8 for i8.
             src = srcloc.value
             res = resloc.value
 
-            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
-            self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
-            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
-            self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
+            #self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
+            #self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
+            #self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
+            #self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
             if count * size == 8:
                 if not IS_BIG_ENDIAN:
                     endian_off = 8
-                off = PARAM_SAVE_AREA_OFFSET
-                off = off + endian_off - (idx * size)
-                assert idx * size + 8 <= 16
-                self.mc.load(r.SCRATCH.value, r.SP.value, off)
-                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
-                self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+                #off = PARAM_SAVE_AREA_OFFSET
+                #off = off + endian_off - (idx * size)
+                #assert idx * size + 8 <= 16
+                #self.mc.load(r.SCRATCH.value, r.SP.value, off)
+                #self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
+                #self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
                 return
 
         not_implemented("%d bit integer, count %d" % \
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -402,10 +402,10 @@
 
         bits = 64
         la = data.draw(st.lists(strat, min_size=10, max_size=150))
-        #la = [1.0] * 10
+        la = [1.0] * 10
         l = len(la)
 
-        accum = 0 #data.draw(strat)
+        accum = data.draw(strat)
         rawstorage = RawStorage()
         va = rawstorage.new(la, type)
         res = self.meta_interp(f, [accum, l*size, va])

From pypy.commits at gmail.com  Fri Sep 16 08:42:07 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 16 Sep 2016 05:42:07 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: implemented all vectro
 resops for s390x
Message-ID: <57dbe89f.e2cdc20a.3739e.e74d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87132:7801b4271e3e
Date: 2016-09-16 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/7801b4271e3e/

Log:	implemented all vectro resops for s390x

diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -298,50 +298,34 @@
 
     def emit_vec_pack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
-        resloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs
+        resloc, vecloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
         residx = residxloc.value
+        srcidx = srcidxloc.value
         count = countloc.value
-        size = op.bytesize
-        assert resultloc.is_vector_reg() # vector <- reg
-        for j in range(count):
-            index = l.addr(j + residx)
-            self.mc.VLVG(resloc, sourceloc, index, l.itemsize_to_mask(size))
+        size = sizeloc.value
+        if count == 1:
+            if resloc.is_core_reg():
+                assert sourceloc.is_vector_reg()
+                index = l.addr(srcidx)
+                self.mc.VLGV(resloc, sourceloc, index, l.itemsize_to_mask(size))
+            else:
+                assert sourceloc.is_core_reg()
+                assert resloc.is_vector_reg()
+                index = l.addr(residx)
+                self.mc.VLR(resloc, vecloc)
+                self.mc.VLVG(resloc, sourceloc, index, l.itemsize_to_mask(size))
+        else:
+            assert resloc.is_vector_reg()
+            assert sourceloc.is_vector_reg()
+            self.mc.VLR(resloc, vecloc)
+            for j in range(count):
+                sindex = l.addr(j + srcidx)
+                # load from sourceloc into GP reg and store back into resloc
+                self.mc.VLGV(r.SCRATCH, sourceloc, sindex, l.itemsize_to_mask(size))
+                rindex = l.addr(j + residx)
+                self.mc.VLVG(resloc, sourceloc, rindex, l.itemsize_to_mask(size))
 
-    def emit_vec_unpack_i(self, op, arglocs, regalloc):
-        assert isinstance(op, VectorOp)
-        resloc, srcloc, idxloc, countloc, sizeloc = arglocs
-        idx = idxloc.value
-        res = resloc.value
-        src = srcloc.value
-        size = sizeloc.value
-        count = countloc.value
-        if count == 1:
-            assert srcloc.is_vector_reg()
-            assert not resloc.is_vector_reg()
-            self.mc.VLGV(resloc, srcloc, index, l.itemsize_to_mask(size))
-        else:
-            # count is not 1, but only 2 is supported for i32
-            # 4 for i16 and 8 for i8.
-            src = srcloc.value
-            res = resloc.value
-
-            #self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
-            #self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
-            #self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
-            #self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            if count * size == 8:
-                if not IS_BIG_ENDIAN:
-                    endian_off = 8
-                #off = PARAM_SAVE_AREA_OFFSET
-                #off = off + endian_off - (idx * size)
-                #assert idx * size + 8 <= 16
-                #self.mc.load(r.SCRATCH.value, r.SP.value, off)
-                #self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
-                #self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
-                return
-
-        not_implemented("%d bit integer, count %d" % \
-                       (size*8, count))
+    emit_vec_unpack_i = emit_vec_pack_i
 
     def emit_vec_pack_f(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
@@ -517,12 +501,33 @@
         count = op.getarg(3)
         assert isinstance(index, ConstInt)
         assert isinstance(count, ConstInt)
-        vloc = self.ensure_vector_reg(op.getarg(0))
+        arg0 = op.getarg(0)
+        assert isinstance(arg0, VectorOp)
+        vloc = self.ensure_vector_reg(arg0)
         srcloc = self.ensure_reg(arg)
         resloc = self.force_allocate_vector_reg(op)
         residx = index.value # where to put it in result?
         srcidx = 0
-        return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value)]
+        return [resloc, vloc, srcloc, imm(residx), imm(srcidx), imm(count.value), imm(arg0.bytesize)]
+
+    def prepare_vec_unpack_i(self, op):
+        assert isinstance(op, VectorOp)
+        index = op.getarg(1)
+        count = op.getarg(2)
+        assert isinstance(index, ConstInt)
+        assert isinstance(count, ConstInt)
+        arg = op.getarg(0)
+        if arg.is_vector():
+            srcloc = self.ensure_vector_reg(arg)
+        else:
+            # unpack
+            srcloc = self.ensure_reg(arg0)
+        if op.is_vector():
+            resloc = self.force_allocate_vector_reg(op)
+        else:
+            resloc = self.force_allocate_reg(op)
+        size = arg.bytesize
+        return [resloc, srcloc, srcloc, imm(0), imm(index.value), imm(count.value), imm(size)]
 
     def prepare_vec_pack_f(self, op):
         # new_res = vec_pack_f(res, src, index, count)
@@ -552,25 +557,6 @@
         resloc = self.force_allocate_reg(op)
         return [resloc, srcloc, imm(index.value), imm(count.value)]
 
-    def prepare_vec_unpack_i(self, op):
-        assert isinstance(op, VectorOp)
-        index = op.getarg(1)
-        count = op.getarg(2)
-        assert isinstance(index, ConstInt)
-        assert isinstance(count, ConstInt)
-        arg = op.getarg(0)
-        if arg.is_vector():
-            srcloc = self.ensure_vector_reg(arg)
-        else:
-            # unpack
-            srcloc = self.ensure_reg(arg0)
-        size = arg.bytesize
-        if op.is_vector():
-            resloc = self.force_allocate_vector_reg(op)
-        else:
-            resloc = self.force_allocate_reg(op)
-        return [resloc, srcloc, imm(index.value), imm(count.value), imm(size)]
-
     def expand_float(self, size, box):
         adr = self.assembler.datablockwrapper.malloc_aligned(16, 16)
         fs = box.getfloatstorage()

From pypy.commits at gmail.com  Fri Sep 16 09:57:06 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 16 Sep 2016 06:57:06 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: test,
 add a unicode-specific getreadbuffer
Message-ID: <57dbfa32.a7bec20a.eaa4d.0890@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87134:ae44485a5595
Date: 2016-09-15 21:57 +0300
http://bitbucket.org/pypy/pypy/changeset/ae44485a5595/

Log:	test, add a unicode-specific getreadbuffer

diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -105,5 +105,7 @@
         # Not really part of array, refactor
         import struct
         module = self.import_module(name='array')
+        val = module.readbuffer_as_string('abcd')
+        assert val == 'abcd'
         val = module.readbuffer_as_string(u'\u03a3')
-        assert val.decode('utf32') == u'\u03a3'
+        assert val is not None
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -553,6 +553,20 @@
     Py_DecRef(space, pyref)
     return space.len_w(w_str)
 
+ at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
+             header=None, error=-1)
+def unicode_getreadbuffer(space, w_str, segment, ref):
+    from pypy.module.cpyext.unicodeobject import (
+                PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
+    if segment != 0:
+        raise oefmt(space.w_SystemError,
+                    "accessing non-existent unicode segment")
+    pyref = make_ref(space, w_str)
+    ref[0] = PyUnicode_AS_UNICODE(space, pyref)
+    # Stolen reference: the object has better exist somewhere else
+    Py_DecRef(space, pyref)
+    return PyUnicode_GET_DATA_SIZE(space, w_str)
+
 @cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
              header=None, error=-1)
 def str_getcharbuffer(space, w_buf, segment, ref):
@@ -576,8 +590,8 @@
 
 def setup_buffer_procs(space, w_type, pto):
     bufspec = w_type.layout.typedef.buffer
-    if bufspec is None:
-        # not a buffer
+    if bufspec is None and not space.is_w(w_type, space.w_unicode):
+        # not a buffer, but let w_unicode be a read buffer
         return
     c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True)
     lltype.render_immortal(c_buf)
@@ -593,6 +607,13 @@
         c_buf.c_bf_getcharbuffer = llhelper(
             str_getcharbuffer.api_func.functype,
             str_getcharbuffer.api_func.get_wrapper(space))
+    elif space.is_w(w_type, space.w_unicode):
+        # Special case: unicode doesn't support get_raw_address(), so we have a
+        # custom get*buffer that instead gives the address of the char* in the
+        # PyUnicodeObject*!
+        c_buf.c_bf_getreadbuffer = llhelper(
+            unicode_getreadbuffer.api_func.functype,
+            unicode_getreadbuffer.api_func.get_wrapper(space))
     elif space.is_w(w_type, space.w_buffer):
         # Special case: we store a permanent address on the cpyext wrapper,
         # so we'll reuse that.
@@ -708,7 +729,7 @@
     # uninitialized fields:
     # c_tp_print
     # XXX implement
-    # c_tp_compare and the following fields (see http://docs.python.org/c-api/typeobj.html )
+    # c_tp_compare and more?
     w_base = best_base(space, w_type.bases_w)
     pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base))
 
@@ -766,7 +787,6 @@
     return find_best_base(bases_w)
 
 def inherit_slots(space, pto, w_base):
-    # XXX missing: nearly everything
     base_pyo = make_ref(space, w_base)
     try:
         base = rffi.cast(PyTypeObjectPtr, base_pyo)
@@ -785,11 +805,13 @@
             pto.c_tp_getattro = base.c_tp_getattro
         if not pto.c_tp_as_buffer:
             pto.c_tp_as_buffer = base.c_tp_as_buffer
-        # XXX need to refactor: what about built-in objects i.e. W_Unicode
         if base.c_tp_as_buffer:
             # also inherit all the base.c_tp_as_buffer functions, since they
             # do not have real __slots__ they are not filled in otherwise
+            #
             # skip bf_getbuffer, which is __buffer__
+            #
+            # note: builtin types are handled in setup_buffer_procs
             pto_as = pto.c_tp_as_buffer
             base_as = base.c_tp_as_buffer
             if not pto_as.c_bf_getreadbuffer:

From pypy.commits at gmail.com  Fri Sep 16 09:57:08 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 16 Sep 2016 06:57:08 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: add cpyext-specific
 __rbuffer__, __wbuffer__ slots for old buffer interface
Message-ID: <57dbfa34.aa52c20a.f1460.0e8f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87135:64f607343212
Date: 2016-09-16 16:55 +0300
http://bitbucket.org/pypy/pypy/changeset/64f607343212/

Log:	add cpyext-specific __rbuffer__, __wbuffer__ slots for old buffer
	interface

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -207,6 +207,12 @@
 
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
+        if w_impl is None:
+            # cpyext types that may have only old buffer interface
+            if flags & space.BUF_WRITABLE:
+                w_impl = space.lookup(self, '__wbuffer__')
+            else:
+                w_impl = space.lookup(self, '__wbuffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self, 
                                         space.newint(flags))
@@ -215,7 +221,10 @@
         raise BufferInterfaceNotFound
 
     def readbuf_w(self, space):
-        w_impl = space.lookup(self, '__buffer__')
+        # cpyext types that may have old buffer protocol
+        w_impl = space.lookup(self, '__rbuffer__')
+        if w_impl is None:
+            w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
@@ -224,7 +233,10 @@
         raise BufferInterfaceNotFound
 
     def writebuf_w(self, space):
-        w_impl = space.lookup(self, '__buffer__')
+        # cpyext types that may have old buffer protocol
+        w_impl = space.lookup(self, '__wbuffer__')
+        if w_impl is None:
+            w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL))
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -344,6 +344,10 @@
     def getndim(self):
         return self.ndim
 
+    def setitem(self, index, char):
+        # absolutely no safety checks, what could go wrong?
+        self.ptr[index] = char
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
@@ -353,6 +357,15 @@
             space.fromcache(State).check_and_raise_exception(always=True)
         return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
 
+def wrap_getwritebuffer(space, w_self, w_args, func):
+    func_target = rffi.cast(readbufferproc, func)
+    with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
+        index = rffi.cast(Py_ssize_t, 0)
+        size = generic_cpy_call(space, func_target, w_self, index, ptr)
+        if size < 0:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        return space.newbuffer(CPyBuffer(ptr[0], size, w_self, readonly=False))
+
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
     with lltype.scoped_alloc(Py_buffer) as pybuf:
@@ -919,13 +932,13 @@
 slotdefs = eval(slotdefs_str)
 # PyPy addition
 slotdefs += (
-    # XXX that might not be what we want!
     TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""),
 )
 
 if not PY3:
     slotdefs += (
-        TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+        TPSLOT("__rbuffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""),
+        TPSLOT("__wbuffer__", "tp_as_buffer.c_bf_getwritebuffer", None, "wrap_getwritebuffer", ""),
     )
 
 
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -1,8 +1,16 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from pypy.conftest import option
 
 class AppTestArrayModule(AppTestCpythonExtensionBase):
-    enable_leak_checking = False
+
+    def setup_class(cls):
+        from rpython.tool.udir import udir
+        if option.runappdirect:
+            cls.w_udir = str(udir)
+        else:
+            cls.w_udir = cls.space.wrap(str(udir))
+
+    #enable_leak_checking = False
 
     def test_basic(self):
         module = self.import_module(name='array')
@@ -109,3 +117,26 @@
         assert val == 'abcd'
         val = module.readbuffer_as_string(u'\u03a3')
         assert val is not None
+
+    def test_readinto(self):
+        module = self.import_module(name='array')
+        a = module.array('c')
+        a.fromstring('0123456789')
+        filename = self.udir + "/_test_file"
+        f = open(filename, 'w+b')
+        f.write('foobar')
+        f.seek(0)
+        n = f.readinto(a)
+        f.close()
+        assert n == 6
+        assert len(a) == 10
+        assert a.tostring() == 'foobar6789'
+
+    def test_iowrite(self):
+        module = self.import_module(name='array')
+        from io import BytesIO
+        a = module.array('c')
+        a.fromstring('0123456789')
+        fd = BytesIO()
+        # only test that it works
+        fd.write(a)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -806,23 +806,23 @@
         if not pto.c_tp_as_buffer:
             pto.c_tp_as_buffer = base.c_tp_as_buffer
         if base.c_tp_as_buffer:
-            # also inherit all the base.c_tp_as_buffer functions, since they
-            # do not have real __slots__ they are not filled in otherwise
-            #
-            # skip bf_getbuffer, which is __buffer__
-            #
+            # inherit base.c_tp_as_buffer functions not inherited from w_type
             # note: builtin types are handled in setup_buffer_procs
             pto_as = pto.c_tp_as_buffer
             base_as = base.c_tp_as_buffer
+            if not pto_as.c_bf_getbuffer:
+                pto_as.c_bf_getbuffer = base_as.c_bf_getbuffer
+            if not pto_as.c_bf_getcharbuffer:
+                pto_as.c_bf_getcharbuffer = base_as.c_bf_getcharbuffer
+            if not pto_as.c_bf_getwritebuffer:
+                pto_as.c_bf_getwritebuffer = base_as.c_bf_getwritebuffer
             if not pto_as.c_bf_getreadbuffer:
                 pto_as.c_bf_getreadbuffer = base_as.c_bf_getreadbuffer
-            if not pto_as.c_bf_getwritebuffer:
-                pto_as.c_bf_getwritebuffer = base_as.c_bf_getwritebuffer
             if not pto_as.c_bf_getsegcount:
                 pto_as.c_bf_getsegcount = base_as.c_bf_getsegcount
             if not pto_as.c_bf_getcharbuffer:
                 pto_as.c_bf_getcharbuffer = base_as.c_bf_getcharbuffer
-            if not pto_as.c_bf_getcharbuffer:
+            if not pto_as.c_bf_releasebuffer:
                 pto_as.c_bf_releasebuffer = base_as.c_bf_releasebuffer
     finally:
         Py_DecRef(space, base_pyo)
@@ -853,13 +853,14 @@
 
     w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
     track_reference(space, py_obj, w_obj)
-    w_obj.__init__(space, py_type)
+    # __init__ wraps all slotdefs functions from py_type via add_operators
+    w_obj.__init__(space, py_type) 
     w_obj.ready()
 
     finish_type_2(space, py_type, w_obj)
-    # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
+        # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots
         if not py_type.c_tp_as_number: 
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
@@ -868,7 +869,7 @@
             py_type.c_tp_as_sequence = base.c_tp_as_sequence
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
-        if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
+        #if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
 
     return w_obj
 
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -32,6 +32,10 @@
     def charbuf_w(self, space):
         return self.buf.as_str()
 
+    def descr_getbuffer(self, space, w_flags):
+        space.check_buf_flags(space.int_w(w_flags), self.buf.readonly)
+        return self
+
     @staticmethod
     @unwrap_spec(offset=int, size=int)
     def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1):
@@ -160,6 +164,7 @@
     __mul__ = interp2app(W_Buffer.descr_mul),
     __rmul__ = interp2app(W_Buffer.descr_mul),
     __repr__ = interp2app(W_Buffer.descr_repr),
+    __buffer__ = interp2app(W_Buffer.descr_getbuffer),
     _pypy_raw_address = interp2app(W_Buffer.descr_pypy_raw_address),
 )
 W_Buffer.typedef.acceptable_as_base_class = False

From pypy.commits at gmail.com  Fri Sep 16 10:00:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 07:00:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Minor test and fix
Message-ID: <57dbfaf5.8a13c20a.e8505.0646@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87136:f134a77f5293
Date: 2016-09-16 15:59 +0200
http://bitbucket.org/pypy/pypy/changeset/f134a77f5293/

Log:	Minor test and fix

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -63,7 +63,7 @@
         """x.__iter__() <==> iter(x)"""
         return self.space.wrap(self)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -221,7 +221,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():

From pypy.commits at gmail.com  Fri Sep 16 10:10:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 07:10:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix tests (test_throw1 and test_throw2
 were identical at this point)
Message-ID: <57dbfd73.57981c0a.7bf0f.c846@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87137:f6eabcc3367c
Date: 2016-09-16 16:10 +0200
http://bitbucket.org/pypy/pypy/changeset/f6eabcc3367c/

Log:	Fix tests (test_throw1 and test_throw2 were identical at this point)

diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,6 +57,7 @@
         def f():
             yield 2
         g = f()
+        g.next()
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
@@ -64,6 +65,7 @@
             yield 2
         g = f()
         raises(NameError, g.throw, NameError("Error"))
+        raises(StopIteration, g.next)
 
     def test_throw3(self):
         def f():

From pypy.commits at gmail.com  Fri Sep 16 10:17:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 07:17:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: rewrite rewrite rewrite in
 progress
Message-ID: <57dbfef5.e17cc20a.65bd9.1235@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87138:7b5b1a370414
Date: 2016-09-16 16:16 +0200
http://bitbucket.org/pypy/pypy/changeset/7b5b1a370414/

Log:	rewrite rewrite rewrite in progress

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.pyopcode import LoopBlock
+from pypy.interpreter.pyopcode import LoopBlock, SApplicationException
 from pypy.interpreter.pycode import CO_YIELD_INSIDE_TRY
 from pypy.interpreter.astcompiler import consts
 from rpython.rlib import jit
@@ -9,6 +9,9 @@
 class GeneratorOrCoroutine(W_Root):
     _immutable_fields_ = ['pycode']
 
+    w_yielded_from = None
+    thrown_operr = None
+
     def __init__(self, frame, name=None, qualname=None):
         self.space = frame.space
         self.frame = frame     # turned into None when frame_finished_execution
@@ -75,23 +78,26 @@
                 assert False
         self.running = self.space.is_true(w_running)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator/coroutine,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
 
-    def send_ex(self, w_arg, operr=None):
+    def send_ex(self, w_arg_or_err):
+        assert w_arg_or_err is not None
         pycode = self.pycode
         if pycode is not None:
             if jit.we_are_jitted() and should_not_inline(pycode):
-                generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                    operr=operr, pycode=pycode)
-        return self._send_ex(w_arg, operr)
+                generatorentry_driver.jit_merge_point(gen=self,
+                                                      w_arg=w_arg_or_err,
+                                                      pycode=pycode)
+        return self._send_ex(w_arg_or_err)
 
-    def _send_ex(self, w_arg, operr):
+    def _send_ex(self, w_arg_or_err):
         space = self.space
         if self.running:
             raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
+
         frame = self.frame
         if frame is None:
             if isinstance(self, Coroutine):
@@ -99,46 +105,64 @@
                             "cannot reuse already awaited coroutine")
             # xxx a bit ad-hoc, but we don't want to go inside
             # execute_frame() if the frame is actually finished
-            if operr is None:
+            if isinstance(w_arg_or_err, SApplicationException):
+                operr = w_arg_or_err.operr
+            else:
                 operr = OperationError(space.w_StopIteration, space.w_None)
             raise operr
 
+        w_result = self._invoke_execute_frame(frame, w_arg_or_err)
+
+        # if the frame is now marked as finished, it was RETURNed from
+        if frame.frame_finished_execution:
+            self.frame = None
+            if space.is_none(w_result):
+                # Delay exception instantiation if we can
+                raise OperationError(space.w_StopIteration, space.w_None)
+            else:
+                raise OperationError(space.w_StopIteration,
+                                     space.newtuple([w_result]))
+        else:
+            return w_result     # YIELDed
+
+    def _invoke_execute_frame(self, frame, w_arg_or_err):
+        space = self.space
+        self.running = True
+        try:
+            w_result = frame.execute_frame(self, w_arg_or_err)
+        except OperationError as e:
+            # errors finish a frame
+            try:
+                if e.match(space, space.w_StopIteration):
+                    self._leak_stopiteration(e)
+            finally:
+                self.frame = None
+            raise
+        finally:
+            frame.f_backref = jit.vref_None
+            self.running = False
+        return w_result
+
+    def resume_execute_frame(self, frame, w_arg_or_err):
+        # Called from execute_frame() just before resuming the bytecode
+        # interpretation.
+        space = self.space
+        if self.w_yielded_from is not None:
+            XXX
+
+        if isinstance(w_arg_or_err, SApplicationException):
+            ec = space.getexecutioncontext()
+            return frame.handle_operation_error(ec, w_arg_or_err.operr)
+
         last_instr = jit.promote(frame.last_instr)
         if last_instr == -1:
-            if w_arg and not space.is_w(w_arg, space.w_None):
+            if not space.is_w(w_arg_or_err, space.w_None):
                 raise oefmt(space.w_TypeError,
                             "can't send non-None value to a just-started %s",
                             self.KIND)
         else:
-            if not w_arg:
-                w_arg = space.w_None
-        self.running = True
-        try:
-            try:
-                w_result = frame.execute_frame(w_arg, operr)
-            except OperationError as e:
-                # errors finish a frame
-                try:
-                    if e.match(space, space.w_StopIteration):
-                        self._leak_stopiteration(e)
-                finally:
-                    self.frame = None
-                raise
-            #
-            # if the frame is now marked as finished, it was RETURNed from
-            if frame.frame_finished_execution:
-                self.frame = None
-                if space.is_none(w_result):
-                    # Delay exception instantiation if we can
-                    raise OperationError(space.w_StopIteration, space.w_None)
-                else:
-                    raise OperationError(space.w_StopIteration,
-                                         space.newtuple([w_result]))
-            else:
-                return w_result     # YIELDed
-        finally:
-            frame.f_backref = jit.vref_None
-            self.running = False
+            frame.pushvalue(w_arg_or_err)
+        return last_instr + 1
 
     def _leak_stopiteration(self, e):
         # Check for __future__ generator_stop and conditionally turn
@@ -166,22 +190,10 @@
             w_val = self.space.w_None
         return self.throw(w_type, w_val, w_tb)
 
-    def _get_yield_from(self):
-        # Probably a hack (but CPython has the same, _PyGen_yf()):
-        # If the current frame is stopped in a "yield from",
-        # return the paused generator.
-        # XXX this is probably very bad for the JIT.  Think again!
-        if not self.frame:
-            return None
-        co_code = self.frame.pycode.co_code
-        opcode = ord(co_code[self.frame.last_instr + 1])
-        if opcode == YIELD_FROM:
-            return self.frame.peekvalue()
-
     def throw(self, w_type, w_val, w_tb):
         space = self.space
 
-        w_yf = self._get_yield_from()
+        w_yf = self.w_yielded_from
         if w_yf is not None:
             # Paused in a "yield from", pass the throw to the inner generator.
             return self._throw_delegate(space, w_yf, w_type, w_val, w_tb)
@@ -252,7 +264,7 @@
                                space.wrap('__traceback__'))
             if not space.is_w(tb, space.w_None):
                 operr.set_traceback(tb)
-        return self.send_ex(space.w_None, operr)
+        return self.send_ex(SApplicationException(operr))
 
     def descr_close(self):
         """close() -> raise GeneratorExit inside generator/coroutine."""
@@ -346,7 +358,8 @@
                     jitdriver.jit_merge_point(self=self, frame=frame,
                                               results=results, pycode=pycode)
                     try:
-                        w_result = frame.execute_frame(space.w_None)
+                        w_result = self._invoke_execute_frame(
+                                                frame, space.w_None)
                     except OperationError as e:
                         if not e.match(space, space.w_StopIteration):
                             raise
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -261,11 +261,11 @@
         else:
             return self.execute_frame()
 
-    def execute_frame(self, w_inputvalue=None, operr=None):
+    def execute_frame(self, in_generator=None, w_arg_or_err=None):
         """Execute this frame.  Main entry point to the interpreter.
-        The optional arguments are there to handle a generator's frame:
-        w_inputvalue is for generator.send() and operr is for
-        generator.throw().
+        'in_generator' is non-None iff we are starting or resuming
+        a generator or coroutine frame; in that case, w_arg_or_err
+        is the input argument -or- an SApplicationException instance.
         """
         # the following 'assert' is an annotation hint: it hides from
         # the annotator all methods that are defined in PyFrame but
@@ -279,17 +279,16 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
+            # Execution starts just after the last_instr.  Initially,
+            # last_instr is -1.  After a generator suspends it points to
+            # the YIELD_VALUE instruction.
+            if in_generator is None:
+                assert self.last_instr == -1
+                next_instr = 0
             else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
+                next_instr = in_generator.resume_execute_frame(
+                                                self, w_arg_or_err)
+            next_instr = r_uint(next_instr)
             #
             try:
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
@@ -898,8 +897,14 @@
             frame = frame.f_backref()
         return None
 
+    def get_generator(self):
+        if space.config.translation.rweakref:
+            return self.f_generator_wref()
+        else:
+            return self.f_generator_nowref
+
     def descr_clear(self, space):
-        # Clears a random subset of the attributes (e.g. some the fast
+        # Clears a random subset of the attributes (e.g. the fast
         # locals, but not f_locals).  Also clears last_exception, which
         # is not quite like CPython when it clears f_exc_* (however
         # there might not be an observable difference).
@@ -907,10 +912,7 @@
             if not self._is_generator_or_coroutine():
                 raise oefmt(space.w_RuntimeError,
                             "cannot clear an executing frame")
-            if space.config.translation.rweakref:
-                gen = self.f_generator_wref()
-            else:
-                gen = self.f_generator_nowref
+            gen = self.get_generator()
             if gen is not None:
                 if gen.running:
                     raise oefmt(space.w_RuntimeError,
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -71,11 +71,11 @@
         try:
             while True:
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
+        except Return:
+            self.last_exception = None
         except Yield:
-            return self.popvalue()
-        except ExitFrame:
-            self.last_exception = None
-            return self.popvalue()
+            pass
+        return self.popvalue()
 
     def handle_bytecode(self, co_code, next_instr, ec):
         try:
@@ -1530,15 +1530,11 @@
         
 ### ____________________________________________________________ ###
 
-class ExitFrame(Exception):
-    pass
-
-
-class Return(ExitFrame):
+class Return(Exception):
     """Raised when exiting a frame via a 'return' statement."""
 
 
-class Yield(ExitFrame):
+class Yield(Exception):
     """Raised when exiting a frame via a 'yield' statement."""
 
 
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,13 +57,16 @@
         def f():
             yield 2
         g = f()
+        next(g)
         raises(NameError, g.throw, NameError, "Error")
+        raises(StopIteration, next, g)
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
         raises(NameError, g.throw, NameError("Error"))
+        raises(StopIteration, next, g)
 
     def test_throw3(self):
         def f():
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -795,18 +795,18 @@
     gi_running = interp_attrproperty('running', cls=GeneratorIterator),
     gi_frame   = GetSetProperty(GeneratorIterator.descr_gicr_frame),
     gi_code    = interp_attrproperty_w('pycode', cls=GeneratorIterator),
-    gi_yieldfrom = XXX,
+    gi_yieldfrom=interp_attrproperty_w('w_yielded_from', cls=GeneratorIterator),
     __name__   = GetSetProperty(GeneratorIterator.descr__name__),
     __qualname__ = GetSetProperty(GeneratorIterator.descr__qualname__),
     __weakref__ = make_weakref_descr(GeneratorIterator),
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
 
-CoroutineWrapper.typedef = TypeDef("coroutine_wrapper",
-    __anext__   = interp2app(CoroutineWrapper.descr_next,
-                            descrmismatch='__anext__'),
-)
-assert not CoroutineWrapper.typedef.acceptable_as_base_class  # no __new__
+#CoroutineWrapper.typedef = TypeDef("coroutine_wrapper",
+#    __anext__   = interp2app(CoroutineWrapper.descr_next,
+#                            descrmismatch='__anext__'),
+#)
+#assert not CoroutineWrapper.typedef.acceptable_as_base_class  # no __new__
 
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
@@ -823,7 +823,7 @@
     cr_running = interp_attrproperty('running', cls=Coroutine),
     cr_frame   = GetSetProperty(Coroutine.descr_gicr_frame),
     cr_code    = interp_attrproperty_w('pycode', cls=Coroutine),
-    cr_await   = XXX,
+    cr_await   = interp_attrproperty_w('w_yielded_from', cls=Coroutine),
     __name__   = GetSetProperty(Coroutine.descr__name__),
     __qualname__ = GetSetProperty(Coroutine.descr__qualname__),
     __weakref__ = make_weakref_descr(Coroutine),
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
--- a/pypy/module/_continuation/interp_pickle.py
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -1,6 +1,6 @@
 from pypy.tool import stdlib_opcode as pythonopcode
 from rpython.rlib import jit
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.pyframe import PyFrame
 from pypy.module._continuation.interp_continuation import State, global_state
 from pypy.module._continuation.interp_continuation import build_sthread
@@ -34,6 +34,8 @@
     return space.newtuple(args)
 
 def setstate(self, w_args):
+    raise oefmt(space.w_NotImplementedError,
+                "continulet.__setstate__() needs to be fixed")
     space = self.space
     if self.sthread is not None:
         raise geterror(space, "continulet.__setstate__() on an already-"

From pypy.commits at gmail.com  Fri Sep 16 10:19:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 07:19:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: oops,
 revert f6eabcc3367c: found the difference between test_throw1 and
Message-ID: <57dbff60.06b0c20a.57753.1123@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87139:2d311b3e8715
Date: 2016-09-16 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/2d311b3e8715/

Log:	oops, revert f6eabcc3367c: found the difference between test_throw1
	and test_throw2

diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,15 +57,15 @@
         def f():
             yield 2
         g = f()
-        g.next()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
-        raises(StopIteration, g.next)
 
     def test_throw3(self):
         def f():

From pypy.commits at gmail.com  Fri Sep 16 10:26:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 07:26:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: in-progress (I think that
 basic generators are working again)
Message-ID: <57dc0122.861b1c0a.6d3a0.ce09@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87140:97a825be3993
Date: 2016-09-16 16:26 +0200
http://bitbucket.org/pypy/pypy/changeset/97a825be3993/

Log:	in-progress (I think that basic generators are working again)

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -83,6 +83,9 @@
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
 
+    def send_error(self, operr):
+        return self.send_ex(SApplicationException(operr))
+
     def send_ex(self, w_arg_or_err):
         assert w_arg_or_err is not None
         pycode = self.pycode
@@ -95,8 +98,6 @@
 
     def _send_ex(self, w_arg_or_err):
         space = self.space
-        if self.running:
-            raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
 
         frame = self.frame
         if frame is None:
@@ -127,6 +128,8 @@
 
     def _invoke_execute_frame(self, frame, w_arg_or_err):
         space = self.space
+        if self.running:
+            raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
         self.running = True
         try:
             w_result = frame.execute_frame(self, w_arg_or_err)
@@ -210,7 +213,7 @@
                 finally:
                     self.running = False
             except OperationError as e:
-                return self.send_ex(space.w_None, e)
+                return self.send_error(e)
             return self._throw_here(space, w_type, w_val, w_tb)
         #
         if isinstance(w_yf, GeneratorIterator):
@@ -246,7 +249,7 @@
                                   space.wrap("value"))
             return self.send_ex(w_val)
         else:
-            return self.send_ex(space.w_None, operr)
+            return self.send_error(operr)
 
     def _throw_here(self, space, w_type, w_val, w_tb):
         from pypy.interpreter.pytraceback import check_traceback
@@ -264,16 +267,16 @@
                                space.wrap('__traceback__'))
             if not space.is_w(tb, space.w_None):
                 operr.set_traceback(tb)
-        return self.send_ex(SApplicationException(operr))
+        return self.send_error(operr)
 
     def descr_close(self):
         """close() -> raise GeneratorExit inside generator/coroutine."""
         if self.frame is None:
             return     # nothing to do in this case
         space = self.space
-        w_yf = self._get_yield_from()
         operr = OperationError(space.w_GeneratorExit,
                                space.call_function(space.w_GeneratorExit))
+        w_yf = self.w_yielded_from
         if w_yf is not None:
             self.running = True
             try:
@@ -283,7 +286,7 @@
             finally:
                 self.running = False
         try:
-            w_retval = self.send_ex(space.w_None, operr)
+            w_retval = self.send_error(operr)
         except OperationError as e:
             if e.match(space, space.w_StopIteration) or \
                     e.match(space, space.w_GeneratorExit):
@@ -344,34 +347,26 @@
         def unpack_into(self, results):
             """This is a hack for performance: runs the generator and collects
             all produced items in a list."""
-            # XXX copied and simplified version of send_ex()
-            space = self.space
-            if self.running:
-                raise oefmt(space.w_ValueError, "generator already executing")
             frame = self.frame
             if frame is None:    # already finished
                 return
-            self.running = True
-            try:
-                pycode = self.pycode
-                while True:
-                    jitdriver.jit_merge_point(self=self, frame=frame,
-                                              results=results, pycode=pycode)
-                    try:
-                        w_result = self._invoke_execute_frame(
-                                                frame, space.w_None)
-                    except OperationError as e:
-                        if not e.match(space, space.w_StopIteration):
-                            raise
-                        break
-                    # if the frame is now marked as finished, it was RETURNed from
-                    if frame.frame_finished_execution:
-                        break
-                    results.append(w_result)     # YIELDed
-            finally:
-                frame.f_backref = jit.vref_None
-                self.running = False
-                self.frame = None
+            pycode = self.pycode
+            while True:
+                jitdriver.jit_merge_point(self=self, frame=frame,
+                                          results=results, pycode=pycode)
+                space = self.space
+                try:
+                    w_result = self._invoke_execute_frame(
+                                            frame, space.w_None)
+                except OperationError as e:
+                    if not e.match(space, space.w_StopIteration):
+                        raise
+                    break
+                # if the frame is now marked as finished, it was RETURNed from
+                if frame.frame_finished_execution:
+                    self.frame = None
+                    break
+                results.append(w_result)     # YIELDed
         return unpack_into
     unpack_into = _create_unpack_into()
     unpack_into_w = _create_unpack_into()
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,16 +57,13 @@
         def f():
             yield 2
         g = f()
-        next(g)
         raises(NameError, g.throw, NameError, "Error")
-        raises(StopIteration, next, g)
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
         raises(NameError, g.throw, NameError("Error"))
-        raises(StopIteration, next, g)
 
     def test_throw3(self):
         def f():

From pypy.commits at gmail.com  Fri Sep 16 10:49:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 07:49:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: Oops,
 bug: if a throw() propagates the error sent in, then the 'return'
Message-ID: <57dc0686.8f081c0a.bd04e.d6d9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87141:47fa78e23bfb
Date: 2016-09-16 16:49 +0200
http://bitbucket.org/pypy/pypy/changeset/47fa78e23bfb/

Log:	Oops, bug: if a throw() propagates the error sent in, then the
	'return' trace was not called. This can confuse debuggers/profilers.

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -276,18 +276,17 @@
                 if next_instr != 0:
                     self.pushvalue(w_inputvalue)
             #
-            try:
-                w_exitvalue = self.dispatch(self.pycode, next_instr,
-                                            executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
+            w_exitvalue = self.dispatch(self.pycode, next_instr,
+                                        executioncontext)
             executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
             # for no good reason
             got_exception = False
+        except Exception:
+            executioncontext.return_trace(self, self.space.w_None)
+            raise
         finally:
             executioncontext.leave(self, w_exitvalue, got_exception)
         return w_exitvalue
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -562,3 +562,21 @@
         res = f(10).g()
         sys.settrace(None)
         assert res == 10
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']

From pypy.commits at gmail.com  Fri Sep 16 10:52:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 07:52:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: Better fix for 47fa78e23bfb (in obscure
 cases like the 'return' trace
Message-ID: <57dc0732.89d11c0a.2d57f.d88f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87142:51f3a7987d0c
Date: 2016-09-16 16:51 +0200
http://bitbucket.org/pypy/pypy/changeset/51f3a7987d0c/

Log:	Better fix for 47fa78e23bfb (in obscure cases like the 'return'
	trace raising)

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -264,29 +264,27 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
-            else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
-            #
-            w_exitvalue = self.dispatch(self.pycode, next_instr,
-                                        executioncontext)
-            executioncontext.return_trace(self, w_exitvalue)
+            try:
+                if operr is not None:
+                    ec = self.space.getexecutioncontext()
+                    next_instr = self.handle_operation_error(ec, operr)
+                    self.last_instr = intmask(next_instr - 1)
+                else:
+                    # Execution starts just after the last_instr.  Initially,
+                    # last_instr is -1.  After a generator suspends it points to
+                    # the YIELD_VALUE instruction.
+                    next_instr = r_uint(self.last_instr + 1)
+                    if next_instr != 0:
+                        self.pushvalue(w_inputvalue)
+                w_exitvalue = self.dispatch(self.pycode, next_instr,
+                                            executioncontext)
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
             # for no good reason
             got_exception = False
-        except Exception:
-            executioncontext.return_trace(self, self.space.w_None)
-            raise
         finally:
             executioncontext.leave(self, w_exitvalue, got_exception)
         return w_exitvalue

From pypy.commits at gmail.com  Fri Sep 16 11:48:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 08:48:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: 'yield from' starts to work
 again, a bit
Message-ID: <57dc1459.6286c20a.3c7e6.3732@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87143:f3788fad20bf
Date: 2016-09-16 17:48 +0200
http://bitbucket.org/pypy/pypy/changeset/f3788fad20bf/

Log:	'yield from' starts to work again, a bit

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.pyopcode import LoopBlock, SApplicationException
+from pypy.interpreter.pyopcode import LoopBlock, SApplicationException, Yield
 from pypy.interpreter.pycode import CO_YIELD_INSIDE_TRY
 from pypy.interpreter.astcompiler import consts
 from rpython.rlib import jit
@@ -151,9 +151,10 @@
         # interpretation.
         space = self.space
         if self.w_yielded_from is not None:
-            XXX
-
-        if isinstance(w_arg_or_err, SApplicationException):
+            w_arg_or_err = self.next_yield_from(frame, w_arg_or_err)
+            # Normal case: the call above raises Yield.
+            # We reach this point if the iterable is exhausted.
+        elif isinstance(w_arg_or_err, SApplicationException):
             ec = space.getexecutioncontext()
             return frame.handle_operation_error(ec, w_arg_or_err.operr)
 
@@ -167,6 +168,40 @@
             frame.pushvalue(w_arg_or_err)
         return last_instr + 1
 
+    def next_yield_from(self, frame, w_inputvalue_or_err):
+        """Fetch the next item of the current 'yield from', push it on
+        the frame stack, and raises Yield.  If there isn't one, push
+        w_stopiteration_value and returns.  May also just raise.
+        """
+        space = self.space
+        w_yf = self.w_yielded_from
+        try:
+            if isinstance(w_yf, GeneratorOrCoroutine):
+                w_retval = w_yf.send_ex(w_inputvalue_or_err)
+            elif space.is_w(w_inputvalue_or_err, space.w_None):
+                w_retval = space.next(w_yf)
+            elif isinstance(w_inputvalue_or_err, SApplicationException):
+                raise w_inputvalue_or_err.operr
+            else:
+                w_retval = space.call_method(w_gen, "send", w_inputvalue_or_err)
+        except OperationError as e:
+            self.w_yielded_from = None
+            if not e.match(space, space.w_StopIteration):
+                raise
+            e.normalize_exception(space)
+            try:
+                w_stop_value = space.getattr(e.get_w_value(space),
+                                             space.wrap("value"))
+            except OperationError as e:
+                if not e.match(space, space.w_AttributeError):
+                    raise
+                w_value = space.w_None
+            frame.pushvalue(w_stop_value)
+            return
+        else:
+            frame.pushvalue(w_retval)
+            raise Yield
+
     def _leak_stopiteration(self, e):
         # Check for __future__ generator_stop and conditionally turn
         # a leaking StopIteration into RuntimeError (with its cause
@@ -194,66 +229,9 @@
         return self.throw(w_type, w_val, w_tb)
 
     def throw(self, w_type, w_val, w_tb):
+        from pypy.interpreter.pytraceback import check_traceback
         space = self.space
 
-        w_yf = self.w_yielded_from
-        if w_yf is not None:
-            # Paused in a "yield from", pass the throw to the inner generator.
-            return self._throw_delegate(space, w_yf, w_type, w_val, w_tb)
-        else:
-            # Not paused in a "yield from", throw inside this generator
-            return self._throw_here(space, w_type, w_val, w_tb)
-
-    def _throw_delegate(self, space, w_yf, w_type, w_val, w_tb):
-        if space.exception_match(w_type, space.w_GeneratorExit):
-            try:
-                self.running = True
-                try:
-                    gen_close_iter(space, w_yf)
-                finally:
-                    self.running = False
-            except OperationError as e:
-                return self.send_error(e)
-            return self._throw_here(space, w_type, w_val, w_tb)
-        #
-        if isinstance(w_yf, GeneratorIterator):
-            self.running = True
-            try:
-                return w_yf.throw(space, w_type, w_val, w_tb)
-            except OperationError as e:
-                operr = e
-            finally:
-                self.running = False
-        else:
-            try:
-                w_throw = space.getattr(w_yf, space.wrap("throw"))
-            except OperationError as e:
-                if not e.match(space, space.w_AttributeError):
-                    raise
-                return self._throw_here(space, w_type, w_val, w_tb)
-            self.running = True
-            try:
-                return space.call_function(w_throw, w_type, w_val, w_tb)
-            except OperationError as e:
-                operr = e
-            finally:
-                self.running = False
-        # Pop subiterator from stack.
-        w_subiter = self.frame.popvalue()
-        assert space.is_w(w_subiter, w_yf)
-        # Termination repetition of YIELD_FROM
-        self.frame.last_instr += 1
-        if operr.match(space, space.w_StopIteration):
-            operr.normalize_exception(space)
-            w_val = space.getattr(operr.get_w_value(space),
-                                  space.wrap("value"))
-            return self.send_ex(w_val)
-        else:
-            return self.send_error(operr)
-
-    def _throw_here(self, space, w_type, w_val, w_tb):
-        from pypy.interpreter.pytraceback import check_traceback
-
         msg = "throw() third argument must be a traceback object"
         if space.is_none(w_tb):
             tb = None
@@ -278,6 +256,7 @@
                                space.call_function(space.w_GeneratorExit))
         w_yf = self.w_yielded_from
         if w_yf is not None:
+            XXX
             self.running = True
             try:
                 gen_close_iter(space, w_yf)
@@ -286,14 +265,13 @@
             finally:
                 self.running = False
         try:
-            w_retval = self.send_error(operr)
+            self.send_error(operr)
         except OperationError as e:
             if e.match(space, space.w_StopIteration) or \
                     e.match(space, space.w_GeneratorExit):
                 return space.w_None
             raise
-
-        if w_retval is not None:
+        else:
             raise oefmt(space.w_RuntimeError,
                         "%s ignored GeneratorExit", self.KIND)
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -267,6 +267,7 @@
         a generator or coroutine frame; in that case, w_arg_or_err
         is the input argument -or- an SApplicationException instance.
         """
+        from pypy.interpreter import pyopcode
         # the following 'assert' is an annotation hint: it hides from
         # the annotator all methods that are defined in PyFrame but
         # overridden in the {,Host}FrameClass subclasses of PyFrame.
@@ -281,22 +282,24 @@
             #
             # Execution starts just after the last_instr.  Initially,
             # last_instr is -1.  After a generator suspends it points to
-            # the YIELD_VALUE instruction.
-            if in_generator is None:
-                assert self.last_instr == -1
-                next_instr = 0
-            else:
-                next_instr = in_generator.resume_execute_frame(
-                                                self, w_arg_or_err)
-            next_instr = r_uint(next_instr)
-            #
+            # the YIELD_VALUE/YIELD_FROM instruction.
             try:
-                w_exitvalue = self.dispatch(self.pycode, next_instr,
-                                            executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
-            executioncontext.return_trace(self, w_exitvalue)
+                if in_generator is None:
+                    assert self.last_instr == -1
+                    next_instr = 0
+                else:
+                    next_instr = in_generator.resume_execute_frame(
+                                                    self, w_arg_or_err)
+                next_instr = r_uint(next_instr)
+                #
+                self.dispatch(self.pycode, next_instr, executioncontext)
+            except pyopcode.Return:
+                self.last_exception = None
+                w_exitvalue = self.popvalue()
+            except pyopcode.Yield:
+                w_exitvalue = self.popvalue()
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
@@ -898,7 +901,7 @@
         return None
 
     def get_generator(self):
-        if space.config.translation.rweakref:
+        if self.space.config.translation.rweakref:
             return self.f_generator_wref()
         else:
             return self.f_generator_nowref
@@ -935,7 +938,6 @@
 
 def get_block_class(opname):
     # select the appropriate kind of block
-    from pypy.interpreter.pyopcode import block_classes
     return block_classes[opname]
 
 def unpickle_block(space, w_tup):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -67,15 +67,8 @@
         # For the sequel, force 'next_instr' to be unsigned for performance
         next_instr = r_uint(next_instr)
         co_code = pycode.co_code
-
-        try:
-            while True:
-                next_instr = self.handle_bytecode(co_code, next_instr, ec)
-        except Return:
-            self.last_exception = None
-        except Yield:
-            pass
-        return self.popvalue()
+        while True:
+            next_instr = self.handle_bytecode(co_code, next_instr, ec)
 
     def handle_bytecode(self, co_code, next_instr, ec):
         try:
@@ -1034,41 +1027,27 @@
         raise Yield
 
     def YIELD_FROM(self, oparg, next_instr):
-        from pypy.interpreter.astcompiler import consts
+        # Unlike CPython, we handle this not by repeating the same
+        # bytecode over and over until the inner iterator is exhausted.
+        # Instead, we directly set the generator's w_yielded_from.
+        # This asks generator.resume_execute_frame() to exhaust that
+        # sub-iterable first before continuing on the next bytecode.
         from pypy.interpreter.generator import Coroutine
-        space = self.space
-        w_value = self.popvalue()
-        w_gen = self.peekvalue()
-        if isinstance(w_gen, Coroutine):
-            if (w_gen.descr_gi_code(space).co_flags & consts.CO_COROUTINE and
-               not self.pycode.co_flags & (consts.CO_COROUTINE |
-                                       consts.CO_ITERABLE_COROUTINE)):
-                raise oefmt(self.space.w_TypeError,
-                            "cannot 'yield from' a coroutine object "
-                            "from a generator")
-        try:
-            if space.is_none(w_value):
-                w_retval = space.next(w_gen)
-            else:
-                w_retval = space.call_method(w_gen, "send", w_value)
-        except OperationError as e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            self.popvalue()  # Remove iter from stack
-            e.normalize_exception(space)
-            try:
-                w_value = space.getattr(e.get_w_value(space), space.wrap("value"))
-            except OperationError as e:
-                if not e.match(space, space.w_AttributeError):
-                    raise
-                w_value = space.w_None
-            self.pushvalue(w_value)
-        else:
-            # iter remains on stack, w_retval is value to be yielded.
-            self.pushvalue(w_retval)
-            # and repeat...
-            self.last_instr = self.last_instr - 1
-            raise Yield
+        in_generator = self.get_generator()
+        assert in_generator is not None
+        w_inputvalue = self.popvalue()
+        w_gen = self.popvalue()
+        if isinstance(w_gen, Coroutine) and not isinstance(self, Coroutine):
+            raise oefmt(self.space.w_TypeError,
+                        "cannot 'yield from' a coroutine object "
+                        "from a generator")
+        #
+        in_generator.w_yielded_from = w_gen
+        in_generator.next_yield_from(self, w_inputvalue)
+        # Common case: the call above raises Yield.
+        # If instead the iterable is empty, next_yield_from() pushed the
+        # final result and returns.  In that case, we can just continue
+        # with the next bytecode.
 
     def jump_absolute(self, jumpto, ec):
         # this function is overridden by pypy.module.pypyjit.interp_jit
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -325,6 +325,24 @@
             assert False, 'Expected StopIteration'
             """
 
+    def test_yield_from_basic(self):
+        """
+        def f1():
+            yield from []
+            yield from [1, 2, 3]
+            yield from f2()
+        def f2():
+            yield 4
+            yield 5
+        gen = f1()
+        assert next(gen) == 1
+        assert next(gen) == 2
+        assert next(gen) == 3
+        assert next(gen) == 4
+        assert next(gen) == 5
+        assert list(gen) == []
+        """
+
     def test_yield_from_return(self):
         """
         def f1():
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -90,6 +90,7 @@
                 self.valuestackdepth = hint(self.valuestackdepth, promote=True)
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
                 is_being_profiled = self.get_is_being_profiled()
+        XXX XXX   # please fix, the base dispatch() changed
         except Yield:
             # preserve self.last_exception between generator yields
             w_result = self.popvalue()

From pypy.commits at gmail.com  Fri Sep 16 11:52:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 08:52:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: fix
Message-ID: <57dc1520.854bc20a.da871.392c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87144:fe2ec186aa1c
Date: 2016-09-16 17:51 +0200
http://bitbucket.org/pypy/pypy/changeset/fe2ec186aa1c/

Log:	fix

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -124,6 +124,7 @@
                 raise OperationError(space.w_StopIteration,
                                      space.newtuple([w_result]))
         else:
+            assert w_result is not None
             return w_result     # YIELDed
 
     def _invoke_execute_frame(self, frame, w_arg_or_err):
@@ -151,10 +152,14 @@
         # interpretation.
         space = self.space
         if self.w_yielded_from is not None:
-            w_arg_or_err = self.next_yield_from(frame, w_arg_or_err)
+            self.next_yield_from(frame, w_arg_or_err)
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
-        elif isinstance(w_arg_or_err, SApplicationException):
+            last_instr = jit.promote(frame.last_instr)
+            assert last_instr >= 0
+            return last_instr + 1
+
+        if isinstance(w_arg_or_err, SApplicationException):
             ec = space.getexecutioncontext()
             return frame.handle_operation_error(ec, w_arg_or_err.operr)
 

From pypy.commits at gmail.com  Fri Sep 16 13:03:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 10:03:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: in-progress
Message-ID: <57dc25cf.05d71c0a.f43d4.0c62@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87145:7a673159303f
Date: 2016-09-16 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/7a673159303f/

Log:	in-progress

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -4,13 +4,13 @@
 from pypy.interpreter.pycode import CO_YIELD_INSIDE_TRY
 from pypy.interpreter.astcompiler import consts
 from rpython.rlib import jit
+from rpython.rlib.objectmodel import specialize
 
 
 class GeneratorOrCoroutine(W_Root):
     _immutable_fields_ = ['pycode']
 
     w_yielded_from = None
-    thrown_operr = None
 
     def __init__(self, frame, name=None, qualname=None):
         self.space = frame.space
@@ -113,18 +113,17 @@
             raise operr
 
         w_result = self._invoke_execute_frame(frame, w_arg_or_err)
+        assert w_result is not None
 
         # if the frame is now marked as finished, it was RETURNed from
         if frame.frame_finished_execution:
             self.frame = None
-            if space.is_none(w_result):
-                # Delay exception instantiation if we can
+            if space.is_w(w_result, space.w_None):
                 raise OperationError(space.w_StopIteration, space.w_None)
             else:
                 raise OperationError(space.w_StopIteration,
-                                     space.newtuple([w_result]))
+                        space.call_function(space.w_StopIteration, w_result))
         else:
-            assert w_result is not None
             return w_result     # YIELDed
 
     def _invoke_execute_frame(self, frame, w_arg_or_err):
@@ -186,9 +185,10 @@
             elif space.is_w(w_inputvalue_or_err, space.w_None):
                 w_retval = space.next(w_yf)
             elif isinstance(w_inputvalue_or_err, SApplicationException):
-                raise w_inputvalue_or_err.operr
+                operr = w_inputvalue_or_err.operr
+                XXXXX
             else:
-                w_retval = space.call_method(w_gen, "send", w_inputvalue_or_err)
+                w_retval = space.call_method(w_yf, "send", w_inputvalue_or_err)
         except OperationError as e:
             self.w_yielded_from = None
             if not e.match(space, space.w_StopIteration):
@@ -229,13 +229,11 @@
     def descr_throw(self, w_type, w_val=None, w_tb=None):
         """throw(typ[,val[,tb]]) -> raise exception in generator/coroutine,
 return next yielded value or raise StopIteration."""
+        from pypy.interpreter.pytraceback import check_traceback
+
+        space = self.space
         if w_val is None:
-            w_val = self.space.w_None
-        return self.throw(w_type, w_val, w_tb)
-
-    def throw(self, w_type, w_val, w_tb):
-        from pypy.interpreter.pytraceback import check_traceback
-        space = self.space
+            w_val = space.w_None
 
         msg = "throw() third argument must be a traceback object"
         if space.is_none(w_tb):
@@ -244,6 +242,15 @@
             tb = check_traceback(space, w_tb, msg)
 
         operr = OperationError(w_type, w_val, tb)
+        w_yf = self.w_yielded_from
+        if (w_yf is not None and
+                    operr.match(space, space.w_GeneratorExit)):
+            self.w_yielded_from = None
+            try:
+                gen_close_iter(space, w_yf)
+            except OperationError as e:
+                return self.send_error(e)
+
         operr.normalize_exception(space)
         if tb is None:
             tb = space.getattr(operr.get_w_value(space),
@@ -257,18 +264,14 @@
         if self.frame is None:
             return     # nothing to do in this case
         space = self.space
-        operr = OperationError(space.w_GeneratorExit,
-                               space.call_function(space.w_GeneratorExit))
+        operr = get_generator_exit(space)
         w_yf = self.w_yielded_from
         if w_yf is not None:
-            XXX
-            self.running = True
+            self.w_yielded_from = None
             try:
                 gen_close_iter(space, w_yf)
             except OperationError as e:
                 operr = e
-            finally:
-                self.running = False
         try:
             self.send_error(operr)
         except OperationError as e:
@@ -402,6 +405,11 @@
         return self
 
 
+ at specialize.memo()
+def get_generator_exit(space):
+    return OperationError(space.w_GeneratorExit,
+                          space.call_function(space.w_GeneratorExit))
+
 def gen_close_iter(space, w_yf):
     # This helper function is used by close() and throw() to
     # close a subiterator being delegated to by yield-from.

From pypy.commits at gmail.com  Fri Sep 16 13:12:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 10:12:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: next fix
Message-ID: <57dc280b.87941c0a.4e555.01d8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87146:70a2383edbd6
Date: 2016-09-16 19:12 +0200
http://bitbucket.org/pypy/pypy/changeset/70a2383edbd6/

Log:	next fix

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -151,7 +151,11 @@
         # interpretation.
         space = self.space
         if self.w_yielded_from is not None:
-            self.next_yield_from(frame, w_arg_or_err)
+            try:
+                self.next_yield_from(frame, w_arg_or_err)
+            except OperationError as operr:
+                ec = space.getexecutioncontext()
+                return frame.handle_operation_error(ec, operr)
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
             last_instr = jit.promote(frame.last_instr)
@@ -242,6 +246,8 @@
             tb = check_traceback(space, w_tb, msg)
 
         operr = OperationError(w_type, w_val, tb)
+        operr.normalize_exception(space)
+
         w_yf = self.w_yielded_from
         if (w_yf is not None and
                     operr.match(space, space.w_GeneratorExit)):
@@ -251,7 +257,6 @@
             except OperationError as e:
                 return self.send_error(e)
 
-        operr.normalize_exception(space)
         if tb is None:
             tb = space.getattr(operr.get_w_value(space),
                                space.wrap('__traceback__'))

From pypy.commits at gmail.com  Fri Sep 16 13:24:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Sep 2016 10:24:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: pass test_generator
Message-ID: <57dc2adf.a3adc20a.6cedf.5c13@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87147:363825d9fe26
Date: 2016-09-16 19:24 +0200
http://bitbucket.org/pypy/pypy/changeset/363825d9fe26/

Log:	pass test_generator

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -188,11 +188,8 @@
                 w_retval = w_yf.send_ex(w_inputvalue_or_err)
             elif space.is_w(w_inputvalue_or_err, space.w_None):
                 w_retval = space.next(w_yf)
-            elif isinstance(w_inputvalue_or_err, SApplicationException):
-                operr = w_inputvalue_or_err.operr
-                XXXXX
             else:
-                w_retval = space.call_method(w_yf, "send", w_inputvalue_or_err)
+                w_retval = delegate_to_nongen(space, w_yf, w_inputvalue_or_err)
         except OperationError as e:
             self.w_yielded_from = None
             if not e.match(space, space.w_StopIteration):
@@ -430,6 +427,27 @@
         else:
             space.call_function(w_close)
 
+def delegate_to_nongen(space, w_yf, w_inputvalue_or_err):
+    # invoke a "send" or "throw" by method name to a non-generator w_yf
+    if isinstance(w_inputvalue_or_err, SApplicationException):
+        operr = w_inputvalue_or_err.operr
+        try:
+            w_meth = space.getattr(w_yf, space.wrap("throw"))
+        except OperationError as e:
+            if not e.match(space, space.w_AttributeError):
+                raise
+            raise operr
+        # bah, CPython calls here with the exact same arguments as
+        # originally passed to throw().  In our case it is far removed.
+        # Let's hope nobody will complain...
+        operr.normalize_exception(space)
+        w_exc = operr.w_type
+        w_val = operr.get_w_value(space)
+        w_tb  = space.wrap(operr.get_traceback())
+        return space.call_function(w_meth, w_exc, w_val, w_tb)
+    else:
+        return space.call_method(w_yf, "send", w_inputvalue_or_err)
+
 
 def get_printable_location_genentry(bytecode):
     return '%s <generator>' % (bytecode.get_repr(),)

From pypy.commits at gmail.com  Fri Sep 16 16:18:53 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 13:18:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: factor out some invariant code from the
 unrolling_iterable loop. no sense in
Message-ID: <57dc53ad.0cce1c0a.792f7.470b@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r87148:0130615b254c
Date: 2016-09-16 12:20 +0200
http://bitbucket.org/pypy/pypy/changeset/0130615b254c/

Log:	factor out some invariant code from the unrolling_iterable loop. no
	sense in having it be there 13 times, identically.

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,16 +30,19 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
+    w_initializer_type = None
+    w_initializer = None
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if space.type(w_initializer) is space.w_str:
+            if w_initializer is not None:
+                if w_initializer_type is space.w_str:
                     a.descr_fromstring(space, w_initializer)
-                elif space.type(w_initializer) is space.w_list:
+                elif w_initializer_type is space.w_list:
                     a.descr_fromlist(space, w_initializer)
                 else:
                     a.extend(w_initializer, True)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode

From pypy.commits at gmail.com  Fri Sep 16 16:46:47 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 13:46:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: don't crash in merge_if_blocks if the
 values are symbolics
Message-ID: <57dc5a37.9102c20a.7fd22.9f1d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r87149:78469828ece4
Date: 2016-09-16 22:45 +0200
http://bitbucket.org/pypy/pypy/changeset/78469828ece4/

Log:	don't crash in merge_if_blocks if the values are symbolics

	(bit obscure case, but managed to get it)

diff --git a/rpython/translator/backendopt/merge_if_blocks.py b/rpython/translator/backendopt/merge_if_blocks.py
--- a/rpython/translator/backendopt/merge_if_blocks.py
+++ b/rpython/translator/backendopt/merge_if_blocks.py
@@ -20,6 +20,14 @@
         return False
     if isinstance(op.args[0], Constant) and isinstance(op.args[1], Constant):
         return False
+    # check that the constant is hashable (ie not a symbolic)
+    try:
+        if isinstance(op.args[0], Constant):
+            hash(op.args[0].value)
+        else:
+            hash(op.args[1].value)
+    except TypeError:
+        return False
     return True
 
 def merge_chain(chain, checkvar, varmap, graph):
diff --git a/rpython/translator/backendopt/test/test_merge_if_blocks.py b/rpython/translator/backendopt/test/test_merge_if_blocks.py
--- a/rpython/translator/backendopt/test/test_merge_if_blocks.py
+++ b/rpython/translator/backendopt/test/test_merge_if_blocks.py
@@ -2,11 +2,12 @@
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator.backendopt.all import backend_optimizations
 from rpython.translator.translator import TranslationContext, graphof as tgraphof
-from rpython.flowspace.model import Block
+from rpython.flowspace.model import Block, checkgraph
 from rpython.translator.backendopt.removenoops import remove_same_as
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, r_int
 from rpython.annotator.model import SomeChar, SomeUnicodeCodePoint
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 
 def do_test_merge(fn, testvalues):
     t = TranslationContext()
@@ -225,3 +226,29 @@
     malloc.remove_mallocs(t, t.graphs)
     from rpython.translator import simplify
     simplify.join_blocks(graph)
+
+def test_switch_on_symbolic():
+    symb1 = CDefinedIntSymbolic("1", 1)
+    symb2 = CDefinedIntSymbolic("2", 2)
+    symb3 = CDefinedIntSymbolic("3", 3)
+    def fn(x):
+        res = 0
+        if x == symb1:
+            res += x + 1
+        elif x == symb2:
+            res += x + 2
+        elif x == symb3:
+            res += x + 3
+        res += 1
+        return res
+    t = TranslationContext()
+    a = t.buildannotator()
+    a.build_types(fn, [int])
+    rtyper = t.buildrtyper()
+    rtyper.specialize()
+    graph = t.graphs[0]
+    remove_same_as(graph)
+    res = merge_if_blocks_once(graph)
+    assert not res
+    checkgraph(graph)
+

From pypy.commits at gmail.com  Fri Sep 16 17:12:48 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:12:48 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: use the write analyzer to know
 when to invalidate the cache
Message-ID: <57dc6050.411d1c0a.1f8fe.5e61@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87151:e53758af61e3
Date: 2016-08-22 13:59 +0100
http://bitbucket.org/pypy/pypy/changeset/e53758af61e3/

Log:	use the write analyzer to know when to invalidate the cache

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -6,10 +6,9 @@
 from rpython.translator.backendopt import removenoops
 from rpython.translator import simplify
 from rpython.translator.backendopt import ssa
+from rpython.translator.backendopt.writeanalyze import WriteAnalyzer
 
 def has_side_effects(op):
-    if op.opname == 'debug_assert' or op.opname == 'jit_force_virtualizable':
-        return False
     try:
         return getattr(llop, op.opname).sideeffects
     except AttributeError:
@@ -22,7 +21,7 @@
     return getattr(llop, op.opname).canfold
 
 class Cache(object):
-    def __init__(self, variable_families, purecache=None, heapcache=None):
+    def __init__(self, variable_families, analyzer, purecache=None, heapcache=None):
         if purecache is None:
             purecache = {}
         if heapcache is None:
@@ -30,10 +29,12 @@
         self.purecache = purecache
         self.heapcache = heapcache
         self.variable_families = variable_families
+        self.analyzer = analyzer
 
     def copy(self):
         return Cache(
-                self.variable_families, self.purecache.copy(),
+                self.variable_families, self.analyzer,
+                self.purecache.copy(),
                 self.heapcache.copy())
 
 
@@ -72,13 +73,21 @@
                     block.inputargs.append(newres)
                 heapcache[key] = newres
 
-        return Cache(self.variable_families, purecache, heapcache)
+        return Cache(
+                self.variable_families, self.analyzer, purecache, heapcache)
 
     def _clear_heapcache_for(self, concretetype, fieldname):
         for k in self.heapcache.keys():
             if k[0].concretetype == concretetype and k[1] == fieldname:
                 del self.heapcache[k]
 
+    def _clear_heapcache_for_effects(self, op):
+        effects = self.analyzer.analyze(op)
+        for k in self.heapcache.keys():
+            key = ('struct', k[0].concretetype, k[1])
+            if key in effects:
+                del self.heapcache[k]
+
     def cse_block(self, block):
         def representative_arg(arg):
             if isinstance(arg, Variable):
@@ -97,8 +106,6 @@
                 else:
                     self.heapcache[tup] = op.result
                 continue
-            if op.opname in ('setarrayitem', 'setinteriorfield', "malloc", "malloc_varsize"):
-                continue
             if op.opname == 'setfield':
                 target = representative_arg(op.args[0])
                 field = op.args[1].value
@@ -106,7 +113,7 @@
                 self.heapcache[target, field] = op.args[2]
                 continue
             if has_side_effects(op):
-                self.heapcache.clear()
+                self._clear_heapcache_for_effects(op)
                 continue
 
             # foldable operations
@@ -124,9 +131,9 @@
                 self.purecache[key] = op.result
         return added_some_same_as
 
-def _merge(tuples, variable_families):
+def _merge(tuples, variable_families, analyzer):
     if not tuples:
-        return Cache(variable_families)
+        return Cache(variable_families, analyzer)
     if len(tuples) == 1:
         (link, cache), = tuples
         return cache.copy()
@@ -136,6 +143,7 @@
 class CSE(object):
     def __init__(self, translator):
         self.translator = translator
+        self.analyzer = WriteAnalyzer(translator)
 
     def transform(self, graph):
         variable_families = ssa.DataFlowFamilyBuilder(graph).get_variable_families()
@@ -156,9 +164,10 @@
 
             if block.operations:
                 if not can_cache:
-                    cache = Cache(variable_families)
+                    cache = Cache(variable_families, self.analyzer)
                 else:
-                    cache = _merge(caches_to_merge[block], variable_families)
+                    cache = _merge(
+                        caches_to_merge[block], variable_families, self.analyzer)
                 changed_block = cache.cse_block(block)
                 added_some_same_as = changed_block or added_some_same_as
             done.add(block)
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -212,3 +212,23 @@
             return j
 
         self.check(f, [int], getfield=0)
+
+    def test_dont_invalidate_on_call(self):
+        class A(object):
+            pass
+        class B(object):
+            pass
+        def g(b, a):
+            b.x = 1
+            a.y = 2
+
+        def f(i):
+            a = A()
+            a.x = i
+            a.y = i + 1
+            b = B()
+            g(b, a)
+            return a.x + a.y
+
+        self.check(f, [int], getfield=1)
+

From pypy.commits at gmail.com  Fri Sep 16 17:12:50 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:12:50 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: support for more complex cases
 in the merge code:
Message-ID: <57dc6052.8aabc20a.1d096.ac93@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87152:ca579ca81332
Date: 2016-08-23 11:40 +0100
http://bitbucket.org/pypy/pypy/changeset/ca579ca81332/

Log:	support for more complex cases in the merge code:

	sometimes there are two operations in the cache that happen on
	different arguments. but those arguments are merged into one
	variable at the merge. this leads to a new entry in the cache of the
	merged block.

	(a bit terrible code, needs a refactoring)

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -43,7 +43,41 @@
         block = firstlink.target
         # copy all operations that exist in *all* blocks over. need to add a new
         # inputarg if the result is really a variable
+
+        # try non-straight merges
+        for argindex, inputarg in enumerate(block.inputargs):
+            # bit slow, but probably ok
+            firstlinkarg = self.variable_families.find_rep(firstlink.args[argindex])
+            results = []
+            for key, res in self.purecache.iteritems():
+                (opname, concretetype, args) = key
+                if args[0] != firstlinkarg: # XXX other args
+                    continue
+                results.append(res)
+                for linkindex, (link, cache) in enumerate(tuples):
+                    if linkindex == 0:
+                        continue
+                    listargs = list(args)
+                    listargs[0] = self.variable_families.find_rep(link.args[argindex])
+                    newkey = (opname, concretetype, tuple(listargs))
+                    otherres = cache.purecache.get(newkey, None)
+                    if otherres is None:
+                        break
+                    results.append(otherres)
+                else:
+                    listargs = list(args)
+                    listargs[0] = self.variable_families.find_rep(inputarg)
+                    newkey = (opname, concretetype, tuple(listargs))
+                    newres = res
+                    if isinstance(res, Variable):
+                        newres = res.copy()
+                        for linkindex, (link, cache) in enumerate(tuples):
+                            link.args.append(results[linkindex])
+                        block.inputargs.append(newres)
+                    purecache[newkey] = newres
+
         for key, res in self.purecache.iteritems():
+            # "straight" merge: the variable is in all other caches
             for link, cache in tuples[1:]:
                 val = cache.purecache.get(key, None)
                 if val is None:
@@ -57,8 +91,41 @@
                     block.inputargs.append(newres)
                 purecache[key] = newres
 
+        # ______________________
         # merge heapcache
         heapcache = {}
+
+        # try non-straight merges
+        for argindex, inputarg in enumerate(block.inputargs):
+            # bit slow, but probably ok
+            firstlinkarg = self.variable_families.find_rep(firstlink.args[argindex])
+            results = []
+            for key, res in self.heapcache.iteritems():
+                (arg, fieldname) = key
+                if arg != firstlinkarg:
+                    continue
+                results.append(res)
+                for linkindex, (link, cache) in enumerate(tuples):
+                    if linkindex == 0:
+                        continue
+                    otherarg = self.variable_families.find_rep(link.args[argindex])
+                    newkey = (otherarg, fieldname)
+                    otherres = cache.heapcache.get(newkey, None)
+                    if otherres is None:
+                        break
+                    results.append(otherres)
+                else:
+                    listargs = list(args)
+                    listargs[0] = inputarg
+                    newkey = (self.variable_families.find_rep(inputarg), fieldname)
+                    newres = res
+                    if isinstance(res, Variable):
+                        newres = res.copy()
+                        for linkindex, (link, cache) in enumerate(tuples):
+                            link.args.append(results[linkindex])
+                        block.inputargs.append(newres)
+                    heapcache[newkey] = newres
+
         for key, res in self.heapcache.iteritems():
             for link, cache in tuples[1:]:
                 val = cache.heapcache.get(key, None)
@@ -73,6 +140,8 @@
                     block.inputargs.append(newres)
                 heapcache[key] = newres
 
+
+
         return Cache(
                 self.variable_families, self.analyzer, purecache, heapcache)
 
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -61,6 +61,21 @@
         # an add in each branch, but not the final block
         self.check(f, [int, int], int_add=2)
 
+    def test_merge2(self):
+        # in this test we add two different values, but the final add is on the
+        # same different value, so it can be shared
+        def f(i, j):
+            if j:
+                x = i
+                y = x + 1
+            else:
+                x = ~i
+                y = x + 1
+            return (x + 1) * y
+
+        # an add in each branch, but not the final block
+        self.check(f, [int, int], int_add=2)
+
     def test_optimize_across_merge(self):
         def f(i, j):
             k = i + 1
@@ -213,6 +228,28 @@
 
         self.check(f, [int], getfield=0)
 
+    def test_merge2_heapcache(self):
+        class A(object):
+            pass
+
+        def f(i):
+            a1 = A()
+            a1.x = i
+            a2 = A()
+            a2.x = i + 1
+            a3 = A()
+            a3.x = 1 # clear other caches
+            if i:
+                a = a1
+                j = a.x
+            else:
+                a = a2
+                j = a.x
+            j += a.x
+            return j
+
+        self.check(f, [int], getfield=2)
+
     def test_dont_invalidate_on_call(self):
         class A(object):
             pass

From pypy.commits at gmail.com  Fri Sep 16 17:12:46 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:12:46 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: experiment with a more general
 common subexpression elimination.
Message-ID: <57dc604e.670dc20a.e0377.b528@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87150:c204353fe193
Date: 2016-08-22 13:47 +0100
http://bitbucket.org/pypy/pypy/changeset/c204353fe193/

Log:	experiment with a more general common subexpression elimination.

	The algorithm is a simple forward pass, merging information at
	control flow merges. It subsumes storesink.py and
	remove_duplicate_casts

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/backendopt/cse.py
@@ -0,0 +1,178 @@
+import collections
+
+from rpython.translator.backendopt import support
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.flowspace.model import mkentrymap, Variable
+from rpython.translator.backendopt import removenoops
+from rpython.translator import simplify
+from rpython.translator.backendopt import ssa
+
+def has_side_effects(op):
+    if op.opname == 'debug_assert' or op.opname == 'jit_force_virtualizable':
+        return False
+    try:
+        return getattr(llop, op.opname).sideeffects
+    except AttributeError:
+        return True
+
+def cse_graph(graph):
+    return CSE([graph]).transform(graph)
+
+def can_fold(op):
+    return getattr(llop, op.opname).canfold
+
+class Cache(object):
+    def __init__(self, variable_families, purecache=None, heapcache=None):
+        if purecache is None:
+            purecache = {}
+        if heapcache is None:
+            heapcache = {}
+        self.purecache = purecache
+        self.heapcache = heapcache
+        self.variable_families = variable_families
+
+    def copy(self):
+        return Cache(
+                self.variable_families, self.purecache.copy(),
+                self.heapcache.copy())
+
+
+    def merge(self, firstlink, tuples):
+        purecache = {}
+        block = firstlink.target
+        # copy all operations that exist in *all* blocks over. need to add a new
+        # inputarg if the result is really a variable
+        for key, res in self.purecache.iteritems():
+            for link, cache in tuples[1:]:
+                val = cache.purecache.get(key, None)
+                if val is None:
+                    break
+            else:
+                newres = res
+                if isinstance(res, Variable):
+                    newres = res.copy()
+                    for link, cache in tuples:
+                        link.args.append(cache.purecache[key])
+                    block.inputargs.append(newres)
+                purecache[key] = newres
+
+        # merge heapcache
+        heapcache = {}
+        for key, res in self.heapcache.iteritems():
+            for link, cache in tuples[1:]:
+                val = cache.heapcache.get(key, None)
+                if val is None:
+                    break
+            else:
+                newres = res
+                if isinstance(res, Variable):
+                    newres = res.copy()
+                    for link, cache in tuples:
+                        link.args.append(cache.heapcache[key])
+                    block.inputargs.append(newres)
+                heapcache[key] = newres
+
+        return Cache(self.variable_families, purecache, heapcache)
+
+    def _clear_heapcache_for(self, concretetype, fieldname):
+        for k in self.heapcache.keys():
+            if k[0].concretetype == concretetype and k[1] == fieldname:
+                del self.heapcache[k]
+
+    def cse_block(self, block):
+        def representative_arg(arg):
+            if isinstance(arg, Variable):
+                return self.variable_families.find_rep(arg)
+            return arg
+        added_some_same_as = False
+        for op in block.operations:
+            # heap operations
+            if op.opname == 'getfield':
+                tup = (representative_arg(op.args[0]), op.args[1].value)
+                res = self.heapcache.get(tup, None)
+                if res is not None:
+                    op.opname = 'same_as'
+                    op.args = [res]
+                    added_some_same_as = True
+                else:
+                    self.heapcache[tup] = op.result
+                continue
+            if op.opname in ('setarrayitem', 'setinteriorfield', "malloc", "malloc_varsize"):
+                continue
+            if op.opname == 'setfield':
+                target = representative_arg(op.args[0])
+                field = op.args[1].value
+                self._clear_heapcache_for(target.concretetype, field)
+                self.heapcache[target, field] = op.args[2]
+                continue
+            if has_side_effects(op):
+                self.heapcache.clear()
+                continue
+
+            # foldable operations
+            if not can_fold(op):
+                continue
+            key = (op.opname, op.result.concretetype,
+                   tuple([representative_arg(arg) for arg in op.args]))
+            res = self.purecache.get(key, None)
+            if res is not None:
+                op.opname = 'same_as'
+                op.args = [res]
+                added_some_same_as = True
+                self.variable_families.union(res, op.result)
+            else:
+                self.purecache[key] = op.result
+        return added_some_same_as
+
+def _merge(tuples, variable_families):
+    if not tuples:
+        return Cache(variable_families)
+    if len(tuples) == 1:
+        (link, cache), = tuples
+        return cache.copy()
+    firstlink, firstcache = tuples[0]
+    return firstcache.merge(firstlink, tuples)
+
+class CSE(object):
+    def __init__(self, translator):
+        self.translator = translator
+
+    def transform(self, graph):
+        variable_families = ssa.DataFlowFamilyBuilder(graph).get_variable_families()
+        entrymap = mkentrymap(graph)
+        backedges = support.find_backedges(graph)
+        todo = collections.deque([graph.startblock])
+        caches_to_merge = collections.defaultdict(list)
+        done = set()
+
+        added_some_same_as = False
+
+        while todo:
+            block = todo.popleft()
+            can_cache = True
+            for link in entrymap[block]:
+                if link in backedges:
+                    can_cache = False
+
+            if block.operations:
+                if not can_cache:
+                    cache = Cache(variable_families)
+                else:
+                    cache = _merge(caches_to_merge[block], variable_families)
+                changed_block = cache.cse_block(block)
+                added_some_same_as = changed_block or added_some_same_as
+            done.add(block)
+            # add all target blocks where all predecessors are already done
+            for exit in block.exits:
+                for lnk in entrymap[exit.target]:
+                    if lnk.prevblock not in done and lnk not in backedges:
+                        break
+                else:
+                    if exit.target not in done:
+                        todo.append(exit.target)
+                caches_to_merge[exit.target].append((exit, cache))
+        if added_some_same_as:
+            ssa.SSA_to_SSI(graph)
+            removenoops.remove_same_as(graph)
+        simplify.transform_dead_op_vars(graph)
+
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -0,0 +1,214 @@
+import pytest
+from rpython.translator.translator import TranslationContext, graphof
+from rpython.translator.backendopt.cse import CSE
+from rpython.translator.backendopt import removenoops
+from rpython.flowspace.model import checkgraph, summary
+from rpython.conftest import option
+
+class TestStoreSink(object):
+    def translate(self, func, argtypes):
+        t = TranslationContext()
+        t.buildannotator().build_types(func, argtypes)
+        t.buildrtyper().specialize()
+        return t
+
+    def check(self, f, argtypes, **expected):
+        from rpython.translator.backendopt import inline, all, constfold
+        t = self.translate(f, argtypes)
+        getfields = 0
+        graph = graphof(t, f)
+        if option.view:
+            t.view()
+        removenoops.remove_same_as(graph)
+        checkgraph(graph)
+        cse = CSE(t)
+        cse.transform(graph)
+        if option.view:
+            t.view()
+        checkgraph(graph)
+        s = summary(graph)
+        for key, val in expected.items():
+            assert s.get(key, 0) == val
+        assert "same_as" not in s
+
+    def test_infrastructure(self):
+        def f(i):
+            x = (i + 1) * (i + 1)
+            y = (i + 1) * (i + 1)
+            return x - y
+
+        self.check(f, [int], int_add=1, int_mul=1)
+
+    def test_split(self):
+        def f(i, j):
+            k = i + 1
+            if j:
+                return i + 1
+            return k * (i + 1)
+
+        self.check(f, [int, int], int_add=1)
+
+    def test_merge(self):
+        def f(i, j):
+            if j:
+                k = i + 1
+                j = 1
+            else:
+                j = i + 1
+                k = 5
+            return k * j * (i + 1)
+
+        # an add in each branch, but not the final block
+        self.check(f, [int, int], int_add=2)
+
+    def test_optimize_across_merge(self):
+        def f(i, j):
+            k = i + 1
+            if j:
+                j = 1
+            else:
+                j = i + 1
+            return k * j * (i + 1)
+        self.check(f, [int, int], int_add=1)
+
+
+    def test_getfield(self):
+        class A(object):
+            pass
+
+        def f(i):
+            a = A()
+            a.x = i
+            return a.x + a.x
+
+        self.check(f, [int], getfield=0)
+
+    def test_irrelevant_setfield(self):
+        class A(object):
+            pass
+
+        def f(i):
+            a = A()
+            a.x = i
+            one = a.x
+            a.y = 3
+            two = a.x
+            return one + two
+
+        self.check(f, [int], getfield=0)
+
+    def test_relevant_setfield(self):
+        class A(object):
+            pass
+
+        def f(i):
+            a = A()
+            b = A()
+            a.x = i
+            b.x = i + 1
+            one = a.x
+            b.x = i
+            two = a.x
+            return one + two
+
+        self.check(f, [int], getfield=2)
+
+    def test_different_concretetype(self):
+        class A(object):
+            pass
+
+        class B(object):
+            pass
+
+        def f(i):
+            a = A()
+            b = B()
+            a.x = i
+            one = a.x
+            b.x = i + 1
+            two = a.x
+            return one + two
+
+        self.check(f, [int], getfield=0)
+
+    def test_subclass(self):
+        class A(object):
+            pass
+
+        class B(A):
+            pass
+
+        def f(i):
+            a = A()
+            b = B()
+            a.x = i
+            one = a.x
+            b.x = i + 1
+            two = a.x
+            return one + two
+
+        self.check(f, [int], getfield=1)
+
+    def test_bug_1(self):
+        class A(object):
+            pass
+
+        def f(i):
+            a = A()
+            a.cond = i > 0
+            n = a.cond
+            if a.cond:
+                return True
+            return n
+
+        self.check(f, [int], getfield=0)
+
+
+    def test_cfg_splits_getfield(self):
+        class A(object):
+            pass
+
+        def f(i):
+            a = A()
+            j = i
+            for i in range(i):
+                a.x = i
+                if i:
+                    j = a.x + a.x
+                else:
+                    j = a.x * 5
+            return j
+
+        self.check(f, [int], getfield=0)
+
+    def test_malloc_does_not_invalidate(self):
+        class A(object):
+            pass
+        class B(object):
+            pass
+
+        def f(i):
+            a = A()
+            a.x = i
+            b = B()
+            return a.x
+
+        self.check(f, [int], getfield=0)
+
+    def test_merge_heapcache(self):
+        class A(object):
+            pass
+
+        def f(i):
+            a = A()
+            j = i
+            for i in range(i):
+                a.x = i
+                if i:
+                    j = a.x + a.x
+                else:
+                    j = a.x * 5
+                j += a.x
+            return j
+
+        self.check(f, [int], getfield=0)

From pypy.commits at gmail.com  Fri Sep 16 17:12:52 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:12:52 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: actually super easy: support
 CSE on loop-invariant operations (SSA makes this
Message-ID: <57dc6054.04141c0a.25833.5ad3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87153:ea93a553b7a0
Date: 2016-09-06 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ea93a553b7a0/

Log:	actually super easy: support CSE on loop-invariant operations (SSA
	makes this simple)

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -14,8 +14,12 @@
     except AttributeError:
         return True
 
-def cse_graph(graph):
-    return CSE([graph]).transform(graph)
+def common_subexpression_elimination(t, graphs=None):
+    if graphs is None:
+        graphs = t.graphs
+    cse = CSE(t)
+    for graph in graphs:
+        cse.transform(graph)
 
 def can_fold(op):
     return getattr(llop, op.opname).canfold
@@ -38,12 +42,16 @@
                 self.heapcache.copy())
 
 
-    def merge(self, firstlink, tuples):
+    def merge(self, firstlink, tuples, backedge):
         purecache = {}
         block = firstlink.target
         # copy all operations that exist in *all* blocks over. need to add a new
         # inputarg if the result is really a variable
 
+        # note that a backedge is not a problem for regular pure operations:
+        # since the argument is a phi node iff it is not loop invariant,
+        # copying things over is always save (yay SSA form!)
+
         # try non-straight merges
         for argindex, inputarg in enumerate(block.inputargs):
             # bit slow, but probably ok
@@ -94,6 +102,10 @@
         # ______________________
         # merge heapcache
         heapcache = {}
+        if backedge:
+            # can't deal with heapcache and backedges yet
+            return Cache(
+                    self.variable_families, self.analyzer, purecache, heapcache)
 
         # try non-straight merges
         for argindex, inputarg in enumerate(block.inputargs):
@@ -200,14 +212,14 @@
                 self.purecache[key] = op.result
         return added_some_same_as
 
-def _merge(tuples, variable_families, analyzer):
+def _merge(tuples, variable_families, analyzer, backedge=False):
     if not tuples:
         return Cache(variable_families, analyzer)
     if len(tuples) == 1:
         (link, cache), = tuples
         return cache.copy()
     firstlink, firstcache = tuples[0]
-    return firstcache.merge(firstlink, tuples)
+    return firstcache.merge(firstlink, tuples, backedge)
 
 class CSE(object):
     def __init__(self, translator):
@@ -226,17 +238,15 @@
 
         while todo:
             block = todo.popleft()
-            can_cache = True
+            backedge = False
             for link in entrymap[block]:
                 if link in backedges:
-                    can_cache = False
+                    backedge = True
+                    break
 
             if block.operations:
-                if not can_cache:
-                    cache = Cache(variable_families, self.analyzer)
-                else:
-                    cache = _merge(
-                        caches_to_merge[block], variable_families, self.analyzer)
+                cache = _merge(
+                    caches_to_merge[block], variable_families, self.analyzer, backedge)
                 changed_block = cache.cse_block(block)
                 added_some_same_as = changed_block or added_some_same_as
             done.add(block)
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -269,3 +269,13 @@
 
         self.check(f, [int], getfield=1)
 
+    def test_loopinvariant(self):
+        def f(i):
+            x = i + 1
+            res = 0
+            while x:
+                x -= 1
+                res += i + 1
+            return res
+        self.check(f, [int], int_add=2)
+

From pypy.commits at gmail.com  Fri Sep 16 17:12:53 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:12:53 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: support for removing
 loop-invariant heap reads
Message-ID: <57dc6055.57981c0a.7bf0f.60cb@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87154:2f9d46642175
Date: 2016-09-06 17:43 +0200
http://bitbucket.org/pypy/pypy/changeset/2f9d46642175/

Log:	support for removing loop-invariant heap reads

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -42,7 +42,7 @@
                 self.heapcache.copy())
 
 
-    def merge(self, firstlink, tuples, backedge):
+    def merge(self, firstlink, tuples):
         purecache = {}
         block = firstlink.target
         # copy all operations that exist in *all* blocks over. need to add a new
@@ -102,10 +102,6 @@
         # ______________________
         # merge heapcache
         heapcache = {}
-        if backedge:
-            # can't deal with heapcache and backedges yet
-            return Cache(
-                    self.variable_families, self.analyzer, purecache, heapcache)
 
         # try non-straight merges
         for argindex, inputarg in enumerate(block.inputargs):
@@ -138,6 +134,7 @@
                         block.inputargs.append(newres)
                     heapcache[newkey] = newres
 
+        # regular merge
         for key, res in self.heapcache.iteritems():
             for link, cache in tuples[1:]:
                 val = cache.heapcache.get(key, None)
@@ -162,13 +159,24 @@
             if k[0].concretetype == concretetype and k[1] == fieldname:
                 del self.heapcache[k]
 
-    def _clear_heapcache_for_effects(self, op):
+    def _clear_heapcache_for_effects_of_op(self, op):
         effects = self.analyzer.analyze(op)
+        self._clear_heapcache_for_effects(effects)
+
+    def _clear_heapcache_for_effects(self, effects):
         for k in self.heapcache.keys():
             key = ('struct', k[0].concretetype, k[1])
             if key in effects:
                 del self.heapcache[k]
 
+    def _clear_heapcache_for_loop_blocks(self, blocks):
+        effects = self.analyzer.bottom_result()
+        for block in blocks:
+            for op in block.operations:
+                effects = self.analyzer.join_two_results(
+                    effects, self.analyzer.analyze(op))
+        self._clear_heapcache_for_effects(effects)
+
     def cse_block(self, block):
         def representative_arg(arg):
             if isinstance(arg, Variable):
@@ -194,7 +202,7 @@
                 self.heapcache[target, field] = op.args[2]
                 continue
             if has_side_effects(op):
-                self._clear_heapcache_for_effects(op)
+                self._clear_heapcache_for_effects_of_op(op)
                 continue
 
             # foldable operations
@@ -212,14 +220,28 @@
                 self.purecache[key] = op.result
         return added_some_same_as
 
-def _merge(tuples, variable_families, analyzer, backedge=False):
+def _merge(tuples, variable_families, analyzer, loop_blocks=None):
     if not tuples:
         return Cache(variable_families, analyzer)
     if len(tuples) == 1:
         (link, cache), = tuples
-        return cache.copy()
-    firstlink, firstcache = tuples[0]
-    return firstcache.merge(firstlink, tuples, backedge)
+        result = cache.copy()
+    else:
+        firstlink, firstcache = tuples[0]
+        result = firstcache.merge(firstlink, tuples)
+    if loop_blocks:
+        # for all blocks in the loop, clean the heapcache for their effects
+        # that way, loop-invariant reads can be removed, if no one writes to
+        # anything that can alias with them.
+        result._clear_heapcache_for_loop_blocks(loop_blocks)
+    return result
+
+def loop_blocks(graph):
+    loop_blocks = support.find_loop_blocks(graph)
+    result = {}
+    for loop_block, start in loop_blocks.iteritems():
+        result.setdefault(start, []).append(loop_block)
+    return result
 
 class CSE(object):
     def __init__(self, translator):
@@ -229,6 +251,7 @@
     def transform(self, graph):
         variable_families = ssa.DataFlowFamilyBuilder(graph).get_variable_families()
         entrymap = mkentrymap(graph)
+        loops = loop_blocks(graph)
         backedges = support.find_backedges(graph)
         todo = collections.deque([graph.startblock])
         caches_to_merge = collections.defaultdict(list)
@@ -238,15 +261,11 @@
 
         while todo:
             block = todo.popleft()
-            backedge = False
-            for link in entrymap[block]:
-                if link in backedges:
-                    backedge = True
-                    break
 
             if block.operations:
                 cache = _merge(
-                    caches_to_merge[block], variable_families, self.analyzer, backedge)
+                    caches_to_merge[block], variable_families, self.analyzer,
+                    loops.get(block, None))
                 changed_block = cache.cse_block(block)
                 added_some_same_as = changed_block or added_some_same_as
             done.add(block)
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -279,3 +279,17 @@
             return res
         self.check(f, [int], int_add=2)
 
+
+    def test_loopinvariant_heap(self):
+        class A(object):
+            pass
+        def f(i):
+            a = A()
+            a.x = i
+            res = 0
+            x = i
+            while x:
+                x -= 1
+                res += a.x
+            return res
+        self.check(f, [int], getfield=0)

From pypy.commits at gmail.com  Fri Sep 16 17:12:55 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:12:55 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: a few corner cases
Message-ID: <57dc6057.822e1c0a.9e8e0.5ad5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87155:a5ab790844f4
Date: 2016-09-06 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/a5ab790844f4/

Log:	a few corner cases

diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py
--- a/rpython/translator/backendopt/all.py
+++ b/rpython/translator/backendopt/all.py
@@ -8,7 +8,7 @@
 from rpython.translator.backendopt import mallocprediction
 from rpython.translator.backendopt.removeassert import remove_asserts
 from rpython.translator.backendopt.support import log
-from rpython.translator.backendopt.storesink import storesink_graph
+from rpython.translator.backendopt.cse import common_subexpression_elimination
 from rpython.translator.backendopt import gilanalysis
 from rpython.flowspace.model import checkgraph
 
@@ -101,8 +101,7 @@
             print_statistics(translator.graphs[0], translator)
 
     if config.storesink:
-        for graph in graphs:
-            storesink_graph(graph)
+        common_subexpression_elimination(translator, graphs)
 
     if config.profile_based_inline and not secondary:
         threshold = config.profile_based_inline_threshold
diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -42,7 +42,7 @@
                 self.heapcache.copy())
 
 
-    def merge(self, firstlink, tuples):
+    def merge(self, firstlink, tuples, backedges):
         purecache = {}
         block = firstlink.target
         # copy all operations that exist in *all* blocks over. need to add a new
@@ -82,6 +82,8 @@
                         for linkindex, (link, cache) in enumerate(tuples):
                             link.args.append(results[linkindex])
                         block.inputargs.append(newres)
+                        for backedge in backedges:
+                            backedge.args.append(newres)
                     purecache[newkey] = newres
 
         for key, res in self.purecache.iteritems():
@@ -97,6 +99,8 @@
                     for link, cache in tuples:
                         link.args.append(cache.purecache[key])
                     block.inputargs.append(newres)
+                    for backedge in backedges:
+                        backedge.args.append(newres)
                 purecache[key] = newres
 
         # ______________________
@@ -132,6 +136,8 @@
                         for linkindex, (link, cache) in enumerate(tuples):
                             link.args.append(results[linkindex])
                         block.inputargs.append(newres)
+                        for backedge in backedges:
+                            backedge.args.append(newres)
                     heapcache[newkey] = newres
 
         # regular merge
@@ -147,6 +153,8 @@
                     for link, cache in tuples:
                         link.args.append(cache.heapcache[key])
                     block.inputargs.append(newres)
+                    for backedge in backedges:
+                        backedge.args.append(newres)
                 heapcache[key] = newres
 
 
@@ -220,7 +228,7 @@
                 self.purecache[key] = op.result
         return added_some_same_as
 
-def _merge(tuples, variable_families, analyzer, loop_blocks=None):
+def _merge(tuples, variable_families, analyzer, loop_blocks, backedges):
     if not tuples:
         return Cache(variable_families, analyzer)
     if len(tuples) == 1:
@@ -228,7 +236,7 @@
         result = cache.copy()
     else:
         firstlink, firstcache = tuples[0]
-        result = firstcache.merge(firstlink, tuples)
+        result = firstcache.merge(firstlink, tuples, backedges)
     if loop_blocks:
         # for all blocks in the loop, clean the heapcache for their effects
         # that way, loop-invariant reads can be removed, if no one writes to
@@ -261,13 +269,19 @@
 
         while todo:
             block = todo.popleft()
+            assert block not in done
+
+            current_backedges = [link for link in entrymap[block]
+                                    if link in backedges]
 
             if block.operations:
                 cache = _merge(
                     caches_to_merge[block], variable_families, self.analyzer,
-                    loops.get(block, None))
+                    loops.get(block, None), current_backedges)
                 changed_block = cache.cse_block(block)
                 added_some_same_as = changed_block or added_some_same_as
+            else:
+                cache = Cache(variable_families, self.analyzer)
             done.add(block)
             # add all target blocks where all predecessors are already done
             for exit in block.exits:
@@ -275,7 +289,7 @@
                     if lnk.prevblock not in done and lnk not in backedges:
                         break
                 else:
-                    if exit.target not in done:
+                    if exit.target not in done and exit.target not in todo: # XXX
                         todo.append(exit.target)
                 caches_to_merge[exit.target].append((exit, cache))
         if added_some_same_as:
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -293,3 +293,33 @@
                 res += a.x
             return res
         self.check(f, [int], getfield=0)
+
+    def test_loopinvariant_heap_merge(self):
+        class A(object):
+            pass
+        def f(i):
+            res = 0
+            x = i
+            a = A()
+            if i == 0:
+                a.x = 1
+            else:
+                a.x = i
+            while x:
+                x -= 1
+                res += a.x
+            return res
+        self.check(f, [int], getfield=0)
+
+    def test_direct_merge(self):
+        def f(i):
+            a = i + 1
+            if i:
+                x = a
+            else:
+                x = a
+            res = 0
+            while a:
+                res += i + 1
+            return a + (i + 1)
+        self.check(f, [int], add=0)

From pypy.commits at gmail.com  Fri Sep 16 17:13:01 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:01 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: add some logging
Message-ID: <57dc605d.9c4a1c0a.dc73a.82eb@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87158:7e9fbd17bdb8
Date: 2016-09-06 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/7e9fbd17bdb8/

Log:	add some logging

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -8,6 +8,8 @@
 from rpython.translator.backendopt import ssa
 from rpython.translator.backendopt.writeanalyze import WriteAnalyzer
 
+from rpython.translator.backendopt.support import log
+
 def has_side_effects(op):
     try:
         return getattr(llop, op.opname).sideeffects
@@ -191,7 +193,7 @@
             if isinstance(arg, Variable):
                 return self.variable_families.find_rep(arg)
             return arg
-        added_some_same_as = False
+        added_same_as = 0
         for op in block.operations:
             # heap operations
             if op.opname == 'getfield':
@@ -200,7 +202,7 @@
                 if res is not None:
                     op.opname = 'same_as'
                     op.args = [res]
-                    added_some_same_as = True
+                    added_same_as += 1
                 else:
                     self.heapcache[tup] = op.result
                 continue
@@ -223,11 +225,11 @@
             if res is not None:
                 op.opname = 'same_as'
                 op.args = [res]
-                added_some_same_as = True
+                added_same_as += 1
                 self.variable_families.union(res, op.result)
             else:
                 self.purecache[key] = op.result
-        return added_some_same_as
+        return added_same_as
 
 def _merge(tuples, variable_families, analyzer, loop_blocks, backedges):
     if not tuples:
@@ -266,7 +268,7 @@
         caches_to_merge = collections.defaultdict(list)
         done = set()
 
-        added_some_same_as = False
+        added_same_as = 0
 
         while todo:
             block = todo.popleft()
@@ -279,8 +281,7 @@
                 cache = _merge(
                     caches_to_merge[block], variable_families, self.analyzer,
                     loops.get(block, None), current_backedges)
-                changed_block = cache.cse_block(block)
-                added_some_same_as = changed_block or added_some_same_as
+                added_same_as += cache.cse_block(block)
             else:
                 cache = Cache(variable_families, self.analyzer)
             done.add(block)
@@ -293,8 +294,10 @@
                     if exit.target not in done and exit.target not in todo: # XXX
                         todo.append(exit.target)
                 caches_to_merge[exit.target].append((exit, cache))
-        if added_some_same_as:
+        if added_same_as:
             ssa.SSA_to_SSI(graph)
             removenoops.remove_same_as(graph)
         simplify.transform_dead_op_vars(graph)
+        if added_same_as:
+            log.cse("removed %s ops in graph %s" % (added_same_as, graph))
 

From pypy.commits at gmail.com  Fri Sep 16 17:12:57 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:12:57 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: add statistics printing after
 store sinking
Message-ID: <57dc6059.02d31c0a.d4777.5c5c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87156:5219caed36d0
Date: 2016-09-06 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/5219caed36d0/

Log:	add statistics printing after store sinking

diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py
--- a/rpython/translator/backendopt/all.py
+++ b/rpython/translator/backendopt/all.py
@@ -102,6 +102,9 @@
 
     if config.storesink:
         common_subexpression_elimination(translator, graphs)
+        if config.print_statistics:
+            print "after store sinking and CSE"
+            print_statistics(translator.graphs[0], translator)
 
     if config.profile_based_inline and not secondary:
         threshold = config.profile_based_inline_threshold

From pypy.commits at gmail.com  Fri Sep 16 17:13:03 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:03 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: better logging
Message-ID: <57dc605f.ce471c0a.6cd2f.b363@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87159:0a08408432ad
Date: 2016-09-06 19:57 +0200
http://bitbucket.org/pypy/pypy/changeset/0a08408432ad/

Log:	better logging

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -20,8 +20,11 @@
     if graphs is None:
         graphs = t.graphs
     cse = CSE(t)
+
+    removed_ops = 0
     for graph in graphs:
-        cse.transform(graph)
+        removed_ops += cse.transform(graph)
+    log.cse("cse removed %s ops" % (removed_ops, ))
 
 def can_fold(op):
     return getattr(llop, op.opname).canfold
@@ -55,7 +58,8 @@
         # copying things over is always save (yay SSA form!)
 
         # try non-straight merges
-        for argindex, inputarg in enumerate(block.inputargs):
+        for argindex in range(len(block.inputargs)):
+            inputarg = block.inputargs[argindex]
             # bit slow, but probably ok
             firstlinkarg = self.variable_families.find_rep(firstlink.args[argindex])
             results = []
@@ -110,7 +114,8 @@
         heapcache = {}
 
         # try non-straight merges
-        for argindex, inputarg in enumerate(block.inputargs):
+        for argindex in range(len(block.inputargs)):
+            inputarg = block.inputargs[argindex]
             # bit slow, but probably ok
             firstlinkarg = self.variable_families.find_rep(firstlink.args[argindex])
             results = []
@@ -299,5 +304,9 @@
             removenoops.remove_same_as(graph)
         simplify.transform_dead_op_vars(graph)
         if added_same_as:
-            log.cse("removed %s ops in graph %s" % (added_same_as, graph))
+            if self.translator.config.translation.verbose:
+                log.cse("cse removed %s ops in graph %s" % (added_same_as, graph))
+            else:
+                log.dot()
+        return added_same_as
 

From pypy.commits at gmail.com  Fri Sep 16 17:12:59 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:12:59 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: two fixes
Message-ID: <57dc605b.2472c20a.2f30e.aaf1@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87157:c97644fd33d0
Date: 2016-09-06 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/c97644fd33d0/

Log:	two fixes

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -127,8 +127,6 @@
                         break
                     results.append(otherres)
                 else:
-                    listargs = list(args)
-                    listargs[0] = inputarg
                     newkey = (self.variable_families.find_rep(inputarg), fieldname)
                     newres = res
                     if isinstance(res, Variable):
@@ -172,10 +170,13 @@
         self._clear_heapcache_for_effects(effects)
 
     def _clear_heapcache_for_effects(self, effects):
-        for k in self.heapcache.keys():
-            key = ('struct', k[0].concretetype, k[1])
-            if key in effects:
-                del self.heapcache[k]
+        if self.analyzer.is_top_result(effects):
+            self.heapcache.clear()
+        else:
+            for k in self.heapcache.keys():
+                key = ('struct', k[0].concretetype, k[1])
+                if key in effects:
+                    del self.heapcache[k]
 
     def _clear_heapcache_for_loop_blocks(self, blocks):
         effects = self.analyzer.bottom_result()

From pypy.commits at gmail.com  Fri Sep 16 17:13:05 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:05 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: fix bug
Message-ID: <57dc6061.6937c20a.c44a4.b171@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87160:f6dc12d932d9
Date: 2016-09-09 12:36 +0200
http://bitbucket.org/pypy/pypy/changeset/f6dc12d932d9/

Log:	fix bug

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -62,12 +62,11 @@
             inputarg = block.inputargs[argindex]
             # bit slow, but probably ok
             firstlinkarg = self.variable_families.find_rep(firstlink.args[argindex])
-            results = []
             for key, res in self.purecache.iteritems():
                 (opname, concretetype, args) = key
                 if args[0] != firstlinkarg: # XXX other args
                     continue
-                results.append(res)
+                results = [res]
                 for linkindex, (link, cache) in enumerate(tuples):
                     if linkindex == 0:
                         continue
@@ -85,6 +84,7 @@
                     newres = res
                     if isinstance(res, Variable):
                         newres = res.copy()
+                        assert len(results) == len(tuples)
                         for linkindex, (link, cache) in enumerate(tuples):
                             link.args.append(results[linkindex])
                         block.inputargs.append(newres)
@@ -118,12 +118,11 @@
             inputarg = block.inputargs[argindex]
             # bit slow, but probably ok
             firstlinkarg = self.variable_families.find_rep(firstlink.args[argindex])
-            results = []
             for key, res in self.heapcache.iteritems():
                 (arg, fieldname) = key
                 if arg != firstlinkarg:
                     continue
-                results.append(res)
+                results = [res]
                 for linkindex, (link, cache) in enumerate(tuples):
                     if linkindex == 0:
                         continue
@@ -299,6 +298,7 @@
                     if exit.target not in done and exit.target not in todo: # XXX
                         todo.append(exit.target)
                 caches_to_merge[exit.target].append((exit, cache))
+        simplify.transform_dead_op_vars(graph)
         if added_same_as:
             ssa.SSA_to_SSI(graph)
             removenoops.remove_same_as(graph)
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -323,3 +323,30 @@
                 res += i + 1
             return a + (i + 1)
         self.check(f, [int], add=0)
+
+    def test_bug_2(self):
+        class A(object):
+            def getslice(self, a, b):
+                return "a" * a * b
+
+        def make(i):
+            a = A()
+            a.size = i * 12
+            a.pos = i * 54
+            return a
+
+        def read(i, num):
+            self = make(i)
+            if num < 0:
+                # read all
+                eol = self.size
+            else:
+                eol = self.pos + num
+                if eol > self.size:
+                    eol = self.size
+
+            res = self.getslice(self.pos, eol - self.pos)
+            self.pos += len(res)
+            return res
+        self.check(read, [int, int])
+

From pypy.commits at gmail.com  Fri Sep 16 17:13:06 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:06 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: refactor
Message-ID: <57dc6062.4756c20a.b4de8.a82d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87161:4432c08bf80b
Date: 2016-09-09 13:00 +0200
http://bitbucket.org/pypy/pypy/changeset/4432c08bf80b/

Log:	refactor

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -46,6 +46,38 @@
                 self.purecache.copy(),
                 self.heapcache.copy())
 
+    def _var_rep(self, var):
+        return self.variable_families.find_rep(var)
+
+    def _key_with_replacement(self, key, index, var):
+        (opname, concretetype, args) = key
+        listargs = list(args)
+        listargs[index] = self._var_rep(var)
+        return (opname, concretetype, tuple(listargs))
+
+    def _merge_results(self, tuples, results, backedges):
+        assert len(results) == len(tuples)
+        for result in results:
+            if isinstance(result, Variable):
+                newres = result.copy()
+                break
+        else:
+            # all constants! check if all the same
+            const = results[0]
+            for result in results:
+                if result != const:
+                    newres = Variable()
+                    newres.concretetype = const.concretetype
+                    break
+            else:
+                # all the same
+                return const
+        for linkindex, (link, cache) in enumerate(tuples):
+            link.args.append(results[linkindex])
+        tuples[0][0].target.inputargs.append(newres)
+        for backedge in backedges:
+            backedge.args.append(newres)
+        return newres
 
     def merge(self, firstlink, tuples, backedges):
         purecache = {}
@@ -61,7 +93,7 @@
         for argindex in range(len(block.inputargs)):
             inputarg = block.inputargs[argindex]
             # bit slow, but probably ok
-            firstlinkarg = self.variable_families.find_rep(firstlink.args[argindex])
+            firstlinkarg = self._var_rep(firstlink.args[argindex])
             for key, res in self.purecache.iteritems():
                 (opname, concretetype, args) = key
                 if args[0] != firstlinkarg: # XXX other args
@@ -70,43 +102,28 @@
                 for linkindex, (link, cache) in enumerate(tuples):
                     if linkindex == 0:
                         continue
-                    listargs = list(args)
-                    listargs[0] = self.variable_families.find_rep(link.args[argindex])
-                    newkey = (opname, concretetype, tuple(listargs))
+                    newkey = self._key_with_replacement(
+                            key, 0, link.args[argindex])
                     otherres = cache.purecache.get(newkey, None)
                     if otherres is None:
                         break
                     results.append(otherres)
                 else:
-                    listargs = list(args)
-                    listargs[0] = self.variable_families.find_rep(inputarg)
-                    newkey = (opname, concretetype, tuple(listargs))
-                    newres = res
-                    if isinstance(res, Variable):
-                        newres = res.copy()
-                        assert len(results) == len(tuples)
-                        for linkindex, (link, cache) in enumerate(tuples):
-                            link.args.append(results[linkindex])
-                        block.inputargs.append(newres)
-                        for backedge in backedges:
-                            backedge.args.append(newres)
+                    newkey = self._key_with_replacement(
+                            key, 0, inputarg)
+                    newres = self._merge_results(tuples, results, backedges)
                     purecache[newkey] = newres
 
         for key, res in self.purecache.iteritems():
             # "straight" merge: the variable is in all other caches
+            results = [res]
             for link, cache in tuples[1:]:
                 val = cache.purecache.get(key, None)
                 if val is None:
                     break
+                results.append(val)
             else:
-                newres = res
-                if isinstance(res, Variable):
-                    newres = res.copy()
-                    for link, cache in tuples:
-                        link.args.append(cache.purecache[key])
-                    block.inputargs.append(newres)
-                    for backedge in backedges:
-                        backedge.args.append(newres)
+                newres = self._merge_results(tuples, results, backedges)
                 purecache[key] = newres
 
         # ______________________
@@ -117,7 +134,7 @@
         for argindex in range(len(block.inputargs)):
             inputarg = block.inputargs[argindex]
             # bit slow, but probably ok
-            firstlinkarg = self.variable_families.find_rep(firstlink.args[argindex])
+            firstlinkarg = self._var_rep(firstlink.args[argindex])
             for key, res in self.heapcache.iteritems():
                 (arg, fieldname) = key
                 if arg != firstlinkarg:
@@ -126,43 +143,28 @@
                 for linkindex, (link, cache) in enumerate(tuples):
                     if linkindex == 0:
                         continue
-                    otherarg = self.variable_families.find_rep(link.args[argindex])
+                    otherarg = self._var_rep(link.args[argindex])
                     newkey = (otherarg, fieldname)
                     otherres = cache.heapcache.get(newkey, None)
                     if otherres is None:
                         break
                     results.append(otherres)
                 else:
-                    newkey = (self.variable_families.find_rep(inputarg), fieldname)
-                    newres = res
-                    if isinstance(res, Variable):
-                        newres = res.copy()
-                        for linkindex, (link, cache) in enumerate(tuples):
-                            link.args.append(results[linkindex])
-                        block.inputargs.append(newres)
-                        for backedge in backedges:
-                            backedge.args.append(newres)
+                    newkey = (self._var_rep(inputarg), fieldname)
+                    newres = self._merge_results(tuples, results, backedges)
                     heapcache[newkey] = newres
 
         # regular merge
         for key, res in self.heapcache.iteritems():
+            results = [res]
             for link, cache in tuples[1:]:
                 val = cache.heapcache.get(key, None)
                 if val is None:
                     break
+                results.append(val)
             else:
-                newres = res
-                if isinstance(res, Variable):
-                    newres = res.copy()
-                    for link, cache in tuples:
-                        link.args.append(cache.heapcache[key])
-                    block.inputargs.append(newres)
-                    for backedge in backedges:
-                        backedge.args.append(newres)
+                newres = self._merge_results(tuples, results, backedges)
                 heapcache[key] = newres
-
-
-
         return Cache(
                 self.variable_families, self.analyzer, purecache, heapcache)
 
@@ -195,7 +197,7 @@
     def cse_block(self, block):
         def representative_arg(arg):
             if isinstance(arg, Variable):
-                return self.variable_families.find_rep(arg)
+                return self._var_rep(arg)
             return arg
         added_same_as = 0
         for op in block.operations:

From pypy.commits at gmail.com  Fri Sep 16 17:13:08 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:08 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: compute the merged result more
 systematically
Message-ID: <57dc6064.448e1c0a.50ced.54bd@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87162:4570a9abf4c0
Date: 2016-09-09 13:41 +0200
http://bitbucket.org/pypy/pypy/changeset/4570a9abf4c0/

Log:	compute the merged result more systematically

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -55,24 +55,31 @@
         listargs[index] = self._var_rep(var)
         return (opname, concretetype, tuple(listargs))
 
+    def _find_new_res(self, results):
+        # helper function for _merge_results
+        first = self._var_rep(results[0])
+        newres = None
+        for result in results:
+            if newres is None and isinstance(result, Variable):
+                # some extra work to get nice var names
+                newres = result.copy()
+            result = self._var_rep(result)
+            if result != first:
+                break
+        else:
+            # all the same!
+            return first, False
+        if newres is None:
+            newres = Variable()
+            newres.concretetype = first.concretetype
+        return newres, True
+
     def _merge_results(self, tuples, results, backedges):
         assert len(results) == len(tuples)
-        for result in results:
-            if isinstance(result, Variable):
-                newres = result.copy()
-                break
-        else:
-            # all constants! check if all the same
-            const = results[0]
-            for result in results:
-                if result != const:
-                    newres = Variable()
-                    newres.concretetype = const.concretetype
-                    break
-            else:
-                # all the same
-                return const
-        for linkindex, (link, cache) in enumerate(tuples):
+        newres, needs_adding = self._find_new_res(results)
+        if not needs_adding:
+            return newres
+        for linkindex, (link, _) in enumerate(tuples):
             link.args.append(results[linkindex])
         tuples[0][0].target.inputargs.append(newres)
         for backedge in backedges:
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -1,6 +1,7 @@
 import pytest
 from rpython.translator.translator import TranslationContext, graphof
-from rpython.translator.backendopt.cse import CSE
+from rpython.translator.backendopt.cse import CSE, Cache
+from rpython.flowspace.model import Variable, Constant
 from rpython.translator.backendopt import removenoops
 from rpython.flowspace.model import checkgraph, summary
 from rpython.conftest import option
@@ -350,3 +351,56 @@
             return res
         self.check(read, [int, int])
 
+def fakevar(name='v'):
+    var = Variable(name)
+    var.concretetype = "fake concrete type"
+    return var
+
+def fakeconst(val):
+    const = Constant(val)
+    const.concretetype = "fake concrete type"
+    return const
+
+def test_find_new_res():
+    # unit test for _find_new_res
+
+    class FakeFamilies(object):
+        def find_rep(self, var):
+            return reps.get(var, var)
+
+    reps = {}
+    c = Cache(FakeFamilies(), None)
+
+    # two different vars
+    v1 = fakevar()
+    v2 = fakevar()
+    res, needs_adding = c._find_new_res([v1, v2])
+    assert needs_adding
+    assert isinstance(res, Variable)
+    assert res.concretetype == v1.concretetype
+
+    # the same var
+    res, needs_adding = c._find_new_res([v1, v1])
+    assert not needs_adding
+    assert res is v1
+
+    # different vars, but same reps
+    reps = {v2: v1}
+    res, needs_adding = c._find_new_res([v1, v2])
+    assert not needs_adding
+    assert res is v1
+
+    # two different consts
+    c1 = fakeconst(1)
+    c2 = fakeconst(2)
+    res, needs_adding = c._find_new_res([c1, c2])
+    assert needs_adding
+    assert isinstance(res, Variable)
+    assert res.concretetype == v1.concretetype
+
+    # the same const
+    c1 = fakeconst(1)
+    res, needs_adding = c._find_new_res([c1, c1])
+    assert not needs_adding
+    assert res is c1
+

From pypy.commits at gmail.com  Fri Sep 16 17:13:12 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:12 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: constant-fold reads from
 immutable fields of constants
Message-ID: <57dc6068.87941c0a.4e555.55df@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87164:d2cc6120b553
Date: 2016-09-09 18:54 +0200
http://bitbucket.org/pypy/pypy/changeset/d2cc6120b553/

Log:	constant-fold reads from immutable fields of constants

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -2,7 +2,7 @@
 
 from rpython.translator.backendopt import support
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.flowspace.model import mkentrymap, Variable
+from rpython.flowspace.model import mkentrymap, Variable, Constant
 from rpython.translator.backendopt import removenoops
 from rpython.translator import simplify
 from rpython.translator.backendopt import ssa
@@ -212,8 +212,19 @@
         for op in block.operations:
             # heap operations
             if op.opname == 'getfield':
-                tup = (representative_arg(op.args[0]), op.args[1].value)
-                res = self.heapcache.get(tup, None)
+                fieldname = op.args[1].value
+                arg0 = representative_arg(op.args[0])
+                res = None
+                if isinstance(arg0, Constant):
+                    PTRTYPE = arg0.concretetype.TO
+                    if PTRTYPE._immutable_field(fieldname).is_immutable:
+                        # can constant-fold:
+                        FIELDTYPE = getattr(PTRTYPE, fieldname)
+                        value = getattr(arg0.value, fieldname)
+                        res = Constant(value, FIELDTYPE)
+                if res is None:
+                    tup = (arg0, fieldname)
+                    res = self.heapcache.get(tup, None)
                 if res is not None:
                     op.opname = 'same_as'
                     op.args = [res]
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -374,6 +374,21 @@
             return res
         self.check(read, [int, int])
 
+    def test_immutable_getfield(self):
+        class A(object):
+            _immutable_fields_ = ['a']
+            def __init__(self, a):
+                self.a = a
+        a1 = A(5)
+        a2 = A(8)
+
+        def read(i):
+            if i:
+                return a1.a
+            return a2.a
+        self.check(read, [int], getfield=0)
+
+
 def fakevar(name='v'):
     var = Variable(name)
     var.concretetype = "fake concrete type"

From pypy.commits at gmail.com  Fri Sep 16 17:13:15 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:15 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: a small tweak and two XXXs
Message-ID: <57dc606b.9102c20a.7fd22.a6fb@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87166:fee47d3d64bd
Date: 2016-09-12 19:33 +0200
http://bitbucket.org/pypy/pypy/changeset/fee47d3d64bd/

Log:	a small tweak and two XXXs

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -183,6 +183,8 @@
                 del self.heapcache[k]
 
     def _clear_heapcache_for_effects_of_op(self, op):
+        if not self.heapcache:
+            return
         effects = self.analyzer.analyze(op)
         self._clear_heapcache_for_effects(effects)
 
@@ -191,11 +193,13 @@
             self.heapcache.clear()
         else:
             for k in self.heapcache.keys():
+                # XXX slow
                 key = ('struct', k[0].concretetype, k[1])
                 if key in effects:
                     del self.heapcache[k]
 
     def _clear_heapcache_for_loop_blocks(self, blocks):
+        # XXX use result builder
         effects = self.analyzer.bottom_result()
         for block in blocks:
             for op in block.operations:

From pypy.commits at gmail.com  Fri Sep 16 17:13:10 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:10 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: safer mechanism for storing
 new equivalencies than updating the variable_families
Message-ID: <57dc6066.6773c20a.b2ece.b2e5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87163:a01cd9612fb4
Date: 2016-09-09 15:26 +0200
http://bitbucket.org/pypy/pypy/changeset/a01cd9612fb4/

Log:	safer mechanism for storing new equivalencies than updating the
	variable_families

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -39,6 +39,7 @@
         self.heapcache = heapcache
         self.variable_families = variable_families
         self.analyzer = analyzer
+        self.new_unions = {} # mapping var from this block -> older var
 
     def copy(self):
         return Cache(
@@ -47,6 +48,7 @@
                 self.heapcache.copy())
 
     def _var_rep(self, var):
+        var = self.new_unions.get(var, var)
         return self.variable_families.find_rep(var)
 
     def _key_with_replacement(self, key, index, var):
@@ -216,6 +218,7 @@
                     op.opname = 'same_as'
                     op.args = [res]
                     added_same_as += 1
+                    self.new_unions[op.result] = res
                 else:
                     self.heapcache[tup] = op.result
                 continue
@@ -239,7 +242,7 @@
                 op.opname = 'same_as'
                 op.args = [res]
                 added_same_as += 1
-                self.variable_families.union(res, op.result)
+                self.new_unions[op.result] = res
             else:
                 self.purecache[key] = op.result
         return added_same_as
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -165,6 +165,29 @@
 
         self.check(f, [int], getfield=1)
 
+    def test_two_getfields(self):
+        class A(object):
+            pass
+        class B(object):
+            pass
+        a1 = A()
+        a1.next = B()
+        a1.next.x = 1
+        a2 = A()
+        a2.next = B()
+        a2.next.x = 5
+
+
+        def f(i):
+            if i:
+                a = a1
+            else:
+                a = a2
+            return a.next.x + a.next.x + i
+
+        self.check(f, [int], getfield=2)
+
+
     def test_bug_1(self):
         class A(object):
             pass

From pypy.commits at gmail.com  Fri Sep 16 17:13:17 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:17 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: used the wrong cache in two
 places while merging
Message-ID: <57dc606d.a3adc20a.6cedf.ac20@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87167:7f1299b60985
Date: 2016-09-12 19:49 +0200
http://bitbucket.org/pypy/pypy/changeset/7f1299b60985/

Log:	used the wrong cache in two places while merging

	(probably benign, but leads to less good code)

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -111,7 +111,7 @@
                 for linkindex, (link, cache) in enumerate(tuples):
                     if linkindex == 0:
                         continue
-                    newkey = self._key_with_replacement(
+                    newkey = cache._key_with_replacement(
                             key, 0, link.args[argindex])
                     otherres = cache.purecache.get(newkey, None)
                     if otherres is None:
@@ -152,7 +152,7 @@
                 for linkindex, (link, cache) in enumerate(tuples):
                     if linkindex == 0:
                         continue
-                    otherarg = self._var_rep(link.args[argindex])
+                    otherarg = cache._var_rep(link.args[argindex])
                     newkey = (otherarg, fieldname)
                     otherres = cache.heapcache.get(newkey, None)
                     if otherres is None:
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -401,6 +401,26 @@
             return a2.a
         self.check(read, [int], getfield=2)
 
+    def test_cast_pointer_introduce_aliases(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        class C(B):
+            pass
+        def f(i):
+            res = 0
+            if i > 10:
+                if i > 20:
+                    a = C()
+                    a.x = 1
+                else:
+                    a = B()
+                    a.x = 2
+                # here a is a subclass of B
+                res += a.x
+            return res
+        self.check(f, [int], getfield=0)
 
 
 def fakevar(name='v'):

From pypy.commits at gmail.com  Fri Sep 16 17:13:14 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:14 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: fix immutable checks
Message-ID: <57dc606a.43e61c0a.64418.7134@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87165:8999d5b9bffd
Date: 2016-09-12 19:33 +0200
http://bitbucket.org/pypy/pypy/changeset/8999d5b9bffd/

Log:	fix immutable checks

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -217,7 +217,7 @@
                 res = None
                 if isinstance(arg0, Constant):
                     PTRTYPE = arg0.concretetype.TO
-                    if PTRTYPE._immutable_field(fieldname).is_immutable:
+                    if PTRTYPE._immutable_field(fieldname):
                         # can constant-fold:
                         FIELDTYPE = getattr(PTRTYPE, fieldname)
                         value = getattr(arg0.value, fieldname)
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -388,6 +388,20 @@
             return a2.a
         self.check(read, [int], getfield=0)
 
+        # not immutable
+        class A(object):
+            def __init__(self, a):
+                self.a = a
+        a1 = A(5)
+        a2 = A(8)
+
+        def read(i):
+            if i:
+                return a1.a
+            return a2.a
+        self.check(read, [int], getfield=2)
+
+
 
 def fakevar(name='v'):
     var = Variable(name)

From pypy.commits at gmail.com  Fri Sep 16 17:13:19 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:19 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: make cast_pointer introduce a
 union to be able to track things better
Message-ID: <57dc606f.2472c20a.2f30e.ab0c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87168:8c2f9afa87bb
Date: 2016-09-13 14:07 +0200
http://bitbucket.org/pypy/pypy/changeset/8c2f9afa87bb/

Log:	make cast_pointer introduce a union to be able to track things
	better

	this requires the keys of the heapcache to also store the
	concretetype of the variable where the actual setfield/getfield is
	performed

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -2,6 +2,7 @@
 
 from rpython.translator.backendopt import support
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem import lltype
 from rpython.flowspace.model import mkentrymap, Variable, Constant
 from rpython.translator.backendopt import removenoops
 from rpython.translator import simplify
@@ -48,7 +49,11 @@
                 self.heapcache.copy())
 
     def _var_rep(self, var):
-        var = self.new_unions.get(var, var)
+        while True:
+            newvar = self.new_unions.get(var, None)
+            if newvar is None:
+                break
+            var = newvar # can take several dereferences
         return self.variable_families.find_rep(var)
 
     def _key_with_replacement(self, key, index, var):
@@ -105,7 +110,7 @@
             firstlinkarg = self._var_rep(firstlink.args[argindex])
             for key, res in self.purecache.iteritems():
                 (opname, concretetype, args) = key
-                if args[0] != firstlinkarg: # XXX other args
+                if self._var_rep(args[0]) != firstlinkarg: # XXX other args
                     continue
                 results = [res]
                 for linkindex, (link, cache) in enumerate(tuples):
@@ -145,21 +150,21 @@
             # bit slow, but probably ok
             firstlinkarg = self._var_rep(firstlink.args[argindex])
             for key, res in self.heapcache.iteritems():
-                (arg, fieldname) = key
-                if arg != firstlinkarg:
+                (arg, concretetype, fieldname) = key
+                if self._var_rep(arg) != firstlinkarg:
                     continue
                 results = [res]
                 for linkindex, (link, cache) in enumerate(tuples):
                     if linkindex == 0:
                         continue
                     otherarg = cache._var_rep(link.args[argindex])
-                    newkey = (otherarg, fieldname)
+                    newkey = (otherarg, concretetype, fieldname)
                     otherres = cache.heapcache.get(newkey, None)
                     if otherres is None:
                         break
                     results.append(otherres)
                 else:
-                    newkey = (self._var_rep(inputarg), fieldname)
+                    newkey = (self._var_rep(inputarg), concretetype, fieldname)
                     newres = self._merge_results(tuples, results, backedges)
                     heapcache[newkey] = newres
 
@@ -179,7 +184,7 @@
 
     def _clear_heapcache_for(self, concretetype, fieldname):
         for k in self.heapcache.keys():
-            if k[0].concretetype == concretetype and k[1] == fieldname:
+            if k[1] == concretetype and k[2] == fieldname:
                 del self.heapcache[k]
 
     def _clear_heapcache_for_effects_of_op(self, op):
@@ -194,7 +199,7 @@
         else:
             for k in self.heapcache.keys():
                 # XXX slow
-                key = ('struct', k[0].concretetype, k[1])
+                key = ('struct', k[1], k[2])
                 if key in effects:
                     del self.heapcache[k]
 
@@ -217,17 +222,19 @@
             # heap operations
             if op.opname == 'getfield':
                 fieldname = op.args[1].value
+                concretetype = op.args[0].concretetype
                 arg0 = representative_arg(op.args[0])
                 res = None
                 if isinstance(arg0, Constant):
-                    PTRTYPE = arg0.concretetype.TO
+                    PTRTYPE = concretetype.TO
                     if PTRTYPE._immutable_field(fieldname):
                         # can constant-fold:
                         FIELDTYPE = getattr(PTRTYPE, fieldname)
-                        value = getattr(arg0.value, fieldname)
+                        const = lltype.cast_pointer(concretetype, arg0.value)
+                        value = getattr(const, fieldname)
                         res = Constant(value, FIELDTYPE)
                 if res is None:
-                    tup = (arg0, fieldname)
+                    tup = (arg0, op.args[0].concretetype, fieldname)
                     res = self.heapcache.get(tup, None)
                 if res is not None:
                     op.opname = 'same_as'
@@ -238,10 +245,11 @@
                     self.heapcache[tup] = op.result
                 continue
             if op.opname == 'setfield':
+                concretetype = op.args[0].concretetype
                 target = representative_arg(op.args[0])
                 field = op.args[1].value
-                self._clear_heapcache_for(target.concretetype, field)
-                self.heapcache[target, field] = op.args[2]
+                self._clear_heapcache_for(concretetype, field)
+                self.heapcache[target, concretetype, field] = op.args[2]
                 continue
             if has_side_effects(op):
                 self._clear_heapcache_for_effects_of_op(op)
@@ -260,6 +268,11 @@
                 self.new_unions[op.result] = res
             else:
                 self.purecache[key] = op.result
+            if op.opname == "cast_pointer":
+                # cast_pointer is a pretty strange operation! it introduces
+                # more aliases, that confuse the CSE pass. Therefore we unify
+                # the two variables in new_unions, to improve the folding.
+                self.new_unions[op.result] = op.args[0]
         return added_same_as
 
 def _merge(tuples, variable_families, analyzer, loop_blocks, backedges):
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -419,9 +419,45 @@
                     a.x = 2
                 # here a is a subclass of B
                 res += a.x
+            else:
+                a = A()
+            res += a.__class__ is A
             return res
         self.check(f, [int], getfield=0)
 
+    def test_cast_pointer_leading_to_constant(self):
+        class Cls(object):
+            pass
+        class Sub(Cls):
+            pass
+        cls1 = Cls()
+        cls2 = Sub()
+        cls2.user_overridden_class = True
+        cls3 = Sub()
+        cls3.user_overridden_class = False
+        class A(object):
+            pass
+        def f(i):
+            res = 0
+            if i > 20:
+                a = A()
+                a.cls = cls1
+                return 1
+            elif i > 30:
+                a = A()
+                a.cls = cls2
+                cls = a.cls
+                assert type(cls) is Sub
+                return cls.user_overridden_class
+            else:
+                a = A()
+                a.cls = cls3
+                cls = a.cls
+                assert type(cls) is Sub
+                return cls.user_overridden_class
+        self.check(f, [int], getfield=2)
+
+
 
 def fakevar(name='v'):
     var = Variable(name)

From pypy.commits at gmail.com  Fri Sep 16 17:13:21 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:21 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: just call constfold after cse
 to remove constant operations, instead of doing it
Message-ID: <57dc6071.ce471c0a.6cd2f.b37a@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87169:a9034e8630f8
Date: 2016-09-13 14:20 +0200
http://bitbucket.org/pypy/pypy/changeset/a9034e8630f8/

Log:	just call constfold after cse to remove constant operations, instead
	of doing it badly in CSE itself

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -6,7 +6,7 @@
 from rpython.flowspace.model import mkentrymap, Variable, Constant
 from rpython.translator.backendopt import removenoops
 from rpython.translator import simplify
-from rpython.translator.backendopt import ssa
+from rpython.translator.backendopt import ssa, constfold
 from rpython.translator.backendopt.writeanalyze import WriteAnalyzer
 
 from rpython.translator.backendopt.support import log
@@ -224,18 +224,8 @@
                 fieldname = op.args[1].value
                 concretetype = op.args[0].concretetype
                 arg0 = representative_arg(op.args[0])
-                res = None
-                if isinstance(arg0, Constant):
-                    PTRTYPE = concretetype.TO
-                    if PTRTYPE._immutable_field(fieldname):
-                        # can constant-fold:
-                        FIELDTYPE = getattr(PTRTYPE, fieldname)
-                        const = lltype.cast_pointer(concretetype, arg0.value)
-                        value = getattr(const, fieldname)
-                        res = Constant(value, FIELDTYPE)
-                if res is None:
-                    tup = (arg0, op.args[0].concretetype, fieldname)
-                    res = self.heapcache.get(tup, None)
+                tup = (arg0, op.args[0].concretetype, fieldname)
+                res = self.heapcache.get(tup, None)
                 if res is not None:
                     op.opname = 'same_as'
                     op.args = [res]
@@ -342,6 +332,7 @@
         if added_same_as:
             ssa.SSA_to_SSI(graph)
             removenoops.remove_same_as(graph)
+            constfold.constant_fold_graph(graph) # make use of extra constants
         simplify.transform_dead_op_vars(graph)
         if added_same_as:
             if self.translator.config.translation.verbose:
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -379,13 +379,18 @@
             _immutable_fields_ = ['a']
             def __init__(self, a):
                 self.a = a
+        class B(object):
+            pass
         a1 = A(5)
         a2 = A(8)
 
         def read(i):
+            b = B()
             if i:
-                return a1.a
-            return a2.a
+                b.a = a1
+                return b.a.a
+            b.a = a2
+            return b.a.a
         self.check(read, [int], getfield=0)
 
         # not immutable
@@ -429,7 +434,7 @@
         class Cls(object):
             pass
         class Sub(Cls):
-            pass
+            _immutable_fields_ = ['user_overridden_class']
         cls1 = Cls()
         cls2 = Sub()
         cls2.user_overridden_class = True
@@ -455,7 +460,7 @@
                 cls = a.cls
                 assert type(cls) is Sub
                 return cls.user_overridden_class
-        self.check(f, [int], getfield=2)
+        self.check(f, [int], getfield=0)
 
 
 

From pypy.commits at gmail.com  Fri Sep 16 17:13:23 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:23 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: just use a second union find
Message-ID: <57dc6073.948e1c0a.3fc8f.5aeb@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87170:a717e883e603
Date: 2016-09-13 19:56 +0200
http://bitbucket.org/pypy/pypy/changeset/a717e883e603/

Log:	just use a second union find

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -8,6 +8,7 @@
 from rpython.translator import simplify
 from rpython.translator.backendopt import ssa, constfold
 from rpython.translator.backendopt.writeanalyze import WriteAnalyzer
+from rpython.tool.algo import unionfind
 
 from rpython.translator.backendopt.support import log
 
@@ -31,29 +32,28 @@
     return getattr(llop, op.opname).canfold
 
 class Cache(object):
-    def __init__(self, variable_families, analyzer, purecache=None, heapcache=None):
+    def __init__(self, variable_families, analyzer, new_unions=None,
+                 purecache=None, heapcache=None):
         if purecache is None:
             purecache = {}
         if heapcache is None:
             heapcache = {}
+        if new_unions is None:
+            new_unions = unionfind.UnionFind()
         self.purecache = purecache
         self.heapcache = heapcache
         self.variable_families = variable_families
         self.analyzer = analyzer
-        self.new_unions = {} # mapping var from this block -> older var
+        self.new_unions = new_unions
 
     def copy(self):
         return Cache(
-                self.variable_families, self.analyzer,
+                self.variable_families, self.analyzer, self.new_unions,
                 self.purecache.copy(),
                 self.heapcache.copy())
 
     def _var_rep(self, var):
-        while True:
-            newvar = self.new_unions.get(var, None)
-            if newvar is None:
-                break
-            var = newvar # can take several dereferences
+        var = self.new_unions.find_rep(var)
         return self.variable_families.find_rep(var)
 
     def _key_with_replacement(self, key, index, var):
@@ -180,7 +180,8 @@
                 newres = self._merge_results(tuples, results, backedges)
                 heapcache[key] = newres
         return Cache(
-                self.variable_families, self.analyzer, purecache, heapcache)
+                self.variable_families, self.analyzer, self.new_unions,
+                purecache, heapcache)
 
     def _clear_heapcache_for(self, concretetype, fieldname):
         for k in self.heapcache.keys():
@@ -230,16 +231,16 @@
                     op.opname = 'same_as'
                     op.args = [res]
                     added_same_as += 1
-                    self.new_unions[op.result] = res
+                    self.new_unions.union(res, op.result)
                 else:
                     self.heapcache[tup] = op.result
                 continue
             if op.opname == 'setfield':
                 concretetype = op.args[0].concretetype
                 target = representative_arg(op.args[0])
-                field = op.args[1].value
-                self._clear_heapcache_for(concretetype, field)
-                self.heapcache[target, concretetype, field] = op.args[2]
+                fieldname = op.args[1].value
+                self._clear_heapcache_for(concretetype, fieldname)
+                self.heapcache[target, concretetype, fieldname] = op.args[2]
                 continue
             if has_side_effects(op):
                 self._clear_heapcache_for_effects_of_op(op)
@@ -255,14 +256,14 @@
                 op.opname = 'same_as'
                 op.args = [res]
                 added_same_as += 1
-                self.new_unions[op.result] = res
+                self.new_unions.union(res, op.result)
             else:
                 self.purecache[key] = op.result
             if op.opname == "cast_pointer":
                 # cast_pointer is a pretty strange operation! it introduces
                 # more aliases, that confuse the CSE pass. Therefore we unify
                 # the two variables in new_unions, to improve the folding.
-                self.new_unions[op.result] = op.args[0]
+                self.new_unions.union(op.args[0], op.result)
         return added_same_as
 
 def _merge(tuples, variable_families, analyzer, loop_blocks, backedges):
@@ -311,7 +312,7 @@
             current_backedges = [link for link in entrymap[block]
                                     if link in backedges]
 
-            if block.operations:
+            if not block.is_final_block():
                 cache = _merge(
                     caches_to_merge[block], variable_families, self.analyzer,
                     loops.get(block, None), current_backedges)
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -419,13 +419,16 @@
                 if i > 20:
                     a = C()
                     a.x = 1
+                    a.__class__
                 else:
                     a = B()
                     a.x = 2
+                    a.__class__
                 # here a is a subclass of B
                 res += a.x
             else:
                 a = A()
+                a.__class__
             res += a.__class__ is A
             return res
         self.check(f, [int], getfield=0)
@@ -515,5 +518,5 @@
     c1 = fakeconst(1)
     res, needs_adding = c._find_new_res([c1, c1])
     assert not needs_adding
-    assert res is c1
+    assert res == c1
 

From pypy.commits at gmail.com  Fri Sep 16 17:13:25 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:25 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: factor out the rewriting to
 same_as, with an extra safety check
Message-ID: <57dc6075.c15e1c0a.21be8.6355@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87171:84b5c1af401b
Date: 2016-09-16 14:46 +0200
http://bitbucket.org/pypy/pypy/changeset/84b5c1af401b/

Log:	factor out the rewriting to same_as, with an extra safety check

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -213,6 +213,12 @@
                     effects, self.analyzer.analyze(op))
         self._clear_heapcache_for_effects(effects)
 
+    def _replace_with_result(self, op, res):
+        assert op.result.concretetype == res.concretetype
+        op.opname = 'same_as'
+        op.args = [res]
+        self.new_unions.union(res, op.result)
+
     def cse_block(self, block):
         def representative_arg(arg):
             if isinstance(arg, Variable):
@@ -228,14 +234,13 @@
                 tup = (arg0, op.args[0].concretetype, fieldname)
                 res = self.heapcache.get(tup, None)
                 if res is not None:
-                    op.opname = 'same_as'
-                    op.args = [res]
+                    self._replace_with_result(op, res)
                     added_same_as += 1
-                    self.new_unions.union(res, op.result)
                 else:
                     self.heapcache[tup] = op.result
                 continue
             if op.opname == 'setfield':
+                # XXX check whether value is the same already
                 concretetype = op.args[0].concretetype
                 target = representative_arg(op.args[0])
                 fieldname = op.args[1].value
@@ -253,10 +258,8 @@
                    tuple([representative_arg(arg) for arg in op.args]))
             res = self.purecache.get(key, None)
             if res is not None:
-                op.opname = 'same_as'
-                op.args = [res]
+                self._replace_with_result(op, res)
                 added_same_as += 1
-                self.new_unions.union(res, op.result)
             else:
                 self.purecache[key] = op.result
             if op.opname == "cast_pointer":

From pypy.commits at gmail.com  Fri Sep 16 17:13:26 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:26 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: be more careful about merging
 value, never make the results of find_rep the
Message-ID: <57dc6076.8aabc20a.1d096.acbc@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87172:50199f21c3b9
Date: 2016-09-16 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/50199f21c3b9/

Log:	be more careful about merging value, never make the results of
	find_rep the result of the merge. it's only for keys. add that as a
	comment.

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -53,6 +53,14 @@
                 self.heapcache.copy())
 
     def _var_rep(self, var):
+        # return the representative variable for var. All variables that must
+        # be equal to each other always have the same representative. The
+        # representative's definition dominates the use of all variables that
+        # it represents. casted pointers are considered the same objects.
+        # NB: it's very important to use _var_rep only when computing keys in
+        # the *cache dictionaries, never to actually put any new variable into
+        # the graph, because the concretetypes can change when calling
+        # _var_rep.
         var = self.new_unions.find_rep(var)
         return self.variable_families.find_rep(var)
 
@@ -75,10 +83,10 @@
                 break
         else:
             # all the same!
-            return first, False
+            return results[0], False
         if newres is None:
             newres = Variable()
-            newres.concretetype = first.concretetype
+            newres.concretetype = results[0].concretetype
         return newres, True
 
     def _merge_results(self, tuples, results, backedges):
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -520,3 +520,9 @@
     assert not needs_adding
     assert res == c1
 
+    # different vars, but same reps, 2
+    reps = {v2: v1}
+    res, needs_adding = c._find_new_res([v2, v2])
+    assert not needs_adding
+    assert res is v2
+

From pypy.commits at gmail.com  Fri Sep 16 17:13:33 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:33 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: merge default
Message-ID: <57dc607d.669fc20a.5756b.aaf1@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87173:3744e806c6b1
Date: 2016-09-16 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/3744e806c6b1/

Log:	merge default

diff too long, truncating to 2000 out of 218021 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,9 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,12 +360,15 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
 
   Heinrich-Heine University, Germany 
   Open End AB (formerly AB Strakt), Sweden
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.3
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.3"
+__version_info__ = (1, 8, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,43 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -515,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -572,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -137,6 +137,8 @@
             lib.gdbm_sync(self.__ll_dbm)
 
 def open(filename, flags='r', mode=0666):
+    if isinstance(filename, unicode):
+        filename = filename.encode()
     if flags[0] == 'r':
         iflags = lib.GDBM_READER
     elif flags[0] == 'w':
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt
--- a/pypy/doc/config/translation.profopt.txt
+++ b/pypy/doc/config/translation.profopt.txt
@@ -3,3 +3,14 @@
 RPython program) to gather profile data. Example for pypy-c: "-c 'from
 richards import main;main(); from test import pystone;
 pystone.main()'"
+
+NOTE: be aware of what this does in JIT-enabled executables.  What it
+does is instrument and later optimize the C code that happens to run in
+the example you specify, ignoring any execution of the JIT-generated
+assembler.  That means that you have to choose the example wisely.  If
+it is something that will just generate assembler and stay there, there
+is little value.  If it is something that exercises heavily library
+routines that are anyway written in C, then it will optimize that.  Most
+interesting would be something that causes a lot of JIT-compilation,
+like running a medium-sized test suite several times in a row, in order
+to optimize the warm-up in general.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,8 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.1.rst
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,218 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -0,0 +1,64 @@
+==========
+PyPy 5.4.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.4.0, released last week,
+due to the following issues:
+
+  * Update list of contributors in documentation and LICENSE file,
+    this was unfortunately left out of 5.4.0. My apologies to the new
+    contributors
+
+  * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a
+    dynamically loadable file
+
+  * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython
+
+  * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
+
+  * Fix for an issue for translating the sandbox
+
+  * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
+    the last byte of a unicode string sometimes
+
+  * Update built-in cffi_ to version 1.8.1
+
+  * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
+    translation with a message asking for help porting it
+
+  * Fix a regression where a PyBytesObject was forced (converted to a RPython
+    object) when not required, reported as issue #2395
+
+Thanks to those who reported the issues.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _cffi: https://cffi.readthedocs.io
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,146 +1,18 @@
 ==========================
-What's new in PyPy2.7 5.3+
+What's new in PyPy2.7 5.4+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
 
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
 
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
-
-.. branch: call-via-pyobj
-
-Use offsets from PyTypeObject to find actual c function to call rather than
-fixed functions, allows function override after PyType_Ready is called
-
-.. branch: issue2335
-
-Avoid exhausting the stack in the JIT due to successive guard
-failures in the same Python function ending up as successive levels of
-RPython functions, while at app-level the traceback is very short
-
-.. branch: use-madv-free
-
-Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
-not show up as a reduction of the VIRT column in ``top``, and the RES
-column might also not show the reduction, particularly on Linux >= 4.5 or
-on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
-the OS if the memory is low.
-
-.. branch: cpyext-slotdefs2
-
-Fill in more slots when creating a PyTypeObject from a W_TypeObject
-More slots are still TBD, like tp_print and richcmp
-
-.. branch: json-surrogates
-
-Align json module decode with the cpython's impl, fixes issue 2345
-
-.. branch: issue2343
-
-Copy CPython's logic more closely for handling of ``__instancecheck__()``
-and ``__subclasscheck__()``.  Fixes issue 2343.
-
-.. branch: msvcrt-cffi
-
-Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
-of ctypes. This avoids importing ctypes in many small programs and
-scripts, which in turn avoids enabling threads (because ctypes
-creates callbacks at import time, and callbacks need threads).
-
-.. branch: new-jit-log
-
-The new logging facility that integrates with and adds features to vmprof.com.
-
-.. branch: jitlog-32bit
-
-Resolve issues to use the new logging facility on a 32bit system
-
-.. branch: ep2016sprint
-
-Trying harder to make hash(-1) return -2, like it does on CPython
-
-.. branch: jitlog-exact-source-lines
-
-Log exact line positions in debug merge points.
-
-.. branch: null_byte_after_str
-
-Allocate all RPython strings with one extra byte, normally unused.
-It is used to hold a final zero in case we need some ``char *``
-representation of the string, together with checks like ``not
-can_move()`` or object pinning. Main new thing that this allows:
-``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
-importantly, CFFI calls that take directly a string as argument don't
-copy the string any more---this is like CFFI on CPython.
-
-.. branch: resource_warning
-
-Add a new command line option -X track-resources which will produce
-ResourceWarnings when the GC closes unclosed files and sockets.
-
-.. branch: cpyext-realloc
-
-Implement PyObject_Realloc
-
-.. branch: inline-blocks
-
-Improve a little bit the readability of the generated C code
-
-.. branch: improve-vmprof-testing
-
-Improved vmprof support: now tries hard to not miss any Python-level
-frame in the captured stacks, even if there is the metainterp or
-blackhole interp involved.  Also fix the stacklet (greenlet) support.
-
-.. branch: py2-mappingproxy
-
-``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
-Previously it returned what looked like a regular dict object (but it
-was already read-only).
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,165 @@
+=========================
+What's new in PyPy2.7 5.4
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -239,6 +239,10 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+            if config.translation.profopt is not None:
+                raise Exception("Cannot use the --profopt option "
+                                "when --shared is on (it is by default). "
+                                "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -208,7 +208,8 @@
     def buffer_w(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self, 
+                                        space.newint(flags))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
@@ -216,7 +217,8 @@
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
@@ -224,7 +226,8 @@
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
@@ -232,7 +235,8 @@
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
-            w_result = space.get_and_call_function(w_impl, self)
+            w_result = space.get_and_call_function(w_impl, self,
+                                        space.newint(space.BUF_FULL_RO))
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
@@ -1424,6 +1428,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
@@ -1967,6 +1974,7 @@
     'ZeroDivisionError',
     'RuntimeWarning',
     'PendingDeprecationWarning',
+    'UserWarning',
 ]
 
 if sys.platform.startswith("win"):
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -23,6 +23,14 @@
         self.w_objtype = w_type
         self.w_self = w_obj_or_type
 
+    def descr_repr(self, space):
+        if self.w_objtype is not None:
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
+        else:
+            objtype_name = 'NULL'
+        return space.wrap("<super: <class '%s'>, %s>" % (
+            self.w_starttype.getname(space), objtype_name))
+
     def get(self, space, w_obj, w_type=None):
         if self.w_self is None or space.is_w(w_obj, space.w_None):
             return self
@@ -84,7 +92,10 @@
     'super',
     __new__          = generic_new_descr(W_Super),
     __init__         = interp2app(W_Super.descr_init),
+    __repr__         = interp2app(W_Super.descr_repr),
     __thisclass__    = interp_attrproperty_w("w_starttype", W_Super),
+    __self__         = interp_attrproperty_w("w_self", W_Super),
+    __self_class__   = interp_attrproperty_w("w_objtype", W_Super),
     __getattribute__ = interp2app(W_Super.getattribute),
     __get__          = interp2app(W_Super.get),
     __doc__          =     """\
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -38,6 +38,8 @@
 
 
 class W_ClassObject(W_Root):
+    _immutable_fields_ = ['bases_w?[*]', 'w_dict?']
+
     def __init__(self, space, w_name, bases, w_dict):
         self.name = space.str_w(w_name)
         make_sure_not_resized(bases)
@@ -75,6 +77,7 @@
                             "__bases__ items must be classes")
         self.bases_w = bases_w
 
+    @jit.unroll_safe
     def is_subclass_of(self, other):
         assert isinstance(other, W_ClassObject)
         if self is other:
@@ -313,7 +316,7 @@
         # This method ignores the instance dict and the __getattr__.
         # Returns None if not found.
         assert isinstance(name, str)
-        w_value = self.w_class.lookup(space, name)
+        w_value = jit.promote(self.w_class).lookup(space, name)
         if w_value is None:
             return None
         w_descr_get = space.lookup(w_value, '__get__')
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -250,6 +250,24 @@
         assert super(B, B()).__thisclass__ is B
         assert super(A, B()).__thisclass__ is A
 
+    def test_super_self_selfclass(self):
+        class A(object):
+            pass
+        class B(A):
+            pass
+        b = B()
+        assert super(A, b).__self__ is b
+        assert super(A).__self__ is None
+        assert super(A, b).__self_class__ is B
+        assert super(A).__self_class__ is None
+
+    def test_super_repr(self):
+        class A(object):
+            def __repr__(self):
+                return super(A, self).__repr__() + '!'
+        assert repr(A()).endswith('>!')
+        assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>"
+
     def test_property_docstring(self):
         assert property.__doc__.startswith('property')
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.3"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer
 from pypy.module._cffi_backend import ctypeprim
 
 
@@ -22,6 +22,7 @@
     is_nonfunc_pointer_or_array = True
 
     def __init__(self, space, ctptr, length, arraysize, extra):
+        assert isinstance(ctptr, W_CTypePointer)
         W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
                                    ctptr.ctitem)
         self.length = length
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -35,8 +35,7 @@
         assert isinstance(ellipsis, bool)
         extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi)
         size = rffi.sizeof(rffi.VOIDP)
-        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult,
-                                could_cast_anything=False)
+        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult)
         self.fargs = fargs
         self.ellipsis = ellipsis
         self.abi = abi
@@ -59,6 +58,16 @@
                     lltype.free(self.cif_descr, flavor='raw')
                     self.cif_descr = lltype.nullptr(CIF_DESCRIPTION)
 
+    def is_unichar_ptr_or_array(self):
+        return False
+
+    def is_char_or_unichar_ptr_or_array(self):
+        return False
+
+    def string(self, cdataobj, maxlen):
+        # Can't use ffi.string() on a function pointer
+        return W_CType.string(self, cdataobj, maxlen)
+
     def new_ctypefunc_completing_argtypes(self, args_w):
         space = self.space
         nargs_declared = len(self.fargs)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -19,7 +19,6 @@
     # XXX this could be improved with an elidable method get_size()
     # that raises in case it's still -1...
 
-    cast_anything = False
     is_primitive_integer = False
     is_nonfunc_pointer_or_array = False
     is_indirect_arg_for_call_python = False
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -120,7 +120,6 @@
 
 class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar):
     _attrs_ = []
-    cast_anything = True
 
     def cast_to_int(self, cdata):
         return self.space.wrap(ord(cdata[0]))
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,12 +14,11 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _attrs_            = ['ctitem', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'accept_str', 'length']
     length = -1
 
-    def __init__(self, space, size, extra, extra_position, ctitem,
-                 could_cast_anything=True):
+    def __init__(self, space, size, extra, extra_position, ctitem):
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
@@ -27,10 +26,11 @@
         #  - for arrays, it is the array item type
         #  - for functions, it is the return type
         self.ctitem = ctitem
-        self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.accept_str = (self.can_cast_anything or
-                            (ctitem.is_primitive_integer and
-                             ctitem.size == rffi.sizeof(lltype.Char)))
+        self.accept_str = (self.is_nonfunc_pointer_or_array and
+                (isinstance(ctitem, ctypevoid.W_CTypeVoid) or
+                 isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or
+                 (ctitem.is_primitive_integer and
+                  ctitem.size == rffi.sizeof(lltype.Char))))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -137,7 +137,10 @@
 
 class W_CTypePtrBase(W_CTypePtrOrArray):
     # base class for both pointers and pointers-to-functions
-    _attrs_ = []
+    _attrs_ = ['is_void_ptr', 'is_voidchar_ptr']
+    _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr']
+    is_void_ptr = False
+    is_voidchar_ptr = False
 
     def convert_to_object(self, cdata):
         ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0]
@@ -154,7 +157,16 @@
             else:
                 raise self._convert_error("compatible pointer", w_ob)
         if self is not other:
-            if not (self.can_cast_anything or other.can_cast_anything):
+            if self.is_void_ptr or other.is_void_ptr:
+                pass     # cast from or to 'void *'
+            elif self.is_voidchar_ptr or other.is_voidchar_ptr:
+                space = self.space
+                msg = ("implicit cast from '%s' to '%s' "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" % (other.name, self.name))
+                space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1)
+            else:
                 raise self._convert_error("compatible pointer", w_ob)
 
         rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
@@ -165,8 +177,8 @@
 
 
 class W_CTypePointer(W_CTypePtrBase):
-    _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types']
-    _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr']
+    _attrs_ = ['is_file', 'cache_array_type', '_array_types']
+    _immutable_fields_ = ['is_file', 'cache_array_type?']
     kind = "pointer"
     cache_array_type = None
     is_nonfunc_pointer_or_array = True
@@ -181,6 +193,8 @@
         self.is_file = (ctitem.name == "struct _IO_FILE" or
                         ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
+        self.is_voidchar_ptr = (self.is_void_ptr or
+                           isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar))
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init, allocator):
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
                 return True
         return False
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        pass
-
     def convert_from_object(self, cdata, w_ob):
         if not self._copy_from_same(cdata, w_ob):
             self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
     )
     def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self.force_lazy_struct()
-        self._check_only_one_argument_for_union(w_ob)
 
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
-            if len(lst_w) > len(self._fields_list):
-                raise oefmt(space.w_ValueError,
-                            "too many initializers for '%s' (got %d)",
-                            self.name, len(lst_w))
-            for i in range(len(lst_w)):
-                optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+            j = 0
+            for w_obj in lst_w:
+                try:
+                    while (self._fields_list[j].flags &
+                               W_CField.BF_IGNORE_IN_CTOR):
+                        j += 1
+                except IndexError:
+                    raise oefmt(space.w_ValueError,
+                                "too many initializers for '%s' (got %d)",
+                                self.name, len(lst_w))
+                optvarsize = self._fields_list[j].write_v(cdata, w_obj,
                                                           optvarsize)
+                j += 1
             return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
 class W_CTypeUnion(W_CTypeStructOrUnion):
     kind = "union"
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        space = self.space
-        n = space.int_w(space.len(w_ob))
-        if n > 1:
-            raise oefmt(space.w_ValueError,
-                        "initializer for '%s': %d items given, but only one "
-                        "supported (use a dict if needed)", self.name, n)
-
 
 class W_CField(W_Root):
     _immutable_ = True
@@ -200,18 +194,21 @@
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
 
-    def __init__(self, ctype, offset, bitshift, bitsize):
+    BF_IGNORE_IN_CTOR = 0x01
+
+    def __init__(self, ctype, offset, bitshift, bitsize, flags):
         self.ctype = ctype
         self.offset = offset
         self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
         self.bitsize = bitsize
+        self.flags = flags       # BF_xxx
 
     def is_bitfield(self):
         return self.bitshift >= 0
 
-    def make_shifted(self, offset):
+    def make_shifted(self, offset, fflags):
         return W_CField(self.ctype, offset + self.offset,
-                        self.bitshift, self.bitsize)
+                        self.bitshift, self.bitsize, self.flags | fflags)
 
     def read(self, cdata):
         cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
     offset = interp_attrproperty('offset', W_CField),
     bitshift = interp_attrproperty('bitshift', W_CField),
     bitsize = interp_attrproperty('bitsize', W_CField),
+    flags = interp_attrproperty('flags', W_CField),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -7,7 +7,6 @@
 
 class W_CTypeVoid(W_CType):
     _attrs_ = []
-    cast_anything = True
     kind = "void"
 
     def __init__(self, space):
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -32,8 +32,8 @@
 @unwrap_spec(w_cdata=cdataobj.W_CData)
 def from_handle(space, w_cdata):
     ctype = w_cdata.ctype
-    if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or
-        not ctype.can_cast_anything):
+    if (not isinstance(ctype, ctypeptr.W_CTypePointer) or
+        not ctype.is_voidchar_ptr):
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
         if alignment < falign and do_align:
             alignment = falign
         #
+        if is_union and i > 0:
+            fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+        else:
+            fflags = 0
+        #
         if fbitsize < 0:
             # not a bitfield: common case
 
@@ -372,7 +377,7 @@
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8)
+                    fld = srcfld.make_shifted(boffset // 8, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                                           fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
@@ -489,7 +495,7 @@
                     bitshift = 8 * ftype.size - fbitsize- bitshift
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
-                                           bitshift, fbitsize)
+                                           bitshift, fbitsize, fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if
@@ -3646,3 +3665,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -503,3 +503,10 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_bug_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        q = ffi.new("char[]", "abcd")
+        p = ffi.cast("char(*)(void)", q)
+        raises(TypeError, ffi.string, p)
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.mixedmodule import MixedModule 
+from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
 
@@ -7,7 +7,7 @@
 
     interpleveldefs = {
         'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
-        'MAGIC':          'space.wrap(interp_sre.MAGIC)',
+        'MAGIC':          'space.newint(20031017)',
         'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -14,7 +14,7 @@
 # Constants and exposed functions
 
 from rpython.rlib.rsre import rsre_core
-from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db
+from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db
 
 
 @unwrap_spec(char_ord=int, flags=int)
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -450,7 +450,12 @@
                 # For compatibility
                 assert exc.value.errno == _ssl.SSL_ERROR_WANT_READ
             finally:
-                c.shutdown()
+                try:
+                    c.shutdown()
+                except _ssl.SSLError:
+                    # If the expected exception was raised, the SSLContext
+                    # can't be shut down yet
+                    pass
         finally:
             s.close()
 
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -358,9 +358,15 @@
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
-            return space.wrap("")
-        s = rffi.charp2strn(rffi.cast(rffi.CCHARP, buf), buflen)
-        return space.wrap(s)
+            s = ""
+        else:
+            # may or may not have a trailing NULL in the buffer.
+            buf = rffi.cast(rffi.CCHARP, buf)
+            if buf[buflen - 1] == '\x00':
+                buflen -= 1
+            s = rffi.charp2strn(buf, buflen)
+        w_s = space.wrap(s)
+        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -460,7 +466,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(retType[0]),
+                        space.wrap(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -612,7 +618,7 @@
                                 space.wrap(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(retType[0]),
+                                space.wrap(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -151,6 +151,7 @@
 
     def test_readValues(self):
         from _winreg import OpenKey, EnumValue, QueryValueEx, EnumKey
+        from _winreg import REG_SZ, REG_EXPAND_SZ
         key = OpenKey(self.root_key, self.test_key_name)
         sub_key = OpenKey(key, "sub_key")
         index = 0
@@ -164,7 +165,10 @@
         assert index == len(self.test_data)
 
         for name, value, type in self.test_data:
-            assert QueryValueEx(sub_key, name) == (value, type)
+            result = QueryValueEx(sub_key, name)
+            assert result == (value, type)
+            if type == REG_SZ or type == REG_EXPAND_SZ:
+                assert isinstance(result[0], unicode)     # not string
 
         assert EnumKey(key, 0) == "sub_key"
         raises(EnvironmentError, EnumKey, key, 1)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -597,6 +597,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,10 +119,10 @@
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
-METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
+METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,10 +645,14 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
         ))
+Py_bufferP = lltype.Ptr(Py_buffer)
 
 @specialize.memo()
 def is_PyObject(TYPE):
@@ -976,8 +980,10 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void,
-                                 compilation_info=eci)
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+                                  lltype.Void, compilation_info=eci)
+    def reinit_tls(space):
+        _reinit_tls()
     add_fork_hook('child', reinit_tls)
 
 def init_function(func):
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,41 +1,15 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer)
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyObject_CheckBuffer(space, w_obj):
+def PyObject_CheckBuffer(space, pyobj):
     """Return 1 if obj supports the buffer interface otherwise 0."""
-    return 0  # the bf_getbuffer field is never filled by cpyext
+    as_buffer = pyobj.c_ob_type.c_tp_as_buffer
+    flags = pyobj.c_ob_type.c_tp_flags
+    if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
+        return 1
+    return 0  

From pypy.commits at gmail.com  Fri Sep 16 17:13:35 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 16 Sep 2016 14:13:35 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: don't optimize array fields of
 virtualizables
Message-ID: <57dc607f.ca0e1c0a.a952.5a48@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87174:6efe86960ad3
Date: 2016-09-16 22:49 +0200
http://bitbucket.org/pypy/pypy/changeset/6efe86960ad3/

Log:	don't optimize array fields of virtualizables

	(otherwise they can end up being passed around too much)

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -255,11 +255,27 @@
                 self._clear_heapcache_for(concretetype, fieldname)
                 self.heapcache[target, concretetype, fieldname] = op.args[2]
                 continue
+            if op.opname == "jit_force_virtualizable":
+                T = op.args[0].concretetype
+                FIELD = getattr(T.TO, op.args[1].value)
+                if hasattr(FIELD, 'TO') and isinstance(FIELD.TO, lltype.GcArray):
+                    # clear the cache for the virtualizable array fields, as
+                    # they run the risk of being passed around too much
+                    self._clear_heapcache_for_effects(
+                        {('struct', T, op.args[1].value)})
+
             if has_side_effects(op):
                 self._clear_heapcache_for_effects_of_op(op)
                 continue
 
             # foldable operations
+            if op.opname == "cast_pointer":
+                # cast_pointer is a pretty strange operation! it introduces
+                # more aliases, that confuse the CSE pass. Therefore we unify
+                # the two variables in new_unions, to improve the folding.
+                self.new_unions.union(op.args[0], op.result)
+                # don't do anything further
+                continue
             if not can_fold(op):
                 continue
             key = (op.opname, op.result.concretetype,
@@ -270,11 +286,6 @@
                 added_same_as += 1
             else:
                 self.purecache[key] = op.result
-            if op.opname == "cast_pointer":
-                # cast_pointer is a pretty strange operation! it introduces
-                # more aliases, that confuse the CSE pass. Therefore we unify
-                # the two variables in new_unions, to improve the folding.
-                self.new_unions.union(op.args[0], op.result)
         return added_same_as
 
 def _merge(tuples, variable_families, analyzer, loop_blocks, backedges):
diff --git a/rpython/translator/backendopt/stat.py b/rpython/translator/backendopt/stat.py
--- a/rpython/translator/backendopt/stat.py
+++ b/rpython/translator/backendopt/stat.py
@@ -1,65 +1,99 @@
 from rpython.translator.simplify import get_graph
 from hashlib import md5
+from collections import defaultdict
+
+def find_reachable_graphs(graph, translator, ignore_stack_checks=False):
+    seen_graphs = set()
+    stack = [graph]
+    while stack:
+        graph = stack.pop()
+        if graph in seen_graphs:
+            continue
+        seen_graphs.add(graph)
+        yield graph
+        for block, op in graph.iterblockops():
+            if op.opname == "direct_call":
+                called_graph = get_graph(op.args[0], translator)
+                if called_graph is not None and ignore_stack_checks:
+                    if called_graph.name.startswith('ll_stack_check'):
+                        continue
+                if called_graph is not None:
+                    stack.append(called_graph)
+            elif op.opname == "indirect_call":
+                called_graphs = op.args[-1].value
+                if called_graphs is not None:
+                    stack.extend(called_graphs)
+
 
 def get_statistics(graph, translator, save_per_graph_details=None, ignore_stack_checks=False):
-    seen_graphs = {}
-    stack = [graph]
     num_graphs = 0
     num_blocks = 0
     num_ops = 0
     num_mallocs = 0
+    num_memory = 0
     per_graph = {}
-    while stack:
-        graph = stack.pop()
-        if graph in seen_graphs:
-            continue
-        seen_graphs[graph] = True
+    for graph in find_reachable_graphs(graph, translator, ignore_stack_checks):
         num_graphs += 1
         old_num_blocks = num_blocks
         old_num_ops = num_ops
         old_num_mallocs = num_mallocs
+        old_num_memory = num_memory
         for block in graph.iterblocks():
             num_blocks += 1
             for op in block.operations:
-                if op.opname == "direct_call":
-                    called_graph = get_graph(op.args[0], translator)
-                    if called_graph is not None and ignore_stack_checks:
-                        if called_graph.name.startswith('ll_stack_check'):
-                            continue
-                    if called_graph is not None:
-                        stack.append(called_graph)
-                elif op.opname == "indirect_call":
-                    called_graphs = op.args[-1].value
-                    if called_graphs is not None:
-                        stack.extend(called_graphs)
-                elif op.opname.startswith("malloc"):
+                if op.opname.startswith("malloc"):
                     num_mallocs += 1
+                elif op.opname.startswith(("get", "set")):
+                    num_memory += 1
                 num_ops += 1
-        per_graph[graph] = (num_blocks-old_num_blocks, num_ops-old_num_ops, num_mallocs-old_num_mallocs)
+        per_graph[graph] = (num_blocks-old_num_blocks, num_ops-old_num_ops, num_mallocs-old_num_mallocs, num_memory-old_num_memory)
     if save_per_graph_details:
         details = []
-        for graph, (nblocks, nops, nmallocs) in per_graph.iteritems():
+        for graph, (nblocks, nops, nmallocs, nmemory) in per_graph.iteritems():
             try:
                 code = graph.func.func_code.co_code
             except AttributeError:
                 code = "None"
             hash = md5(code).hexdigest()
-            details.append((hash, graph.name, nblocks, nops, nmallocs))
+            details.append((hash, graph.name, nblocks, nops, nmallocs, nmemory))
         details.sort()
         f = open(save_per_graph_details, "w")
         try:
-            for hash, name, nblocks, nops, nmallocs in details:
-                print >>f, hash, name, nblocks, nops, nmallocs
+            for hash, name, nblocks, nops, nmallocs, nmemory in details:
+                print >>f, hash, name, nblocks, nops, nmallocs, nmemory
         finally:
             f.close()
-    return num_graphs, num_blocks, num_ops, num_mallocs
+    return num_graphs, num_blocks, num_ops, num_mallocs, num_memory
 
 def print_statistics(graph, translator, save_per_graph_details=None, ignore_stack_checks=False):
-    num_graphs, num_blocks, num_ops, num_mallocs = get_statistics(
+    num_graphs, num_blocks, num_ops, num_mallocs, num_memory = get_statistics(
             graph, translator, save_per_graph_details,
             ignore_stack_checks=ignore_stack_checks)
     print ("Statistics:\nnumber of graphs %s\n"
            "number of blocks %s\n"
            "number of operations %s\n"
            "number of mallocs %s\n"
-           ) % (num_graphs, num_blocks, num_ops, num_mallocs)
+           "number of memory operations %s\n"
+           ) % (num_graphs, num_blocks, num_ops, num_mallocs, num_memory)
+    calls = defaultdict(int)
+    opnames = defaultdict(int)
+    for graph in find_reachable_graphs(graph, translator):
+        for block, op in graph.iterblockops():
+            opnames[op.opname] += 1
+            if op.opname == "direct_call":
+                called_graph = get_graph(op.args[0], translator)
+                if called_graph is not None and ignore_stack_checks:
+                    if called_graph.name.startswith('ll_stack_check'):
+                        continue
+                if called_graph is not None:
+                    calls[called_graph] += 1
+            elif op.opname == "indirect_call":
+                called_graphs = op.args[-1].value
+                if called_graphs is not None:
+                    for called_graph in called_graphs:
+                        calls[called_graph] += 1
+    for num, name in sorted((num, name) for (name, num) in opnames.iteritems()):
+        print name, num
+    print
+    for num, graph in sorted((num, graph) for (graph, num) in calls.iteritems())[-100:]:
+        print graph.name, num
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -465,6 +465,28 @@
                 return cls.user_overridden_class
         self.check(f, [int], getfield=0)
 
+    def test_dont_fold_virtualizable(self):
+        class A(object):
+            _virtualizable_ = ["x[*]", "y"]
+
+        a1 = A()
+        a1.x = [1, 2, 3]
+        a1.y = 2
+        a2 = A()
+        a2.x = [65, 4, 3]
+        a2.y = 8
+        def f(i):
+            if i:
+                a = a1
+            else:
+                a = a2
+            res = a.x[0]
+            res += a.y
+            if i == 10:
+                res += a.x[1]
+                res += a.y
+            return res
+        self.check(f, [int], getfield=3)
 
 
 def fakevar(name='v'):

From pypy.commits at gmail.com  Sat Sep 17 03:28:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 00:28:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document branch
Message-ID: <57dcf0a7.8759c20a.1522a.313b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87175:cb6b4b7d4509
Date: 2016-09-17 09:28 +0200
http://bitbucket.org/pypy/pypy/changeset/cb6b4b7d4509/

Log:	Document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,8 @@
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.

From pypy.commits at gmail.com  Sat Sep 17 03:34:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 00:34:12 -0700 (PDT)
Subject: [pypy-commit] pypy sys-getsizeof: A branch to "implement"
 sys.getsizeof(), or at least give some number.
Message-ID: <57dcf1f4.a468c20a.69aee.2dff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sys-getsizeof
Changeset: r87176:29bf9d9c7bbf
Date: 2016-09-17 09:33 +0200
http://bitbucket.org/pypy/pypy/changeset/29bf9d9c7bbf/

Log:	A branch to "implement" sys.getsizeof(), or at least give some
	number.

	Add rgc.getsizeof().

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -583,6 +583,12 @@
                          resulttype=lltype.Void)
 
 
+def getsizeof(instance):
+    """Get the size of an object as seen by the GC.
+    Only includes this object, not anything else it depends on!
+    """
+    return get_rpy_memory_usage(cast_instance_to_gcref(instance))
+
 def get_rpy_memory_usage(gcref):
     "NOT_RPYTHON"
     # approximate implementation using CPython's type info
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -239,6 +239,7 @@
     x1 = X()
     n = rgc.get_rpy_memory_usage(rgc.cast_instance_to_gcref(x1))
     assert n >= 8 and n <= 64
+    assert n == rgc.getsizeof(x1)
 
 def test_register_custom_trace_hook():
     TP = lltype.GcStruct('X')

From pypy.commits at gmail.com  Sat Sep 17 04:49:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 01:49:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document sys.getsizeof()
Message-ID: <57dd0397.448e1c0a.50ced.f1b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87177:2b6a25a67d6b
Date: 2016-09-17 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/2b6a25a67d6b/

Log:	Document sys.getsizeof()

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/

From pypy.commits at gmail.com  Sat Sep 17 04:57:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 01:57:02 -0700 (PDT)
Subject: [pypy-commit] pypy default: Put the explanation in the docstring
 and in the error message
Message-ID: <57dd055e.4398c20a.ed3b3.508c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87178:a5599ad7bcdd
Date: 2016-09-17 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/a5599ad7bcdd/

Log:	Put the explanation in the docstring and in the error message

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -253,9 +253,28 @@
     from rpython.rtyper.lltypesystem import lltype, rffi
     return space.wrap(rffi.cast(lltype.Signed, handle))
 
+getsizeof_missing = """sys.getsizeof() is not implemented on PyPy.
+
+A memory profiler using this function is most likely to give results
+inconsistent with reality on PyPy.  It would be possible to have
+sys.getsizeof() return a number (with enough work), but that may or
+may not represent how much memory the object uses.  It doesn't even
+make really sense to ask how much *one* object uses, in isolation
+with the rest of the system.  For example, instances have maps,
+which are often shared across many instances; in this case the maps
+would probably be ignored by an implementation of sys.getsizeof(),
+but their overhead is important in some cases if they are many
+instances with unique maps.  Conversely, equal strings may share
+their internal string data even if they are different objects---or
+empty containers may share parts of their internals as long as they
+are empty.  Even stranger, some lists create objects as you read
+them; if you try to estimate the size in memory of range(10**6) as
+the sum of all items' size, that operation will by itself create one
+million integer objects that never existed in the first place.
+"""
+
 def getsizeof(space, w_object, w_default=None):
-    """Not implemented on PyPy."""
     if w_default is None:
-        raise oefmt(space.w_TypeError,
-                    "sys.getsizeof() not implemented on PyPy")
+        raise oefmt(space.w_TypeError, getsizeof_missing)
     return w_default
+getsizeof.__doc__ = getsizeof_missing

From pypy.commits at gmail.com  Sat Sep 17 06:23:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 03:23:34 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.8: hg merge default
Message-ID: <57dd19a6.e97bc20a.d4c0e.6676@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.8
Changeset: r2772:86bff8a11651
Date: 2016-09-17 11:31 +0200
http://bitbucket.org/cffi/cffi/changeset/86bff8a11651/

Log:	hg merge default

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.8.2"
+#define CFFI_VERSION  "1.8.3"
 
 #ifdef MS_WIN32
 #include <windows.h>
@@ -128,7 +128,7 @@
 #define CT_VOID             512    /* void */
 
 /* other flags that may also be set in addition to the base flag: */
-#define CT_CAST_ANYTHING         1024    /* 'char *' and 'void *' only */
+#define CT_IS_VOIDCHAR_PTR       1024
 #define CT_PRIMITIVE_FITS_LONG   2048
 #define CT_IS_OPAQUE             4096
 #define CT_IS_ENUM               8192
@@ -1358,9 +1358,26 @@
             }
         }
         if (ctinit != ct) {
-            if ((ct->ct_flags & CT_CAST_ANYTHING) ||
-                (ctinit->ct_flags & CT_CAST_ANYTHING))
-                ;   /* accept void* or char* as either source or target */
+            int combined_flags = ct->ct_flags | ctinit->ct_flags;
+            if (combined_flags & CT_IS_VOID_PTR)
+                ;   /* accept "void *" as either source or target */
+            else if (combined_flags & CT_IS_VOIDCHAR_PTR) {
+                /* for backward compatibility, accept "char *" as either
+                   source of target.  This is not what C does, though,
+                   so emit a warning that will eventually turn into an
+                   error. */
+                char *msg = (ct->ct_flags & CT_IS_VOIDCHAR_PTR ?
+                    "implicit cast to 'char *' from a different pointer type: "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" :
+                    "implicit cast from 'char *' to a different pointer type: "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)");
+                if (PyErr_WarnEx(PyExc_UserWarning, msg, 1))
+                    return -1;
+            }
             else {
                 expected = "pointer to same type";
                 goto cannot_convert;
@@ -2476,7 +2493,7 @@
     if (PyBytes_Check(init)) {
         /* from a string: just returning the string here is fine.
            We assume that the C code won't modify the 'char *' data. */
-        if ((ctptr->ct_flags & CT_CAST_ANYTHING) ||
+        if ((ctptr->ct_flags & CT_IS_VOIDCHAR_PTR) ||
             ((ctitem->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED))
              && (ctitem->ct_size == sizeof(char)))) {
 #if defined(CFFI_MEM_DEBUG) || defined(CFFI_MEM_LEAK)
@@ -3929,7 +3946,7 @@
     if ((ctitem->ct_flags & CT_VOID) ||
         ((ctitem->ct_flags & CT_PRIMITIVE_CHAR) &&
          ctitem->ct_size == sizeof(char)))
-        td->ct_flags |= CT_CAST_ANYTHING;   /* 'void *' or 'char *' only */
+        td->ct_flags |= CT_IS_VOIDCHAR_PTR;   /* 'void *' or 'char *' only */
     unique_key[0] = ctitem;
     return get_unique_type(td, unique_key, 1);
 }
@@ -5903,7 +5920,7 @@
         return NULL;
     }
     ct = ((CDataObject *)arg)->c_type;
-    if (!(ct->ct_flags & CT_CAST_ANYTHING)) {
+    if (!(ct->ct_flags & CT_IS_VOIDCHAR_PTR)) {
         PyErr_Format(PyExc_TypeError,
                      "expected a 'cdata' object with a 'void *' out of "
                      "new_handle(), got '%s'", ct->ct_name);
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3676,3 +3676,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.2"
-__version_info__ = (1, 8, 2)
+__version__ = "1.8.3"
+__version_info__ = (1, 8, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.2"
+                               "\ncompiled with cffi version: 1.8.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.8'
 # The full version, including alpha/beta/rc tags.
-release = '1.8.2'
+release = '1.8.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.2.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.3.tar.gz
 
-   - MD5: 538f307b6c5169bba41fbfda2b070762
+   - MD5: ...
 
-   - SHA: 9d2722ba9241b232b980bb9243e12451513a8000
+   - SHA: ...
 
-   - SHA256: 2b636db1a179439d73ae0a090479e179a43df5d4eddc7e4c4067f960d4038530
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -575,12 +575,12 @@
 +---------------+------------------------+------------------+----------------+
 |  pointers     | another <cdata> with   | a <cdata>        |``[]`` `(****)`,|
 |               | a compatible type (i.e.|                  |``+``, ``-``,   |
-|               | same type or ``char*`` |                  |bool()          |
+|               | same type              |                  |bool()          |
 |               | or ``void*``, or as an |                  |                |
 |               | array instead) `(*)`   |                  |                |
 +---------------+------------------------+                  |                |
-|  ``void *``,  | another <cdata> with   |                  |                |
-|  ``char *``   | any pointer or array   |                  |                |
+|  ``void *``   | another <cdata> with   |                  |                |
+|               | any pointer or array   |                  |                |
 |               | type                   |                  |                |
 +---------------+------------------------+                  +----------------+
 |  pointers to  | same as pointers       |                  | ``[]``, ``+``, |
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,19 @@
 ======================
 
 
+v1.8.3
+======
+
+* When passing a ``void *`` argument to a function with a different
+  pointer type, or vice-versa, the cast occurs automatically, like in C.
+  The same occurs for initialization with ``ffi.new()`` and a few other
+  places.  However, I thought that ``char *`` had the same
+  property---but I was mistaken.  In C you get the usual warning if you
+  try to give a ``char *`` to a ``char **`` argument, for example.
+  Sorry about the confusion.  This has been fixed in CFFI by giving for
+  now a warning, too.  It will turn into an error in a future version.
+
+
 v1.8.2
 ======
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.8.2',
+        version='1.8.3',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Sat Sep 17 06:23:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 03:23:36 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.8: md5/sha
Message-ID: <57dd19a8.43921c0a.ceddd.1789@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.8
Changeset: r2773:20e7bc5ffd1e
Date: 2016-09-17 12:22 +0200
http://bitbucket.org/cffi/cffi/changeset/20e7bc5ffd1e/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.8.3.tar.gz
 
-   - MD5: ...
+   - MD5: c8e877fe0426a99d0cf5872cf2f95b27
 
-   - SHA: ...
+   - SHA: 2a5be10205bf3f6b21070a71ee2546d838868172
 
-   - SHA256: ...
+   - SHA256: c321bd46faa7847261b89c0469569530cad5a41976bb6dba8202c0159f476568
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sat Sep 17 06:23:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 03:23:38 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge release-1.8
Message-ID: <57dd19aa.c549c20a.9e49f.9aa9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2774:a41b29125858
Date: 2016-09-17 12:22 +0200
http://bitbucket.org/cffi/cffi/changeset/a41b29125858/

Log:	hg merge release-1.8

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.8.3.tar.gz
 
-   - MD5: ...
+   - MD5: c8e877fe0426a99d0cf5872cf2f95b27
 
-   - SHA: ...
+   - SHA: 2a5be10205bf3f6b21070a71ee2546d838868172
 
-   - SHA256: ...
+   - SHA256: c321bd46faa7847261b89c0469569530cad5a41976bb6dba8202c0159f476568
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sat Sep 17 11:36:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 08:36:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Setting __name__ on
 generator-iterator objects (uh?)
Message-ID: <57dd62ec.657bc20a.4520.cf70@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87179:4411daaf0ffe
Date: 2016-09-17 17:35 +0200
http://bitbucket.org/pypy/pypy/changeset/4411daaf0ffe/

Log:	Setting __name__ on generator-iterator objects (uh?)

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -36,6 +36,7 @@
                           'closure?[*]',
                           'defs_w?[*]',
                           'name?',
+                          'qualname?',
                           'w_kw_defs?']
 
     w_kw_defs = None
@@ -117,7 +118,7 @@
                 for i in funccallunrolling:
                     if i < nargs:
                         new_frame.locals_cells_stack_w[i] = args_w[i]
-                return new_frame.run()
+                return new_frame.run(self.name, self.qualname)
         elif nargs >= 1 and fast_natural_arity == Code.PASSTHROUGHARGS1:
             assert isinstance(code, gateway.BuiltinCodePassThroughArguments1)
             return code.funcrun_obj(self, args_w[0],
@@ -184,7 +185,7 @@
             w_arg = frame.peekvalue(nargs-1-i)
             new_frame.locals_cells_stack_w[i] = w_arg
 
-        return new_frame.run()
+        return new_frame.run(self.name, self.qualname)
 
     @jit.unroll_safe
     def _flat_pycall_defaults(self, code, nargs, frame, defs_to_load):
@@ -201,7 +202,7 @@
         for j in xrange(start, ndefs):
             new_frame.locals_cells_stack_w[i] = self.defs_w[j]
             i += 1
-        return new_frame.run()
+        return new_frame.run(self.name, self.qualname)
 
     def getdict(self, space):
         if self.w_func_dict is None:
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -23,6 +23,7 @@
             self.register_finalizer(self.space)
 
     def get_name(self):
+        # 'name' is a byte string that is valid utf-8
         if self._name is not None:
             return self._name
         elif self.pycode is None:
@@ -31,9 +32,10 @@
             return self.pycode.co_name
 
     def get_qualname(self):
+        # 'qualname' is a unicode string
         if self._qualname is not None:
             return self._qualname
-        return self.get_name()
+        return self.get_name().decode('utf-8')
 
     def descr__repr__(self, space):
         addrstring = self.getaddrstring(space)
@@ -292,11 +294,27 @@
             return space.w_None
 
     def descr__name__(self, space):
-        return space.wrap(self.get_name())
+        return space.wrap(self.get_name().decode('utf-8'))
+
+    def descr_set__name__(self, space, w_name):
+        if space.isinstance_w(w_name, space.w_unicode):
+            self._name = space.str_w(w_name)
+        else:
+            raise oefmt(space.w_TypeError,
+                        "__name__ must be set to a string object")
 
     def descr__qualname__(self, space):
         return space.wrap(self.get_qualname())
 
+    def descr_set__qualname__(self, space, w_name):
+        try:
+            self._qualname = space.unicode_w(w_name)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                raise oefmt(space.w_TypeError,
+                            "__qualname__ must be set to a string object")
+            raise
+
     def _finalize_(self):
         # This is only called if the CO_YIELD_INSIDE_TRY flag is set
         # on the code object.  If the frame is still not finished and
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -265,7 +265,7 @@
         args.parse_into_scope(None, fresh_frame.locals_cells_stack_w, func.name,
                               sig, func.defs_w, func.w_kw_defs)
         fresh_frame.init_cells()
-        return frame.run()
+        return frame.run(func.name, func.qualname)
 
     def funcrun_obj(self, func, w_obj, args):
         frame = self.space.createframe(self, func.w_func_globals,
@@ -277,7 +277,7 @@
         args.parse_into_scope(w_obj, fresh_frame.locals_cells_stack_w, func.name,
                               sig, func.defs_w, func.w_kw_defs)
         fresh_frame.init_cells()
-        return frame.run()
+        return frame.run(func.name, func.qualname)
 
     def getvarnames(self):
         return self.co_varnames
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -244,22 +244,26 @@
         return (self.getcode().co_flags & (pycode.CO_COROUTINE |
                                            pycode.CO_GENERATOR)) != 0
 
-    def run(self):
+    def run(self, name=None, qualname=None):
         """Start this frame's execution."""
         if self._is_generator_or_coroutine():
-            if self.getcode().co_flags & pycode.CO_COROUTINE:
-                from pypy.interpreter.generator import Coroutine
-                gen = Coroutine(self)
-            else:
-                from pypy.interpreter.generator import GeneratorIterator
-                gen = GeneratorIterator(self)
-            if self.space.config.translation.rweakref:
-                self.f_generator_wref = rweakref.ref(gen)
-            else:
-                self.f_generator_nowref = gen
-            return self.space.wrap(gen)
+            return self.initialize_as_generator(name, qualname)
         else:
             return self.execute_frame()
+    run._always_inline_ = True
+
+    def initialize_as_generator(self, name, qualname):
+        if self.getcode().co_flags & pycode.CO_COROUTINE:
+            from pypy.interpreter.generator import Coroutine
+            gen = Coroutine(self, name, qualname)
+        else:
+            from pypy.interpreter.generator import GeneratorIterator
+            gen = GeneratorIterator(self, name, qualname)
+        if self.space.config.translation.rweakref:
+            self.f_generator_wref = rweakref.ref(gen)
+        else:
+            self.f_generator_nowref = gen
+        return self.space.wrap(gen)
 
     def execute_frame(self, in_generator=None, w_arg_or_err=None):
         """Execute this frame.  Main entry point to the interpreter.
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -32,6 +32,12 @@
         assert f.__qualname__ == 'qualname'
         raises(TypeError, "f.__qualname__ = b'name'")
 
+    def test_qualname_method(self):
+        class A:
+            def f(self):
+                pass
+        assert A.f.__qualname__ == 'test_qualname_method.<locals>.A.f'
+
     def test_annotations(self):
         def f(): pass
         ann = f.__annotations__
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -379,6 +379,22 @@
             assert False, 'Expected StopIteration'
             """
 
+    def test_set_name_qualname(self):
+        class A:
+            def f(self):
+                yield 5
+        g = A().f()
+        assert g.__name__ == "f"
+        assert g.__qualname__ == "test_set_name_qualname.<locals>.A.f"
+        g.__name__ = "h.i"
+        g.__qualname__ = "j.k"
+        assert g.__name__ == "h.i"
+        assert g.__qualname__ == "j.k"
+        raises(TypeError, "g.__name__ = 42")
+        raises(TypeError, "g.__qualname__ = 42")
+        raises((TypeError, AttributeError), "del g.__name__")
+        raises((TypeError, AttributeError), "del g.__qualname__")
+
 
 def test_should_not_inline(space):
     from pypy.interpreter.generator import should_not_inline
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -796,8 +796,10 @@
     gi_frame   = GetSetProperty(GeneratorIterator.descr_gicr_frame),
     gi_code    = interp_attrproperty_w('pycode', cls=GeneratorIterator),
     gi_yieldfrom=interp_attrproperty_w('w_yielded_from', cls=GeneratorIterator),
-    __name__   = GetSetProperty(GeneratorIterator.descr__name__),
-    __qualname__ = GetSetProperty(GeneratorIterator.descr__qualname__),
+    __name__   = GetSetProperty(GeneratorIterator.descr__name__,
+                                GeneratorIterator.descr_set__name__),
+    __qualname__ = GetSetProperty(GeneratorIterator.descr__qualname__,
+                                  GeneratorIterator.descr_set__qualname__),
     __weakref__ = make_weakref_descr(GeneratorIterator),
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
@@ -824,8 +826,10 @@
     cr_frame   = GetSetProperty(Coroutine.descr_gicr_frame),
     cr_code    = interp_attrproperty_w('pycode', cls=Coroutine),
     cr_await   = interp_attrproperty_w('w_yielded_from', cls=Coroutine),
-    __name__   = GetSetProperty(Coroutine.descr__name__),
-    __qualname__ = GetSetProperty(Coroutine.descr__qualname__),
+    __name__   = GetSetProperty(Coroutine.descr__name__,
+                                Coroutine.descr_set__name__),
+    __qualname__ = GetSetProperty(Coroutine.descr__qualname__,
+                                  Coroutine.descr_set__qualname__),
     __weakref__ = make_weakref_descr(Coroutine),
 )
 assert not Coroutine.typedef.acceptable_as_base_class  # no __new__

From pypy.commits at gmail.com  Sat Sep 17 11:41:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 08:41:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Test for gi_yieldfrom
Message-ID: <57dd6421.02d31c0a.d4777.80af@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87180:24ab5abc2c65
Date: 2016-09-17 17:40 +0200
http://bitbucket.org/pypy/pypy/changeset/24ab5abc2c65/

Log:	Test for gi_yieldfrom

diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -395,6 +395,20 @@
         raises((TypeError, AttributeError), "del g.__name__")
         raises((TypeError, AttributeError), "del g.__qualname__")
 
+    def test_gi_yieldfrom(self): """
+        def g(x):
+            yield x
+        def f(x):
+            yield from g(x)
+            yield 42
+        gen = f(5)
+        assert gen.gi_yieldfrom is None
+        assert next(gen) == 5
+        assert gen.gi_yieldfrom.__name__ == 'g'
+        assert next(gen) == 42
+        assert gen.gi_yieldfrom is None
+        """
+
 
 def test_should_not_inline(space):
     from pypy.interpreter.generator import should_not_inline

From pypy.commits at gmail.com  Sat Sep 17 12:33:51 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sat, 17 Sep 2016 09:33:51 -0700 (PDT)
Subject: [pypy-commit] pypy stdlib-2.7.11: merge default
Message-ID: <57dd706f.079bc20a.97325.b598@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: stdlib-2.7.11
Changeset: r87181:c46dac3ea8bf
Date: 2016-09-17 09:32 -0700
http://bitbucket.org/pypy/pypy/changeset/c46dac3ea8bf/

Log:	merge default

diff too long, truncating to 2000 out of 259284 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -22,3 +22,14 @@
 bbd45126bc691f669c4ebdfbd74456cd274c6b92 release-5.0.1
 3260adbeba4a8b6659d1cc0d0b41f266769b74da release-5.1
 b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1
+80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
+c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
+7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -43,17 +43,17 @@
   Samuele Pedroni
   Matti Picus
   Alex Gaynor
+  Philip Jenvey
   Brian Kearns
-  Philip Jenvey
+  Ronan Lamy
   Michael Hudson
-  Ronan Lamy
+  Manuel Jacob
   David Schneider
-  Manuel Jacob
   Holger Krekel
   Christian Tismer
   Hakan Ardo
+  Richard Plangger
   Benjamin Peterson
-  Richard Plangger
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -93,9 +94,9 @@
   stian
   Jan de Mooij
   Tyler Wade
+  Vincent Legoll
   Michael Foord
   Stephan Diehl
-  Vincent Legoll
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
@@ -104,14 +105,16 @@
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Mark Young
   Timo Paulssen
   Squeaky
+  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
+  Stefano Rivera
   Martin Matusiak
   Konstantin Lopuhin
-  Stefano Rivera
   Wenzhu Man
   John Witulski
   Laurence Tratt
@@ -122,13 +125,13 @@
   Simon Cross
   Edd Barrett
   Andreas St&#252;hrk
+  Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
   Spenser Bauman
   Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
-  Tobias Pape
   Paul deGrandis
   Ilya Osadchiy
   marky1991
@@ -138,9 +141,9 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
-  Mark Young
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
@@ -156,11 +159,13 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  William Leslie
   Ned Batchelder
   Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
+  Sergey Matyunin
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
@@ -171,9 +176,9 @@
   Yichao Yu
   Rocco Moretti
   Gintautas Miliauskas
-  Devin Jeanpierre
   Michael Twomey
   Lucian Branescu Mihaila
+  anatoly techtonik
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -183,8 +188,6 @@
   Brian Dorsey
   Victor Stinner
   Andrews Medina
-  anatoly techtonik
-  Sergey Matyunin
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -208,17 +211,19 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
-  William Leslie
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
+  Florin Papa
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -227,13 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
-  Florin Papa
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -270,8 +276,9 @@
   Yury V. Zaytsev
   Anna Katrina Dominguez
   Bobby Impollonia
-  timo at eistee.fritz.box
+  Vasantha Ganesh K
   Andrew Thompson
+  florinpapa
   Yusei Tahara
   Aaron Tubbs
   Ben Darnell
@@ -295,9 +302,9 @@
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  florinpapa
   Rafa&#322; Ga&#322;czy&#324;ski
   Matt Bogosian
+  timo
   Christian Muirhead
   Berker Peksag
   James Lan
@@ -353,12 +360,15 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
 
   Heinrich-Heine University, Germany 
   Open End AB (formerly AB Strakt), Sweden
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -85,10 +85,11 @@
     pass
 
 def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')):
+    import ast
     result = []
     for word in re_word.findall(line):
         if word.startswith('"'):
-            word = eval(word)
+            word = ast.literal_eval(word)
         result.append(word)
     return result
 
diff --git a/dotviewer/graphserver.py b/dotviewer/graphserver.py
--- a/dotviewer/graphserver.py
+++ b/dotviewer/graphserver.py
@@ -143,6 +143,11 @@
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
+        if len(sys.argv) == 1:
+            # start locally
+            import sshgraphserver
+            sshgraphserver.ssh_graph_server(['LOCAL'])
+            sys.exit(0)
         print >> sys.stderr, __doc__
         sys.exit(2)
     if sys.argv[1] == '--stdio':
diff --git a/dotviewer/sshgraphserver.py b/dotviewer/sshgraphserver.py
--- a/dotviewer/sshgraphserver.py
+++ b/dotviewer/sshgraphserver.py
@@ -4,11 +4,14 @@
 
 Usage:
     sshgraphserver.py  hostname  [more args for ssh...]
+    sshgraphserver.py  LOCAL
 
 This logs in to 'hostname' by passing the arguments on the command-line
 to ssh.  No further configuration is required: it works for all programs
 using the dotviewer library as long as they run on 'hostname' under the
 same username as the one sshgraphserver logs as.
+
+If 'hostname' is the string 'LOCAL', then it starts locally without ssh.
 """
 
 import graphserver, socket, subprocess, random
@@ -18,12 +21,19 @@
     s1 = socket.socket()
     s1.bind(('127.0.0.1', socket.INADDR_ANY))
     localhost, localport = s1.getsockname()
-    remoteport = random.randrange(10000, 20000)
-    #  ^^^ and just hope there is no conflict
 
-    args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (remoteport, localport)]
-    args = args + sshargs + ['python -u -c "exec input()"']
-    print ' '.join(args[:-1])
+    if sshargs[0] != 'LOCAL':
+        remoteport = random.randrange(10000, 20000)
+        #  ^^^ and just hope there is no conflict
+
+        args = ['ssh', '-S', 'none', '-C', '-R%d:127.0.0.1:%d' % (
+            remoteport, localport)]
+        args = args + sshargs + ['python -u -c "exec input()"']
+    else:
+        remoteport = localport
+        args = ['python', '-u', '-c', 'exec input()']
+
+    print ' '.join(args)
     p = subprocess.Popen(args, bufsize=0,
                          stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE)
diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -834,54 +834,63 @@
             c2pread, c2pwrite = None, None
             errread, errwrite = None, None
 
+            ispread = False
             if stdin is None:
                 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE)
                 if p2cread is None:
                     p2cread, _ = _subprocess.CreatePipe(None, 0)
+                    ispread = True
             elif stdin == PIPE:
                 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0)
+                ispread = True
             elif isinstance(stdin, int):
                 p2cread = msvcrt.get_osfhandle(stdin)
             else:
                 # Assuming file-like object
                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
-            p2cread = self._make_inheritable(p2cread)
+            p2cread = self._make_inheritable(p2cread, ispread)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(p2cread)
             if stdin == PIPE:
                 to_close.add(p2cwrite)
 
+            ispwrite = False
             if stdout is None:
                 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE)
                 if c2pwrite is None:
                     _, c2pwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stdout == PIPE:
                 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif isinstance(stdout, int):
                 c2pwrite = msvcrt.get_osfhandle(stdout)
             else:
                 # Assuming file-like object
                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-            c2pwrite = self._make_inheritable(c2pwrite)
+            c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(c2pwrite)
             if stdout == PIPE:
                 to_close.add(c2pread)
 
+            ispwrite = False
             if stderr is None:
                 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE)
                 if errwrite is None:
                     _, errwrite = _subprocess.CreatePipe(None, 0)
+                    ispwrite = True
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
+                ispwrite = True
             elif stderr == STDOUT:
-                errwrite = c2pwrite.handle # pass id to not close it
+                errwrite = c2pwrite
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
                 # Assuming file-like object
                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
-            errwrite = self._make_inheritable(errwrite)
+            errwrite = self._make_inheritable(errwrite, ispwrite)
             # We just duplicated the handle, it has to be closed at the end
             to_close.add(errwrite)
             if stderr == PIPE:
@@ -892,13 +901,14 @@
                     errread, errwrite), to_close
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
-            # If the initial handle was obtained with CreatePipe, close it.
-            if not isinstance(handle, int):
+            # PyPy: If the initial handle was obtained with CreatePipe,
+            # close it.
+            if close:
                 handle.Close()
             return dupl
 
diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py
--- a/lib-python/2.7/test/test_descr.py
+++ b/lib-python/2.7/test/test_descr.py
@@ -1735,7 +1735,6 @@
             ("__reversed__", reversed, empty_seq, set(), {}),
             ("__length_hint__", list, zero, set(),
              {"__iter__" : iden, "next" : stop}),
-            ("__sizeof__", sys.getsizeof, zero, set(), {}),
             ("__instancecheck__", do_isinstance, return_true, set(), {}),
             ("__missing__", do_dict_missing, some_number,
              set(("__class__",)), {}),
@@ -1747,6 +1746,8 @@
             ("__format__", format, format_impl, set(), {}),
             ("__dir__", dir, empty_seq, set(), {}),
             ]
+        if test_support.check_impl_detail():
+            specials.append(("__sizeof__", sys.getsizeof, zero, set(), {}))
 
         class Checker(object):
             def __getattr__(self, attr, test=self):
@@ -1768,10 +1769,6 @@
                 raise MyException
 
         for name, runner, meth_impl, ok, env in specials:
-            if name == '__length_hint__' or name == '__sizeof__':
-                if not test_support.check_impl_detail():
-                    continue
-
             class X(Checker):
                 pass
             for attr, obj in env.iteritems():
diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -174,7 +174,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py
--- a/lib-python/2.7/test/test_sys_settrace.py
+++ b/lib-python/2.7/test/test_sys_settrace.py
@@ -328,8 +328,8 @@
 
     def test_13_genexp(self):
         if self.using_gc:
+            gc.enable()
             test_support.gc_collect()
-            gc.enable()
         try:
             self.run_test(generator_example)
             # issue1265: if the trace function contains a generator,
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
@@ -199,10 +199,13 @@
     return tp._alignmentofinstances()
 
 @builtinify
-def byref(cdata):
+def byref(cdata, offset=0):
     # "pointer" is imported at the end of this module to avoid circular
     # imports
-    return pointer(cdata)
+    ptr = pointer(cdata)
+    if offset != 0:
+        ptr._buffer[0] += offset
+    return ptr
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -6,7 +6,7 @@
 irc_header = "And now for something completely different"
 
 
-def interactive_console(mainmodule=None, quiet=False):
+def interactive_console(mainmodule=None, quiet=False, future_flags=0):
     # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
     # mimics what CPython does in pythonrun.c
     if not hasattr(sys, 'ps1'):
@@ -37,15 +37,17 @@
             raise ImportError
         from pyrepl.simple_interact import run_multiline_interactive_console
     except ImportError:
-        run_simple_interactive_console(mainmodule)
+        run_simple_interactive_console(mainmodule, future_flags=future_flags)
     else:
-        run_multiline_interactive_console(mainmodule)
+        run_multiline_interactive_console(mainmodule, future_flags=future_flags)
 
-def run_simple_interactive_console(mainmodule):
+def run_simple_interactive_console(mainmodule, future_flags=0):
     import code
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
     # some parts of code.py are copied here because it seems to be impossible
     # to start an interactive console without printing at least one line
     # of banner
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -224,23 +224,9 @@
 va ClCl orvat bayl zbqrengryl zntvp vf n tbbq guvat <psobym>
 """
 
-from string import ascii_uppercase, ascii_lowercase
-
 def rot13(data):
-    """ A simple rot-13 encoder since `str.encode('rot13')` was removed from
-        Python as of version 3.0.  It rotates both uppercase and lowercase letters individually.
-    """
-    total = []
-    for char in data:
-        if char in ascii_uppercase:
-            index = (ascii_uppercase.find(char) + 13) % 26
-            total.append(ascii_uppercase[index])
-        elif char in ascii_lowercase:
-            index = (ascii_lowercase.find(char) + 13) % 26
-            total.append(ascii_lowercase[index])
-        else:
-            total.append(char)
-    return "".join(total)
+    return ''.join(chr(ord(c)+(13 if 'A'<=c.upper()<='M' else
+                              -13 if 'N'<=c.upper()<='Z' else 0)) for c in data)
 
 def some_topic():
     import time
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -0,0 +1,91 @@
+# Note: uses the CFFI out-of-line ABI mode.  We can't use the API
+# mode because ffi.compile() needs to run the compiler, which
+# needs 'subprocess', which needs 'msvcrt' and '_subprocess',
+# which depend on '_pypy_winbase_cffi' already.
+#
+# Note that if you need to regenerate _pypy_winbase_cffi and
+# can't use a preexisting PyPy to do that, then running this
+# file should work as long as 'subprocess' is not imported
+# by cffi.  I had to hack in 'cffi._pycparser' to move an
+#'import subprocess' to the inside of a function.  (Also,
+# CPython+CFFI should work as well.)
+#
+# This module supports both msvcrt.py and _subprocess.py.
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pypy_winbase_cffi", None)
+
+# ---------- MSVCRT ----------
+
+ffi.cdef("""
+typedef unsigned short wint_t;
+
+int _open_osfhandle(intptr_t osfhandle, int flags);
+intptr_t _get_osfhandle(int fd);
+int _setmode(int fd, int mode);
+int _locking(int fd, int mode, long nbytes);
+
+int _kbhit(void);
+int _getch(void);
+wint_t _getwch(void);
+int _getche(void);
+wint_t _getwche(void);
+int _putch(int);
+wint_t _putwch(wchar_t);
+int _ungetch(int);
+wint_t _ungetwch(wint_t);
+""")
+
+# ---------- SUBPROCESS ----------
+
+ffi.cdef("""
+typedef struct {
+    DWORD  cb;
+    char * lpReserved;
+    char * lpDesktop;
+    char * lpTitle;
+    DWORD  dwX;
+    DWORD  dwY;
+    DWORD  dwXSize;
+    DWORD  dwYSize;
+    DWORD  dwXCountChars;
+    DWORD  dwYCountChars;
+    DWORD  dwFillAttribute;
+    DWORD  dwFlags;
+    WORD   wShowWindow;
+    WORD   cbReserved2;
+    LPBYTE lpReserved2;
+    HANDLE hStdInput;
+    HANDLE hStdOutput;
+    HANDLE hStdError;
+} STARTUPINFO, *LPSTARTUPINFO;
+
+typedef struct {
+    HANDLE hProcess;
+    HANDLE hThread;
+    DWORD  dwProcessId;
+    DWORD  dwThreadId;
+} PROCESS_INFORMATION, *LPPROCESS_INFORMATION;
+
+DWORD WINAPI GetVersion(void);
+BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD);
+BOOL WINAPI CloseHandle(HANDLE);
+HANDLE WINAPI GetCurrentProcess(void);
+BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
+                            DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
+BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
+BOOL WINAPI TerminateProcess(HANDLE, UINT);
+HANDLE WINAPI GetStdHandle(DWORD);
+""")
+
+# --------------------
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -0,0 +1,10 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
+    _version = 0x2601,
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+)
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -4,151 +4,105 @@
 subprocess module on Windows.
 """
 
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The '_subprocess' module is only available on Windows")
 
 # Declare external Win32 functions
 
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
+from _pypy_winbase_cffi import ffi as _ffi
+_kernel32 = _ffi.dlopen('kernel32')
 
 GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
 
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
-
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
-
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
-
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
-
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessA
-_CreateProcess.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
 
 # Now the _subprocess module implementation
 
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
+def _WinError():
+    code, message = _ffi.getwinerror()
+    raise WindowsError(code, message)
 
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
+_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+
+class _handle(object):
+    def __init__(self, c_handle):
+        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
+        self.c_handle = c_handle
+        if int(self) != -1:
+            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
 
     def __int__(self):
-        return self.handle
+        return int(_ffi.cast("intptr_t", self.c_handle))
 
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
+    def __repr__(self):
+        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
 
     def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
+        h = int(self)
+        if h != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+        return h
 
     def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
+        if int(self) != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+            _kernel32.CloseHandle(c_handle)
 
 def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
+    handles = _ffi.new("HANDLE[2]")
 
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
+    res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size)
 
     if not res:
         raise _WinError()
 
-    return _handle(read.value), _handle(write.value)
+    return _handle(handles[0]), _handle(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
+    return _handle(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
+    # CPython: the first three arguments are expected to be integers
+    target = _ffi.new("HANDLE[1]")
 
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
+    res = _kernel32.DuplicateHandle(
+        _ffi.cast("HANDLE", source_process),
+        _ffi.cast("HANDLE", source),
+        _ffi.cast("HANDLE", target_process),
+        target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target.value)
+    return _handle(target[0])
+
+def _z(input):
+    if input is None:
+        return _ffi.NULL
+    if isinstance(input, basestring):
+        return str(input)
+    raise TypeError("string/unicode/None expected, got %r" % (
+        type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
                   inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
+    si = _ffi.new("STARTUPINFO *")
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
+        # CPython: these three handles are expected to be _handle objects
         if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
+            si.hStdInput = startup_info.hStdInput.c_handle
         if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
+            si.hStdOutput = startup_info.hStdOutput.c_handle
         if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
+            si.hStdError = startup_info.hStdError.c_handle
 
-    pi = _PROCESS_INFORMATION()
+    pi = _ffi.new("PROCESS_INFORMATION *")
 
     if env is not None:
         envbuf = ""
@@ -156,47 +110,55 @@
             envbuf += "%s=%s\0" % (k, v)
         envbuf += '\0'
     else:
-        envbuf = None
+        envbuf = _ffi.NULL
 
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
+    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+                                   _ffi.NULL, inherit, flags, envbuf,
+                                   _z(start_dir), si, pi)
 
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
 
 def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
+    # CPython: the first argument is expected to be an integer.
+    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
+                                        milliseconds)
     if res < 0:
         raise _WinError()
 
     return res
 
 def GetExitCodeProcess(handle):
-    code = _c_int()
+    # CPython: the first argument is expected to be an integer.
+    code = _ffi.new("DWORD[1]")
 
-    res = _GetExitCodeProcess(int(handle), _byref(code))
+    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
 
     if not res:
         raise _WinError()
 
-    return code.value
+    return code[0]
 
 def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
+    # CPython: the first argument is expected to be an integer.
+    # The second argument is silently wrapped in a UINT.
+    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+                                     _ffi.cast("UINT", exitcode))
 
     if not res:
         raise _WinError()
 
 def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
+    stdhandle = _ffi.cast("DWORD", stdhandle)
+    res = _kernel32.GetStdHandle(stdhandle)
 
     if not res:
         return None
     else:
-        return res
+        # note: returns integer, not handle object
+        return int(_ffi.cast("intptr_t", res))
 
 STD_INPUT_HANDLE = -10
 STD_OUTPUT_HANDLE = -11
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.6.0
+Version: 1.8.3
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.6.0"
-__version_info__ = (1, 6, 0)
+__version__ = "1.8.3"
+__version_info__ = (1, 8, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
@@ -42,7 +58,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -57,6 +75,12 @@
 # define _CFFI_UNUSED_FN  /* nothing */
 #endif
 
+#ifdef __cplusplus
+# ifndef _Bool
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
+# endif
+#endif
+
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
@@ -190,20 +214,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.6.0"
+                               "\ncompiled with cffi version: 1.8.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -10,7 +10,6 @@
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
 __version__ = '2.14'
 
-from subprocess import Popen, PIPE
 from .c_parser import CParser
 
 
@@ -28,6 +27,7 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
+    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -332,8 +332,8 @@
     def from_buffer(self, python_buffer):
         """Return a <cdata 'char[]'> that points to the data of the
         given Python object, which must support the buffer interface.
-        Note that this is not meant to be used on the built-in types str,
-        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        Note that this is not meant to be used on the built-in types
+        str or unicode (you can build 'char[]' arrays explicitly)
         but only on objects containing large quantities of raw data
         in some other format, like 'array.array' or numpy arrays.
         """
@@ -397,20 +397,7 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
-        try:
-            gcp = self._backend.gcp
-        except AttributeError:
-            pass
-        else:
-            return gcp(cdata, destructor)
-        #
-        with self._lock:
-            try:
-                gc_weakrefs = self.gc_weakrefs
-            except AttributeError:
-                from .gc_weakref import GcWeakrefs
-                gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
-            return gc_weakrefs.build(cdata, destructor)
+        return self._backend.gcp(cdata, destructor)
 
     def _get_cached_btype(self, type):
         assert self._lock.acquire(False) is False
@@ -665,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -682,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -205,9 +205,7 @@
 
     def __nonzero__(self):
         return bool(self._address)
-    
-    def __bool__(self):
-        return bool(self._address)
+    __bool__ = __nonzero__
 
     @classmethod
     def _to_ctypes(cls, value):
@@ -460,6 +458,12 @@
                         return x._value
                     raise TypeError("character expected, got %s" %
                                     type(x).__name__)
+                def __nonzero__(self):
+                    return ord(self._value) != 0
+            else:
+                def __nonzero__(self):
+                    return self._value != 0
+            __bool__ = __nonzero__
 
             if kind == 'float':
                 @staticmethod
@@ -993,6 +997,45 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
+    def gcp(self, cdata, destructor):
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
+
+        if destructor is None:
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
+                raise TypeError("Can remove destructor only on a object "
+                                "previously returned by ffi.gc()")
+            return None
+
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
+        return new_cdata
+
     typeof = type
 
     def getcname(self, BType, replace_with):
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -35,8 +35,11 @@
                                "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == cdecl:
-                raise api.FFIError("Unsupported type: %r.  Please file a bug "
-                                   "if you think it should be." % (commontype,))
+                raise api.FFIError(
+                    "Unsupported type: %r.  Please look at "
+        "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations "
+                    "and file an issue if you think this type should really "
+                    "be supported." % (commontype,))
             result, quals = parser.parse_type_and_quals(cdecl)   # recursive
 
         assert isinstance(result, model.BaseTypeByIdentity)
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -513,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -570,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -814,7 +814,7 @@
             try:
                 if ftype.is_integer_type() or fbitsize >= 0:
                     # accept all integers, but complain on float or double
-                    prnt("  (void)((p->%s) << 1);  /* check that '%s.%s' is "
+                    prnt("  (void)((p->%s) | 0);  /* check that '%s.%s' is "
                          "an integer */" % (fname, cname, fname))
                     continue
                 # only accept exactly the type declared, except that '[]'
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
@@ -991,7 +1000,7 @@
             prnt('static int %s(unsigned long long *o)' % funcname)
             prnt('{')
             prnt('  int n = (%s) <= 0;' % (name,))
-            prnt('  *o = (unsigned long long)((%s) << 0);'
+            prnt('  *o = (unsigned long long)((%s) | 0);'
                  '  /* check that %s is an integer */' % (name, name))
             if check_value is not None:
                 if check_value > 0:
@@ -1250,7 +1259,7 @@
 
     def _emit_bytecode_UnknownIntegerType(self, tp, index):
         s = ('_cffi_prim_int(sizeof(%s), (\n'
-             '           ((%s)-1) << 0 /* check that %s is an integer type */\n'
+             '           ((%s)-1) | 0 /* check that %s is an integer type */\n'
              '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
@@ -1422,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1458,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -839,7 +839,7 @@
             month = self._month
         if day is None:
             day = self._day
-        return date(year, month, day)
+        return date.__new__(type(self), year, month, day)
 
     # Comparisons of date objects with other.
 
@@ -1356,7 +1356,8 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return time(hour, minute, second, microsecond, tzinfo)
+        return time.__new__(type(self),
+                            hour, minute, second, microsecond, tzinfo)
 
     def __nonzero__(self):
         if self.second or self.microsecond:
@@ -1566,8 +1567,9 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return datetime(year, month, day, hour, minute, second, microsecond,
-                        tzinfo)
+        return datetime.__new__(type(self),
+                                year, month, day, hour, minute, second,
+                                microsecond, tzinfo)
 
     def astimezone(self, tz):
         if not isinstance(tz, tzinfo):
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -137,6 +137,8 @@
             lib.gdbm_sync(self.__ll_dbm)
 
 def open(filename, flags='r', mode=0666):
+    if isinstance(filename, unicode):
+        filename = filename.encode()
     if flags[0] == 'r':
         iflags = lib.GDBM_READER
     elif flags[0] == 'w':
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.9
+Version: 0.4.10
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.9"
+__version__ = "0.4.10"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -7,26 +7,39 @@
 # XXX incomplete: implemented only functions needed by subprocess.py
 # PAC: 2010/08 added MS locking for Whoosh
 
-import ctypes
+# 07/2016: rewrote in CFFI
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The 'msvcrt' module is only available on Windows")
+
+import _rawffi
+from _pypy_winbase_cffi import ffi as _ffi
+_lib = _ffi.dlopen(_rawffi.get_libc().name)
+
 import errno
-from ctypes_support import standard_c_lib as _c
-from ctypes_support import get_errno
-
-try:
-    open_osfhandle = _c._open_osfhandle
-except AttributeError: # we are not on windows
-    raise ImportError
 
 try: from __pypy__ import builtinify, validate_fd
 except ImportError: builtinify = validate_fd = lambda f: f
 
 
-open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int]
-open_osfhandle.restype = ctypes.c_int
+def _ioerr():
+    e = _ffi.errno
+    raise IOError(e, errno.errorcode[e])
 
-_get_osfhandle = _c._get_osfhandle
-_get_osfhandle.argtypes = [ctypes.c_int]
-_get_osfhandle.restype = ctypes.c_int
+
+ at builtinify
+def open_osfhandle(fd, flags):
+    """"open_osfhandle(handle, flags) -> file descriptor
+
+    Create a C runtime file descriptor from the file handle handle. The
+    flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,
+    and os.O_TEXT. The returned file descriptor may be used as a parameter
+    to os.fdopen() to create a file object."""
+    fd = _lib._open_osfhandle(fd, flags)
+    if fd == -1:
+        _ioerr()
+    return fd
 
 @builtinify
 def get_osfhandle(fd):
@@ -38,62 +51,74 @@
         validate_fd(fd)
     except OSError as e:
         raise IOError(*e.args)
-    return _get_osfhandle(fd)
+    result = _lib._get_osfhandle(fd)
+    if result == -1:
+        _ioerr()
+    return result
 
-setmode = _c._setmode
-setmode.argtypes = [ctypes.c_int, ctypes.c_int]
-setmode.restype = ctypes.c_int
+ at builtinify
+def setmode(fd, flags):
+    """setmode(fd, mode) -> Previous mode
+
+    Set the line-end translation mode for the file descriptor fd. To set
+    it to text mode, flags should be os.O_TEXT; for binary, it should be
+    os.O_BINARY."""
+    flags = _lib._setmode(fd, flags)
+    if flags == -1:
+        _ioerr()
+    return flags
 
 LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5)
 
-_locking = _c._locking
-_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_locking.restype = ctypes.c_int
-
 @builtinify
 def locking(fd, mode, nbytes):
-    '''lock or unlock a number of bytes in a file.'''
-    rv = _locking(fd, mode, nbytes)
+    """"locking(fd, mode, nbytes) -> None
+
+    Lock part of a file based on file descriptor fd from the C runtime.
+    Raises IOError on failure. The locked region of the file extends from
+    the current file position for nbytes bytes, and may continue beyond
+    the end of the file. mode must be one of the LK_* constants listed
+    below. Multiple regions in a file may be locked at the same time, but
+    may not overlap. Adjacent regions are not merged; they must be unlocked
+    individually."""
+    rv = _lib._locking(fd, mode, nbytes)
     if rv != 0:
-        e = get_errno()
-        raise IOError(e, errno.errorcode[e])
+        _ioerr()
 
 # Console I/O routines
 
-kbhit = _c._kbhit
-kbhit.argtypes = []
-kbhit.restype = ctypes.c_int
+kbhit = _lib._kbhit
 
-getch = _c._getch
-getch.argtypes = []
-getch.restype = ctypes.c_char
+ at builtinify
+def getch():
+    return chr(_lib._getch())
 
-getwch = _c._getwch
-getwch.argtypes = []
-getwch.restype = ctypes.c_wchar
+ at builtinify
+def getwch():
+    return unichr(_lib._getwch())
 
-getche = _c._getche
-getche.argtypes = []
-getche.restype = ctypes.c_char
+ at builtinify
+def getche():
+    return chr(_lib._getche())
 
-getwche = _c._getwche
-getwche.argtypes = []
-getwche.restype = ctypes.c_wchar
+ at builtinify
+def getwche():
+    return unichr(_lib._getwche())
 
-putch = _c._putch
-putch.argtypes = [ctypes.c_char]
-putch.restype = None
+ at builtinify
+def putch(ch):
+    _lib._putch(ord(ch))
 
-putwch = _c._putwch
-putwch.argtypes = [ctypes.c_wchar]
-putwch.restype = None
+ at builtinify
+def putwch(ch):
+    _lib._putwch(ord(ch))
 
-ungetch = _c._ungetch
-ungetch.argtypes = [ctypes.c_char]
-ungetch.restype = None
+ at builtinify
+def ungetch(ch):
+    if _lib._ungetch(ord(ch)) == -1:   # EOF
+        _ioerr()
 
-ungetwch = _c._ungetwch
-ungetwch.argtypes = [ctypes.c_wchar]
-ungetwch.restype = None
-
-del ctypes
+ at builtinify
+def ungetwch(ch):
+    if _lib._ungetwch(ord(ch)) == -1:   # EOF
+        _ioerr()
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -43,11 +43,13 @@
         return short
     return text
 
-def run_multiline_interactive_console(mainmodule=None):
+def run_multiline_interactive_console(mainmodule=None, future_flags=0):
     import code
     if mainmodule is None:
         import __main__ as mainmodule
     console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
+    if future_flags:
+        console.compile.compiler.flags |= future_flags
 
     def more_lines(unicodetext):
         # ooh, look at the hack:
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/__init__.py b/pypy/__init__.py
--- a/pypy/__init__.py
+++ b/pypy/__init__.py
@@ -1,4 +1,5 @@
-# Empty
+import os
+pypydir = os.path.realpath(os.path.dirname(__file__))
 
 # XXX Should be empty again, soon.
 # XXX hack for win64:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -36,7 +36,7 @@
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
-    "_csv", "cppyy", "_pypyjson",
+    "_csv", "cppyy", "_pypyjson", "_jitlog"
 ])
 
 from rpython.jit.backend import detect_cpu
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -1,4 +1,4 @@
-import py, pytest, sys, os, textwrap
+import py, pytest, sys, textwrap
 from inspect import isclass
 
 # pytest settings
@@ -10,8 +10,6 @@
 #
 option = None
 
-pypydir = os.path.realpath(os.path.dirname(__file__))
-
 def braindead_deindent(self):
     """monkeypatch that wont end up doing stupid in the python tokenizer"""
     text = '\n'.join(self.lines)
@@ -78,6 +76,20 @@
 def pytest_pycollect_makemodule(path, parent):
     return PyPyModule(path, parent)
 
+def is_applevel(item):
+    from pypy.tool.pytest.apptest import AppTestFunction
+    return isinstance(item, AppTestFunction)
+
+def pytest_collection_modifyitems(config, items):
+    if config.option.runappdirect:
+        return
+    for item in items:
+        if isinstance(item, py.test.Function):
+            if is_applevel(item):
+                item.add_marker('applevel')
+            else:
+                item.add_marker('interplevel')
+
 class PyPyModule(py.test.collect.Module):
     """ we take care of collecting classes both at app level
         and at interp-level (because we need to stick a space
@@ -112,9 +124,6 @@
             if name.startswith('AppTest'):
                 from pypy.tool.pytest.apptest import AppClassCollector
                 return AppClassCollector(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntClassCollector
-                return IntClassCollector(name, parent=self)
 
         elif hasattr(obj, 'func_code') and self.funcnamefilter(name):
             if name.startswith('app_test_'):
@@ -122,11 +131,7 @@
                     "generator app level functions? you must be joking"
                 from pypy.tool.pytest.apptest import AppTestFunction
                 return AppTestFunction(name, parent=self)
-            elif obj.func_code.co_flags & 32: # generator function
-                return pytest.Generator(name, parent=self)
-            else:
-                from pypy.tool.pytest.inttest import IntTestFunction
-                return IntTestFunction(name, parent=self)
+        return super(PyPyModule, self).makeitem(name, obj)
 
 def skip_on_missing_buildoption(**ropts):
     __tracebackhide__ = True
@@ -155,35 +160,19 @@
 
 def pytest_runtest_setup(__multicall__, item):
     if isinstance(item, py.test.collect.Function):
-        appclass = item.getparent(PyPyClassCollector)
+        appclass = item.getparent(py.test.Class)
         if appclass is not None:
             # Make cls.space and cls.runappdirect available in tests.
             spaceconfig = getattr(appclass.obj, 'spaceconfig', None)
             if spaceconfig is not None:
                 from pypy.tool.pytest.objspace import gettestobjspace
                 appclass.obj.space = gettestobjspace(**spaceconfig)
+            else:
+                appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
     __multicall__.execute()
 
-def pytest_runtest_teardown(__multicall__, item):
-    __multicall__.execute()
-
-    if 'pygame' in sys.modules:
-        assert option.view, ("should not invoke Pygame "
-                             "if conftest.option.view is False")
-
-
-class PyPyClassCollector(py.test.collect.Class):
-    # All pypy Test classes have a "space" member.
-    def setup(self):
-        cls = self.obj
-        if not hasattr(cls, 'spaceconfig'):
-            cls.space = LazyObjSpaceGetter()
-        else:
-            assert hasattr(cls, 'space') # set by pytest_runtest_setup
-        super(PyPyClassCollector, self).setup()
-
 
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -70,9 +70,6 @@
 bz2
     libbz2
 
-lzma (PyPy3 only)
-    liblzma
-
 pyexpat
     libexpat1
 
@@ -98,31 +95,33 @@
 tk
     tk-dev
 
+lzma (PyPy3 only)
+    liblzma
+
+To run untranslated tests, you need the Boehm garbage collector libgc.
+
 On Debian, this is the command to install all build-time dependencies::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev
-
-For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+    tk-dev libgc-dev \
+    liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel
-
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+    gdbm-devel \
+    xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses
+    libexpat-devel libffi-devel python-curses \
+    xz-devel  # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
-
 On Mac OS X, most of these build-time dependencies are installed alongside
 the Developer Tools. However, note that in order for the installation to
 find them you may need to run::
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/config/commandline.txt b/pypy/doc/config/commandline.txt
--- a/pypy/doc/config/commandline.txt
+++ b/pypy/doc/config/commandline.txt
@@ -9,7 +9,7 @@
 PyPy Python interpreter options
 -------------------------------
 
-The following options can be used after ``translate.py
+The following options can be used after ``rpython
 targetpypystandalone`` or as options to ``py.py``.
 
 .. GENERATE: objspace
@@ -22,7 +22,7 @@
 General translation options
 ---------------------------
 
-The following are options of ``translate.py``.  They must be
+The following are options of ``bin/rpython``.  They must be
 given before the ``targetxxx`` on the command line.
 
 * `--opt -O:`__ set the optimization level `[0, 1, size, mem, 2, 3]`
diff --git a/pypy/doc/config/index.rst b/pypy/doc/config/index.rst
--- a/pypy/doc/config/index.rst
+++ b/pypy/doc/config/index.rst
@@ -15,12 +15,12 @@
 
     ./py.py <`objspace options`_>
 
-and the ``translate.py`` translation entry
+and the ``rpython/bin/rpython`` translation entry
 point which takes arguments of this form:
 
 .. parsed-literal::
 
-    ./translate.py <`translation options`_> <target>
+    ./rpython/bin/rpython <`translation options`_> <target>

From pypy.commits at gmail.com  Sat Sep 17 12:37:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 09:37:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Refactor/cleanup/complete the
 opcodes
Message-ID: <57dd715b.c19d1c0a.8cbaa.92a7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87182:cebe85144e77
Date: 2016-09-17 18:37 +0200
http://bitbucket.org/pypy/pypy/changeset/cebe85144e77/

Log:	Refactor/cleanup/complete the opcodes

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -39,8 +39,10 @@
 
     def descr__repr__(self, space):
         addrstring = self.getaddrstring(space)
-        return space.wrap("<%s object %s at 0x%s>" %
-                          (self.KIND, self.get_qualname(), addrstring))
+        return space.wrap(u"<%s object %s at 0x%s>" %
+                          (unicode(self.KIND),
+                           self.get_qualname(),
+                           unicode(addrstring)))
 
     def descr__reduce__(self, space):
         from pypy.interpreter.mixedmodule import MixedModule
@@ -377,28 +379,6 @@
     unpack_into = _create_unpack_into()
     unpack_into_w = _create_unpack_into()
 
-    def _GetAwaitableIter(self, space):
-        #check if generator is a coroutine
-        if self.pycode.co_flags & consts.CO_ITERABLE_COROUTINE:
-            return self
-        w_await = space.lookup(self, "__await__")
-        if w_await is None:
-            raise oefmt(space.w_AttributeError,
-                        "object %T does not have __await__ method",
-                        self)
-        res = space.get_and_call_function(w_await, self)
-        if res is not None:
-            if (isinstance(res, Coroutine) or
-                (isinstance(res, GeneratorIterator) and \
-                 res.pycode.co_flags & consts.CO_ITERABLE_COROUTINE)):
-                raise oefmt(space.w_TypeError,
-                            "__await__() returned a coroutine")
-            elif space.lookup(self, "__next__") is None:
-                raise oefmt(space.w_TypeError,
-                        "__await__() returned non-iterator "
-                        "of type '%T'", res)
-        return res
-
 
 class Coroutine(GeneratorOrCoroutine):
     "A coroutine object."
@@ -421,9 +401,6 @@
                         self.get_qualname())
         GeneratorOrCoroutine._finalize_(self)
 
-    def _GetAwaitableIter(self, space):
-        return self
-
 
 @specialize.memo()
 def get_generator_exit(space):
@@ -466,6 +443,37 @@
     else:
         return space.call_method(w_yf, "send", w_inputvalue_or_err)
 
+def gen_is_coroutine(w_obj):
+    return (isinstance(w_obj, GeneratorIterator) and
+            (w_obj.pycode.co_flags & consts.CO_ITERABLE_COROUTINE) != 0)
+
+def get_awaitable_iter(space, w_obj):
+    # This helper function returns an awaitable for `o`:
+    #    - `o` if `o` is a coroutine-object;
+    #    - otherwise, o.__await__()
+
+    if isinstance(w_obj, Coroutine) or gen_is_coroutine(w_obj):
+        return w_obj
+
+    w_await = space.lookup(w_obj, "__await__")
+    if w_await is None:
+        raise oefmt(space.w_AttributeError,
+                    "object %T can't be used in 'await' expression"
+                    " (no __await__ method)", w_obj)
+    w_res = space.get_and_call_function(w_await, w_obj)
+    if isinstance(w_res, Coroutine) or gen_is_coroutine(w_res):
+        raise oefmt(space.w_TypeError,
+                    "__await__() returned a coroutine (it must return an "
+                    "iterator instead, see PEP 492)")
+    elif space.lookup(w_res, "__next__") is None:
+        raise oefmt(space.w_TypeError,
+                "__await__() returned non-iterator "
+                "of type '%T'", w_res)
+    return w_res
+
+
+# ----------
+
 
 def get_printable_location_genentry(bytecode):
     return '%s <generator>' % (bytecode.get_repr(),)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1441,19 +1441,20 @@
         elif not isinstance(w_iterable, GeneratorIterator):
             w_iterator = self.space.iter(w_iterable)
             self.settopvalue(w_iterator)
-    
+
     def GET_AWAITABLE(self, oparg, next_instr):
-        from pypy.objspace.std.noneobject import W_NoneObject
-        if isinstance(self.peekvalue(), W_NoneObject):
-            #switch NoneObject with iterable on stack (kind of a dirty fix)
-            w_firstnone = self.popvalue()
-            w_i = self.popvalue()
-            self.pushvalue(w_firstnone)
-            self.pushvalue(w_i)
-        w_iterable = self.peekvalue()
-        w_iter = w_iterable._GetAwaitableIter(self.space)
-        self.settopvalue(w_iter)
-    
+        from pypy.interpreter.generator import get_awaitable_iter
+        from pypy.interpreter.generator import Coroutine
+        w_iterable = self.popvalue()
+        w_iter = get_awaitable_iter(self.space, w_iterable)
+        if isinstance(w_iter, Coroutine):
+            if w_iter.w_yielded_from is not None:
+                # 'w_iter' is a coroutine object that is being awaited,
+                # '.w_yielded_from' is the current awaitable being awaited on.
+                raise oefmt(self.space.w_RuntimeError,
+                            "coroutine is being awaited already")
+        self.pushvalue(w_iter)
+
     def SETUP_ASYNC_WITH(self, offsettoend, next_instr):
         res = self.popvalue()
         block = WithBlock(self.valuestackdepth,
@@ -1474,39 +1475,68 @@
         self.settopvalue(w_exit)
         w_result = space.get_and_call_function(w_enter, w_manager)
         self.pushvalue(w_result)
-    
+
     def GET_AITER(self, oparg, next_instr):
+        from pypy.interpreter.generator import AIterWrapper, get_awaitable_iter
+
         space = self.space
-        w_obj = self.peekvalue()
+        w_obj = self.popvalue()
         w_func = space.lookup(w_obj, "__aiter__")
         if w_func is None:
-            raise oefmt(space.w_AttributeError,
-                        "object %T does not have __aiter__ method",
+            raise oefmt(space.w_TypeError,
+                        "'async for' requires an object with "
+                        "__aiter__ method, got %T",
                         w_obj)
         w_iter = space.get_and_call_function(w_func, w_obj)
-        w_awaitable = w_iter._GetAwaitableIter(space)
-        if w_awaitable is None:
-            raise oefmt(space.w_TypeError,
-                        "'async for' received an invalid object "
-                        "from __aiter__: %T", w_iter)
-        self.settopvalue(w_awaitable)
-    
+
+        # If __aiter__() returns an object with a __anext__() method,
+        # wrap it in a awaitable that resolves to 'w_iter'.
+        if space.lookup(w_iter, "__anext__") is not None:
+            w_awaitable = AIterWrapper(w_iter)
+        else:
+            try:
+                w_awaitable = get_awaitable_iter(space, w_iter)
+            except OperationError as e:
+                # yay! get_awaitable_iter() carefully builds a useful
+                # error message, but here we're eating *all errors*
+                # to replace it with a generic one.
+                if e.async(space):
+                    raise
+                raise oefmt(space.w_TypeError,
+                            "'async for' received an invalid object "
+                            "from __aiter__: %T", w_iter)
+            space.warn(space.wrap(
+                u"'%s' implements legacy __aiter__ protocol; "
+                u"__aiter__ should return an asynchronous "
+                u"iterator, not awaitable" %
+                    space.type(w_obj).name.decode('utf-8')))
+        self.pushvalue(w_awaitable)
+
     def GET_ANEXT(self, oparg, next_instr):
+        from pypy.interpreter.generator import get_awaitable_iter
+
         space = self.space
         w_aiter = self.peekvalue()
         w_func = space.lookup(w_aiter, "__anext__")
         if w_func is None:
-            raise oefmt(space.w_AttributeError,
-                        "object %T does not have __anext__ method",
+            raise oefmt(space.w_TypeError,
+                        "'async for' requires an iterator with "
+                        "__anext__ method, got %T",
                         w_aiter)
         w_next_iter = space.get_and_call_function(w_func, w_aiter)
-        w_awaitable = w_next_iter._GetAwaitableIter(space)
-        if w_awaitable is None:
+        try:
+            w_awaitable = get_awaitable_iter(space, w_next_iter)
+        except OperationError as e:
+            # yay! get_awaitable_iter() carefully builds a useful
+            # error message, but here we're eating *all errors*
+            # to replace it with a generic one.
+            if e.async(space):
+                raise
             raise oefmt(space.w_TypeError,
                         "'async for' received an invalid object "
                         "from __anext__: %T", w_next_iter)
         self.pushvalue(w_awaitable)
-        
+
 ### ____________________________________________________________ ###
 
 class Return(Exception):
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -280,7 +280,7 @@
             yield 1
         g = myFunc()
         r = repr(g)
-        assert r.startswith("<generator object myFunc at 0x")
+        assert r.startswith("<generator object test_repr.<locals>.myFunc at 0x")
         assert list(g) == [1]
         assert repr(g) == r
 

From pypy.commits at gmail.com  Sat Sep 17 13:04:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 10:04:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: A direct test for "async for"
 (I have still no idea how we're supposed
Message-ID: <57dd77ad.ce471c0a.6cd2f.ec3e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87183:a8123d99589e
Date: 2016-09-17 19:04 +0200
http://bitbucket.org/pypy/pypy/changeset/a8123d99589e/

Log:	A direct test for "async for" (I have still no idea how we're
	supposed to call a coroutine)

diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -8,3 +8,26 @@
         raises(TypeError, iter, f(5))
         raises(TypeError, next, f(5))
         """
+
+    def test_async_for(self): """
+        class X:
+            def __aiter__(self):
+                return MyAIter()
+        class MyAIter:
+            async def __anext__(self):
+                return 42
+        async def f(x):
+            sum = 0
+            async for a in x:
+                sum += a
+                if sum > 100:
+                    break
+            return sum
+        cr = f(X())
+        try:
+            next(cr.__await__())
+        except StopIteration as e:
+            assert e.value == 42 * 3
+        else:
+            assert False, "should have raised"
+        """

From pypy.commits at gmail.com  Sat Sep 17 13:10:47 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sat, 17 Sep 2016 10:10:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge py3k
Message-ID: <57dd7917.e840c20a.b19b2.e1d6@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r87184:d90b226fa2b8
Date: 2016-09-17 10:09 -0700
http://bitbucket.org/pypy/pypy/changeset/d90b226fa2b8/

Log:	merge py3k


From pypy.commits at gmail.com  Sat Sep 17 13:18:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 10:18:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: CoroutineWrapper
Message-ID: <57dd7af5.4abf1c0a.14cde.98b5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87185:53f068b021d8
Date: 2016-09-17 19:18 +0200
http://bitbucket.org/pypy/pypy/changeset/53f068b021d8/

Log:	CoroutineWrapper

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -234,6 +234,9 @@
     def descr_throw(self, w_type, w_val=None, w_tb=None):
         """throw(typ[,val[,tb]]) -> raise exception in generator/coroutine,
 return next yielded value or raise StopIteration."""
+        return self.throw(w_type, w_val, w_tb)
+
+    def throw(self, w_type, w_val, w_tb):
         from pypy.interpreter.pytraceback import check_traceback
 
         space = self.space
@@ -385,10 +388,7 @@
     KIND = "coroutine"
 
     def descr__await__(self, space):
-        # implement this function:
-        # https://github.com/python/cpython/blob/3.5/Objects/genobject.c#L786
-        # you need a new CoroutineWrapper object + CoroutineWrapperType
-        return self
+        return space.wrap(CoroutineWrapper(self))
 
     def _finalize_(self):
         # If coroutine was never awaited on issue a RuntimeWarning.
@@ -402,6 +402,31 @@
         GeneratorOrCoroutine._finalize_(self)
 
 
+class CoroutineWrapper(W_Root):
+    _immutable_ = True
+
+    def __init__(self, coroutine):
+        self.coroutine = coroutine
+
+    def descr__iter__(self, space):
+        return space.wrap(self)
+
+    def descr__next__(self, space):
+        return self.coroutine.send_ex(space.w_None)
+
+    def descr_send(self, space, w_arg):
+        return self.coroutine.send_ex(w_arg)
+    descr_send.__doc__ = Coroutine.descr_send.__doc__
+
+    def descr_throw(self, w_type, w_val=None, w_tb=None):
+        return self.coroutine.throw(w_type, w_val, w_tb)
+    descr_throw.__doc__ = Coroutine.descr_throw.__doc__
+
+    def descr_close(self):
+        return self.coroutine.descr_close()
+    descr_close.__doc__ = Coroutine.descr_close.__doc__
+
+
 @specialize.memo()
 def get_generator_exit(space):
     return OperationError(space.w_GeneratorExit,
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1478,6 +1478,7 @@
 
     def GET_AITER(self, oparg, next_instr):
         from pypy.interpreter.generator import AIterWrapper, get_awaitable_iter
+        xxxx
 
         space = self.space
         w_obj = self.popvalue()
@@ -1514,6 +1515,7 @@
 
     def GET_ANEXT(self, oparg, next_instr):
         from pypy.interpreter.generator import get_awaitable_iter
+        zzzz
 
         space = self.space
         w_aiter = self.peekvalue()
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -468,6 +468,7 @@
     ClassMethod, BuiltinFunction, descr_function_get)
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter.generator import GeneratorIterator, Coroutine
+from pypy.interpreter.generator import CoroutineWrapper
 from pypy.interpreter.nestedscope import Cell
 from pypy.interpreter.special import NotImplemented, Ellipsis
 
@@ -834,6 +835,15 @@
 )
 assert not Coroutine.typedef.acceptable_as_base_class  # no __new__
 
+CoroutineWrapper.typedef = TypeDef("coroutine_wrapper",
+    __iter__     = interp2app(CoroutineWrapper.descr__iter__),
+    __next__     = interp2app(CoroutineWrapper.descr__next__),
+    send         = interp2app(CoroutineWrapper.descr_send),
+    throw        = interp2app(CoroutineWrapper.descr_throw),
+    close        = interp2app(CoroutineWrapper.descr_close),
+)
+assert not CoroutineWrapper.typedef.acceptable_as_base_class  # no __new__
+
 Cell.typedef = TypeDef("cell",
     __total_ordering__ = 'auto',
     __lt__       = interp2app(Cell.descr__lt__),

From pypy.commits at gmail.com  Sat Sep 17 13:29:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 10:29:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: aiter_wrapper
Message-ID: <57dd7d7e.48cf1c0a.c5e18.7a9f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87186:d136ac649c2a
Date: 2016-09-17 19:29 +0200
http://bitbucket.org/pypy/pypy/changeset/d136ac649c2a/

Log:	aiter_wrapper

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -427,6 +427,23 @@
     descr_close.__doc__ = Coroutine.descr_close.__doc__
 
 
+class AIterWrapper(W_Root):
+    # NB. this type was added in CPython 3.5.2
+    _immutable_ = True
+
+    def __init__(self, w_aiter):
+        self.w_aiter = w_aiter
+
+    def descr__await__(self, space):
+        return space.wrap(self)
+
+    def descr__iter__(self, space):
+        return space.wrap(self)
+
+    def descr__next__(self, space):
+        raise OperationError(space.w_StopIteration, self.w_aiter)
+
+
 @specialize.memo()
 def get_generator_exit(space):
     return OperationError(space.w_GeneratorExit,
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1478,7 +1478,6 @@
 
     def GET_AITER(self, oparg, next_instr):
         from pypy.interpreter.generator import AIterWrapper, get_awaitable_iter
-        xxxx
 
         space = self.space
         w_obj = self.popvalue()
@@ -1515,7 +1514,6 @@
 
     def GET_ANEXT(self, oparg, next_instr):
         from pypy.interpreter.generator import get_awaitable_iter
-        zzzz
 
         space = self.space
         w_aiter = self.peekvalue()
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -468,7 +468,7 @@
     ClassMethod, BuiltinFunction, descr_function_get)
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter.generator import GeneratorIterator, Coroutine
-from pypy.interpreter.generator import CoroutineWrapper
+from pypy.interpreter.generator import CoroutineWrapper, AIterWrapper
 from pypy.interpreter.nestedscope import Cell
 from pypy.interpreter.special import NotImplemented, Ellipsis
 
@@ -805,12 +805,6 @@
 )
 assert not GeneratorIterator.typedef.acceptable_as_base_class  # no __new__
 
-#CoroutineWrapper.typedef = TypeDef("coroutine_wrapper",
-#    __anext__   = interp2app(CoroutineWrapper.descr_next,
-#                            descrmismatch='__anext__'),
-#)
-#assert not CoroutineWrapper.typedef.acceptable_as_base_class  # no __new__
-
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
     __reduce__   = interp2app(Coroutine.descr__reduce__),
@@ -844,6 +838,13 @@
 )
 assert not CoroutineWrapper.typedef.acceptable_as_base_class  # no __new__
 
+AIterWrapper.typedef = TypeDef("aiter_wrapper",
+    __await__    = interp2app(AIterWrapper.descr__await__),
+    __iter__     = interp2app(AIterWrapper.descr__iter__),
+    __next__     = interp2app(AIterWrapper.descr__next__),
+)
+assert not AIterWrapper.typedef.acceptable_as_base_class  # no __new__
+
 Cell.typedef = TypeDef("cell",
     __total_ordering__ = 'auto',
     __lt__       = interp2app(Cell.descr__lt__),

From pypy.commits at gmail.com  Sat Sep 17 13:45:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 10:45:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Fixes
Message-ID: <57dd8137.8759c20a.1522a.f4e6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87187:8f48a804defb
Date: 2016-09-17 19:44 +0200
http://bitbucket.org/pypy/pypy/changeset/8f48a804defb/

Log:	Fixes

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -395,10 +395,9 @@
         if self.pycode is not None and \
            self.frame is not None and \
            self.frame.last_instr == -1:
-            # XXX PyErr_Occured in condition?
-            raise oefmt(self.space.w_RuntimeWarning,
-                        "coroutine '%s' was never awaited",
-                        self.get_qualname())
+            space = self.space
+            msg = u"coroutine '%s' was never awaited" % self.get_qualname()
+            space.warn(space.w_RuntimeWarning, space.wrap(msg))
         GeneratorOrCoroutine._finalize_(self)
 
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1037,10 +1037,6 @@
         assert in_generator is not None
         w_inputvalue = self.popvalue()
         w_gen = self.popvalue()
-        if isinstance(w_gen, Coroutine) and not isinstance(self, Coroutine):
-            raise oefmt(self.space.w_TypeError,
-                        "cannot 'yield from' a coroutine object "
-                        "from a generator")
         #
         in_generator.w_yielded_from = w_gen
         in_generator.next_yield_from(self, w_inputvalue)
@@ -1425,7 +1421,7 @@
             self.popvalue()
             itemcount -= 1
         self.pushvalue(w_dict)
-    
+
     def GET_YIELD_FROM_ITER(self, oparg, next_instr):
         from pypy.interpreter.astcompiler import consts
         from pypy.interpreter.generator import GeneratorIterator, Coroutine

From pypy.commits at gmail.com  Sat Sep 17 13:52:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 10:52:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Test and fix
Message-ID: <57dd82e7.48cf1c0a.c5e18.8230@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87188:42c3a9990f34
Date: 2016-09-17 19:52 +0200
http://bitbucket.org/pypy/pypy/changeset/42c3a9990f34/

Log:	Test and fix

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1505,7 +1505,8 @@
                 u"'%s' implements legacy __aiter__ protocol; "
                 u"__aiter__ should return an asynchronous "
                 u"iterator, not awaitable" %
-                    space.type(w_obj).name.decode('utf-8')))
+                    space.type(w_obj).name.decode('utf-8')),
+                space.w_PendingDeprecationWarning)
         self.pushvalue(w_awaitable)
 
     def GET_ANEXT(self, oparg, next_instr):
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -31,3 +31,21 @@
         else:
             assert False, "should have raised"
         """
+
+    def test_async_for_old_style(self): """
+        class X:
+            def __aiter__(self):
+                return MyAIter()
+        class MyAIter:
+            def __await__(self):
+                return iter([20, 30])
+        async def f(x):
+            sum = 0
+            async for a in x:
+                sum += a
+                if sum > 100:
+                    break
+            return sum
+        cr = f(X())
+        assert next(cr.__await__()) == 20
+        """

From pypy.commits at gmail.com  Sat Sep 17 14:16:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 11:16:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: sys.set_coroutine_wrapper()
Message-ID: <57dd887a.c336c20a.19b0c.01e5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87189:72809bf56b82
Date: 2016-09-17 20:15 +0200
http://bitbucket.org/pypy/pypy/changeset/72809bf56b82/

Log:	sys.set_coroutine_wrapper()

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -22,7 +22,8 @@
     # XXX [fijal] but they're not. is_being_profiled is guarded a bit all
     #     over the place as well as w_tracefunc
 
-    _immutable_fields_ = ['profilefunc?', 'w_tracefunc?']
+    _immutable_fields_ = ['profilefunc?', 'w_tracefunc?',
+                          'w_coroutine_wrapper_fn?']
 
     def __init__(self, space):
         self.space = space
@@ -33,6 +34,8 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
+        self.w_coroutine_wrapper_fn = None
+        self.in_coroutine_wrapper = False
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -253,17 +253,36 @@
     run._always_inline_ = True
 
     def initialize_as_generator(self, name, qualname):
+        space = self.space
         if self.getcode().co_flags & pycode.CO_COROUTINE:
             from pypy.interpreter.generator import Coroutine
             gen = Coroutine(self, name, qualname)
+            ec = space.getexecutioncontext()
+            w_wrapper = ec.w_coroutine_wrapper_fn
         else:
             from pypy.interpreter.generator import GeneratorIterator
             gen = GeneratorIterator(self, name, qualname)
-        if self.space.config.translation.rweakref:
+            ec = None
+            w_wrapper = None
+
+        if space.config.translation.rweakref:
             self.f_generator_wref = rweakref.ref(gen)
         else:
             self.f_generator_nowref = gen
-        return self.space.wrap(gen)
+        w_gen = space.wrap(gen)
+
+        if w_wrapper is not None:
+            if ec.in_coroutine_wrapper:
+                raise oefmt(space.w_RuntimeError,
+                            "coroutine wrapper %R attempted "
+                            "to recursively wrap %R",
+                            w_wrapper, w_gen)
+            ec.in_coroutine_wrapper = True
+            try:
+                w_gen = space.call_function(w_wrapper, w_gen)
+            finally:
+                ec.in_coroutine_wrapper = False
+        return w_gen
 
     def execute_frame(self, in_generator=None, w_arg_or_err=None):
         """Execute this frame.  Main entry point to the interpreter.
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -49,3 +49,20 @@
         cr = f(X())
         assert next(cr.__await__()) == 20
         """
+
+    def test_set_coroutine_wrapper(self): """
+        import sys
+        async def f():
+            pass
+        seen = []
+        def my_wrapper(cr):
+            seen.append(cr)
+            return 42
+        assert sys.get_coroutine_wrapper() is None
+        sys.set_coroutine_wrapper(my_wrapper)
+        assert sys.get_coroutine_wrapper() is my_wrapper
+        cr = f()
+        assert cr == 42
+        sys.set_coroutine_wrapper(None)
+        assert sys.get_coroutine_wrapper() is None
+        """
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -91,6 +91,9 @@
         'float_repr_style'      : 'system.get_float_repr_style(space)',
         'getdlopenflags'        : 'system.getdlopenflags',
         'setdlopenflags'        : 'system.setdlopenflags',
+
+        'get_coroutine_wrapper' : 'vm.get_coroutine_wrapper',
+        'set_coroutine_wrapper' : 'vm.set_coroutine_wrapper',
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -258,3 +258,19 @@
         return space.new_interned_w_str(w_str)
     raise oefmt(space.w_TypeError, "intern() argument must be string.")
 
+def get_coroutine_wrapper(space):
+    "Return the wrapper for coroutine objects set by sys.set_coroutine_wrapper."
+    ec = space.getexecutioncontext()
+    if ec.w_coroutine_wrapper_fn is None:
+        return space.w_None
+    return ec.w_coroutine_wrapper_fn
+
+def set_coroutine_wrapper(space, w_wrapper):
+    "Set a wrapper for coroutine objects."
+    ec = space.getexecutioncontext()
+    if space.is_w(w_wrapper, space.w_None):
+        ec.w_coroutine_wrapper_fn = None
+    elif space.is_true(space.callable(w_wrapper)):
+        ec.w_coroutine_wrapper_fn = w_wrapper
+    else:
+        raise oefmt(space.w_TypeError, "callable expected, got %T", w_wrapper)

From pypy.commits at gmail.com  Sat Sep 17 14:24:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 11:24:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Complain more clearly
Message-ID: <57dd8a4e.e97bc20a.d4c0e.fd05@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87190:273f590b4ede
Date: 2016-09-17 20:23 +0200
http://bitbucket.org/pypy/pypy/changeset/273f590b4ede/

Log:	Complain more clearly

diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -115,10 +115,16 @@
         return self.space.newbool(valid)
 
     def descr__reduce__(self):
+        raise oefmt(space.w_NotImplementedError,
+                    "continulet's pickle support is currently disabled")
         from pypy.module._continuation import interp_pickle
         return interp_pickle.reduce(self)
 
     def descr__setstate__(self, w_args):
+        # XXX: review direct calls to frame.run(), notably when
+        # unpickling generators (or coroutines!)
+        raise oefmt(space.w_NotImplementedError,
+                    "continulet's pickle support is currently disabled")
         from pypy.module._continuation import interp_pickle
         interp_pickle.setstate(self, w_args)
 
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
--- a/pypy/module/_continuation/interp_pickle.py
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -34,8 +34,6 @@
     return space.newtuple(args)
 
 def setstate(self, w_args):
-    raise oefmt(space.w_NotImplementedError,
-                "continulet.__setstate__() needs to be fixed")
     space = self.space
     if self.sthread is not None:
         raise geterror(space, "continulet.__setstate__() on an already-"

From pypy.commits at gmail.com  Sat Sep 17 14:24:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 11:24:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Next failing test
Message-ID: <57dd8a50.cbc11c0a.f35c9.b98a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87191:d4946a59a6b9
Date: 2016-09-17 20:23 +0200
http://bitbucket.org/pypy/pypy/changeset/d4946a59a6b9/

Log:	Next failing test

diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -66,3 +66,23 @@
         sys.set_coroutine_wrapper(None)
         assert sys.get_coroutine_wrapper() is None
         """
+
+    def test_async_with(self): """
+        seen = []
+        class X:
+            async def __aenter__(self):
+                seen.append('aenter')
+            async def __aexit__(self, *args):
+                seen.append('aexit')
+        async def f(x):
+            async with x:
+                return 42
+        c = f(X())
+        try:
+            next(c.__await__())
+        except StopIteration as e:
+            assert e.value == 42
+        else:
+            assert False, "should have raised"
+        assert seen == ['aenter', 'aexit']
+        """

From pypy.commits at gmail.com  Sat Sep 17 14:41:04 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sat, 17 Sep 2016 11:41:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: avoid sys.version during bootstrapping as
 its setup lazily (3.6 uses
Message-ID: <57dd8e40.6222c20a.e8564.045d@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r87192:8adf0a90a67b
Date: 2016-09-17 11:40 -0700
http://bitbucket.org/pypy/pypy/changeset/8adf0a90a67b/

Log:	avoid sys.version during bootstrapping as its setup lazily (3.6 uses
	sys.version_info anyway: cpython issue #25985)

diff --git a/lib-python/3/importlib/_bootstrap_external.py b/lib-python/3/importlib/_bootstrap_external.py
--- a/lib-python/3/importlib/_bootstrap_external.py
+++ b/lib-python/3/importlib/_bootstrap_external.py
@@ -606,7 +606,7 @@
         else:
             registry_key = cls.REGISTRY_KEY
         key = registry_key.format(fullname=fullname,
-                                  sys_version=sys.version[:3])
+                                  sys_version='%d.%d' % sys.version_info[:2])
         try:
             with cls._open_registry(key) as hkey:
                 filepath = _winreg.QueryValue(hkey, '')

From pypy.commits at gmail.com  Sat Sep 17 15:59:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 12:59:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: one translation fix
Message-ID: <57dda09e.e17cc20a.65bd9.243e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87193:c4982d46ad7b
Date: 2016-09-17 21:58 +0200
http://bitbucket.org/pypy/pypy/changeset/c4982d46ad7b/

Log:	one translation fix

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -519,7 +519,7 @@
 def get_printable_location_genentry(bytecode):
     return '%s <generator>' % (bytecode.get_repr(),)
 generatorentry_driver = jit.JitDriver(greens=['pycode'],
-                                      reds=['gen', 'w_arg', 'operr'],
+                                      reds=['gen', 'w_arg'],
                                       get_printable_location =
                                           get_printable_location_genentry,
                                       name='generatorentry')

From pypy.commits at gmail.com  Sat Sep 17 15:59:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 12:59:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: fix the stack depth
 computation
Message-ID: <57dda0a0.a22fc20a.6ebfb.26bf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87194:ef22f59c45aa
Date: 2016-09-17 21:58 +0200
http://bitbucket.org/pypy/pypy/changeset/ef22f59c45aa/

Log:	fix the stack depth computation

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -664,8 +664,8 @@
     ops.DELETE_DEREF: 0,
     
     ops.GET_AWAITABLE: 0,
-    ops.SETUP_ASYNC_WITH: 2,
-    ops.BEFORE_ASYNC_WITH: -1,
+    ops.SETUP_ASYNC_WITH: 0,
+    ops.BEFORE_ASYNC_WITH: 1,
     ops.GET_AITER: 0,
     ops.GET_ANEXT: 1,
     ops.GET_YIELD_FROM_ITER: 0,

From pypy.commits at gmail.com  Sat Sep 17 15:59:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 12:59:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: reverse the order of these
 two values on the valuestack, as needed for
Message-ID: <57dda0a2.8f081c0a.bd04e.db50@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87195:1e2081d76357
Date: 2016-09-17 21:58 +0200
http://bitbucket.org/pypy/pypy/changeset/1e2081d76357/

Log:	reverse the order of these two values on the valuestack, as needed
	for "async with"

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1161,12 +1161,12 @@
                 self.space.w_None,
                 self.space.w_None,
                 self.space.w_None)
+        self.pushvalue(w_unroller)
         self.pushvalue(w_res)
-        self.pushvalue(w_unroller)
 
     def WITH_CLEANUP_FINISH(self, oparg, next_instr):
+        w_suppress = self.popvalue()
         w_unroller = self.popvalue()
-        w_suppress = self.popvalue()
         if isinstance(w_unroller, SApplicationException):
             if self.space.is_true(w_suppress):
                 # __exit__() returned True -> Swallow the exception.

From pypy.commits at gmail.com  Sat Sep 17 16:25:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 13:25:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Direct test for 'await'
Message-ID: <57dda6c7.c186c20a.ff68f.24dd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87196:3821014886f7
Date: 2016-09-17 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/3821014886f7/

Log:	Direct test for 'await'

diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -25,7 +25,7 @@
             return sum
         cr = f(X())
         try:
-            next(cr.__await__())
+            cr.send(None)
         except StopIteration as e:
             assert e.value == 42 * 3
         else:
@@ -79,10 +79,28 @@
                 return 42
         c = f(X())
         try:
-            next(c.__await__())
+            c.send(None)
         except StopIteration as e:
             assert e.value == 42
         else:
             assert False, "should have raised"
         assert seen == ['aenter', 'aexit']
         """
+
+    def test_await(self): """
+        class X:
+            def __await__(self):
+                i1 = yield 40
+                assert i1 == 82
+                i2 = yield 41
+                assert i2 == 93
+        async def f():
+            await X()
+            await X()
+        c = f()
+        assert c.send(None) == 40
+        assert c.send(82) == 41
+        assert c.send(93) == 40
+        assert c.send(82) == 41
+        raises(StopIteration, c.send, 93)
+        """

From pypy.commits at gmail.com  Sat Sep 17 17:05:14 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 17 Sep 2016 14:05:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: test, fix for returning a W_LongObject
Message-ID: <57ddb00a.aa52c20a.f1460.38d2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87197:f133ee180e4f
Date: 2016-09-17 23:24 +0300
http://bitbucket.org/pypy/pypy/changeset/f133ee180e4f/

Log:	test, fix for returning a W_LongObject

diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import widen
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -34,7 +34,7 @@
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
@@ -203,7 +203,7 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    return space.newlong(rffi.cast(ADDR, p))
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -8,18 +8,20 @@
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
-        value = api.PyLong_FromLong(3)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == 3
+        w_value = api.PyLong_FromLong(3)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == 3
 
-        value = api.PyLong_FromLong(sys.maxint)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == sys.maxint
+        w_value = api.PyLong_FromLong(sys.maxint)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == sys.maxint
 
     def test_aslong(self, space, api):
         w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+        assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsLong(w_value)
         assert value == (sys.maxint - 1)
 
@@ -35,12 +37,16 @@
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsSsize_t(w_value)
         assert value == 2
-        assert space.eq_w(w_value, api.PyLong_FromSsize_t(2))
+        w_val2 = api.PyLong_FromSsize_t(2)
+        assert isinstance(w_val2, W_LongObject)
+        assert space.eq_w(w_value, w_val2)
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
+        assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -103,6 +109,7 @@
 
     def test_as_voidptr(self, space, api):
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
+        assert isinstance(w_l, W_LongObject)
         assert space.unwrap(w_l) == 0L
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
 
@@ -128,23 +135,58 @@
         module = self.import_extension('foo', [
             ("from_unsignedlong", "METH_NOARGS",
              """
-                 return PyLong_FromUnsignedLong((unsigned long)-1);
+                 PyObject * obj;
+                 obj = PyLong_FromUnsignedLong((unsigned long)-1);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
         import sys
         assert module.from_unsignedlong() == 2 * sys.maxint + 1
 
     def test_fromlonglong(self):
         module = self.import_extension('foo', [
-            ("from_longlong", "METH_NOARGS",
+            ("from_longlong", "METH_VARARGS",
              """
-                 return PyLong_FromLongLong((long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """),
-            ("from_unsignedlonglong", "METH_NOARGS",
+            ("from_unsignedlonglong", "METH_VARARGS",
              """
-                 return PyLong_FromUnsignedLongLong((unsigned long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromUnsignedLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
-        assert module.from_longlong() == -1
-        assert module.from_unsignedlonglong() == (1<<64) - 1
+        assert module.from_longlong(-1) == -1
+        assert module.from_longlong(0) == 0
+        assert module.from_unsignedlonglong(0) == 0
+        assert module.from_unsignedlonglong(-1) == (1<<64) - 1
 
     def test_from_size_t(self):
         module = self.import_extension('foo', [
@@ -232,10 +274,15 @@
             ("has_sub", "METH_NOARGS",
              """
                 PyObject *ret, *obj = PyLong_FromLong(42);
-                if (obj->ob_type->tp_as_number->nb_subtract)
-                    ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                if (obj->ob_type != &PyLong_Type)
+                    ret = PyLong_FromLong(-2);
                 else
-                    ret = PyLong_FromLong(-1);
+                {
+                    if (obj->ob_type->tp_as_number->nb_subtract)
+                        ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                    else
+                        ret = PyLong_FromLong(-1);
+                }
                 Py_DECREF(obj);
                 return ret;
              """),

From pypy.commits at gmail.com  Sat Sep 17 17:05:16 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 17 Sep 2016 14:05:16 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: merge default into branch
Message-ID: <57ddb00c.c19d1c0a.8cbaa.e973@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87198:a3e17bbc7b47
Date: 2016-09-17 23:24 +0300
http://bitbucket.org/pypy/pypy/changeset/a3e17bbc7b47/

Log:	merge default into branch

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.2
+Version: 1.8.3
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.2"
-__version_info__ = (1, 8, 2)
+__version__ = "1.8.3"
+__version_info__ = (1, 8, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.2"
+                               "\ncompiled with cffi version: 1.8.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt
--- a/pypy/doc/config/translation.profopt.txt
+++ b/pypy/doc/config/translation.profopt.txt
@@ -3,3 +3,14 @@
 RPython program) to gather profile data. Example for pypy-c: "-c 'from
 richards import main;main(); from test import pystone;
 pystone.main()'"
+
+NOTE: be aware of what this does in JIT-enabled executables.  What it
+does is instrument and later optimize the C code that happens to run in
+the example you specify, ignoring any execution of the JIT-generated
+assembler.  That means that you have to choose the example wisely.  If
+it is something that will just generate assembler and stay there, there
+is little value.  If it is something that exercises heavily library
+routines that are anyway written in C, then it will optimize that.  Most
+interesting would be something that causes a lot of JIT-compilation,
+like running a medium-sized test suite several times in a row, in order
+to optimize the warm-up in general.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,8 @@
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -239,6 +239,10 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+            if config.translation.profopt is not None:
+                raise Exception("Cannot use the --profopt option "
+                                "when --shared is on (it is by default). "
+                                "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1986,6 +1986,7 @@
     'ZeroDivisionError',
     'RuntimeWarning',
     'PendingDeprecationWarning',
+    'UserWarning',
 ]
 
 if sys.platform.startswith("win"):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -63,7 +63,7 @@
         """x.__iter__() <==> iter(x)"""
         return self.space.wrap(self)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -264,25 +264,22 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
-            else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
-            #
             try:
+                if operr is not None:
+                    ec = self.space.getexecutioncontext()
+                    next_instr = self.handle_operation_error(ec, operr)
+                    self.last_instr = intmask(next_instr - 1)
+                else:
+                    # Execution starts just after the last_instr.  Initially,
+                    # last_instr is -1.  After a generator suspends it points to
+                    # the YIELD_VALUE instruction.
+                    next_instr = r_uint(self.last_instr + 1)
+                    if next_instr != 0:
+                        self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
-            executioncontext.return_trace(self, w_exitvalue)
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,12 +57,14 @@
         def f():
             yield 2
         g = f()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
 
     def test_throw3(self):
@@ -221,7 +223,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -562,3 +562,21 @@
         res = f(10).g()
         sys.settrace(None)
         assert res == 10
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.2"
+VERSION = "1.8.3"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer
 from pypy.module._cffi_backend import ctypeprim
 
 
@@ -22,6 +22,7 @@
     is_nonfunc_pointer_or_array = True
 
     def __init__(self, space, ctptr, length, arraysize, extra):
+        assert isinstance(ctptr, W_CTypePointer)
         W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
                                    ctptr.ctitem)
         self.length = length
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -35,8 +35,7 @@
         assert isinstance(ellipsis, bool)
         extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi)
         size = rffi.sizeof(rffi.VOIDP)
-        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult,
-                                could_cast_anything=False)
+        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult)
         self.fargs = fargs
         self.ellipsis = ellipsis
         self.abi = abi
@@ -59,6 +58,16 @@
                     lltype.free(self.cif_descr, flavor='raw')
                     self.cif_descr = lltype.nullptr(CIF_DESCRIPTION)
 
+    def is_unichar_ptr_or_array(self):
+        return False
+
+    def is_char_or_unichar_ptr_or_array(self):
+        return False
+
+    def string(self, cdataobj, maxlen):
+        # Can't use ffi.string() on a function pointer
+        return W_CType.string(self, cdataobj, maxlen)
+
     def new_ctypefunc_completing_argtypes(self, args_w):
         space = self.space
         nargs_declared = len(self.fargs)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -19,7 +19,6 @@
     # XXX this could be improved with an elidable method get_size()
     # that raises in case it's still -1...
 
-    cast_anything = False
     is_primitive_integer = False
     is_nonfunc_pointer_or_array = False
     is_indirect_arg_for_call_python = False
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -120,7 +120,6 @@
 
 class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar):
     _attrs_ = []
-    cast_anything = True
 
     def cast_to_int(self, cdata):
         return self.space.wrap(ord(cdata[0]))
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,12 +14,11 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _attrs_            = ['ctitem', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'accept_str', 'length']
     length = -1
 
-    def __init__(self, space, size, extra, extra_position, ctitem,
-                 could_cast_anything=True):
+    def __init__(self, space, size, extra, extra_position, ctitem):
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
@@ -27,10 +26,11 @@
         #  - for arrays, it is the array item type
         #  - for functions, it is the return type
         self.ctitem = ctitem
-        self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.accept_str = (self.can_cast_anything or
-                            (ctitem.is_primitive_integer and
-                             ctitem.size == rffi.sizeof(lltype.Char)))
+        self.accept_str = (self.is_nonfunc_pointer_or_array and
+                (isinstance(ctitem, ctypevoid.W_CTypeVoid) or
+                 isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or
+                 (ctitem.is_primitive_integer and
+                  ctitem.size == rffi.sizeof(lltype.Char))))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -137,7 +137,10 @@
 
 class W_CTypePtrBase(W_CTypePtrOrArray):
     # base class for both pointers and pointers-to-functions
-    _attrs_ = []
+    _attrs_ = ['is_void_ptr', 'is_voidchar_ptr']
+    _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr']
+    is_void_ptr = False
+    is_voidchar_ptr = False
 
     def convert_to_object(self, cdata):
         ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0]
@@ -154,7 +157,16 @@
             else:
                 raise self._convert_error("compatible pointer", w_ob)
         if self is not other:
-            if not (self.can_cast_anything or other.can_cast_anything):
+            if self.is_void_ptr or other.is_void_ptr:
+                pass     # cast from or to 'void *'
+            elif self.is_voidchar_ptr or other.is_voidchar_ptr:
+                space = self.space
+                msg = ("implicit cast from '%s' to '%s' "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" % (other.name, self.name))
+                space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1)
+            else:
                 raise self._convert_error("compatible pointer", w_ob)
 
         rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
@@ -165,8 +177,8 @@
 
 
 class W_CTypePointer(W_CTypePtrBase):
-    _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types']
-    _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr']
+    _attrs_ = ['is_file', 'cache_array_type', '_array_types']
+    _immutable_fields_ = ['is_file', 'cache_array_type?']
     kind = "pointer"
     cache_array_type = None
     is_nonfunc_pointer_or_array = True
@@ -181,6 +193,8 @@
         self.is_file = (ctitem.name == "struct _IO_FILE" or
                         ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
+        self.is_voidchar_ptr = (self.is_void_ptr or
+                           isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar))
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init, allocator):
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -7,7 +7,6 @@
 
 class W_CTypeVoid(W_CType):
     _attrs_ = []
-    cast_anything = True
     kind = "void"
 
     def __init__(self, space):
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -32,8 +32,8 @@
 @unwrap_spec(w_cdata=cdataobj.W_CData)
 def from_handle(space, w_cdata):
     ctype = w_cdata.ctype
-    if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or
-        not ctype.can_cast_anything):
+    if (not isinstance(ctype, ctypeptr.W_CTypePointer) or
+        not ctype.is_voidchar_ptr):
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3665,3 +3665,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -503,3 +503,10 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_bug_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        q = ffi.new("char[]", "abcd")
+        p = ffi.cast("char(*)(void)", q)
+        raises(TypeError, ffi.string, p)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,16 +30,19 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
+    w_initializer_type = None
+    w_initializer = None
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if space.type(w_initializer) is space.w_str:
+            if w_initializer is not None:
+                if w_initializer_type is space.w_str:
                     a.descr_fromstring(space, w_initializer)
-                elif space.type(w_initializer) is space.w_list:
+                elif w_initializer_type is space.w_list:
                     a.descr_fromlist(space, w_initializer)
                 else:
                     a.extend(w_initializer, True)
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import widen
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -34,7 +34,7 @@
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
@@ -203,7 +203,7 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    return space.newlong(rffi.cast(ADDR, p))
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -8,18 +8,20 @@
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
-        value = api.PyLong_FromLong(3)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == 3
+        w_value = api.PyLong_FromLong(3)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == 3
 
-        value = api.PyLong_FromLong(sys.maxint)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == sys.maxint
+        w_value = api.PyLong_FromLong(sys.maxint)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == sys.maxint
 
     def test_aslong(self, space, api):
         w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+        assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsLong(w_value)
         assert value == (sys.maxint - 1)
 
@@ -35,12 +37,16 @@
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsSsize_t(w_value)
         assert value == 2
-        assert space.eq_w(w_value, api.PyLong_FromSsize_t(2))
+        w_val2 = api.PyLong_FromSsize_t(2)
+        assert isinstance(w_val2, W_LongObject)
+        assert space.eq_w(w_value, w_val2)
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
+        assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -103,6 +109,7 @@
 
     def test_as_voidptr(self, space, api):
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
+        assert isinstance(w_l, W_LongObject)
         assert space.unwrap(w_l) == 0L
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
 
@@ -128,23 +135,58 @@
         module = self.import_extension('foo', [
             ("from_unsignedlong", "METH_NOARGS",
              """
-                 return PyLong_FromUnsignedLong((unsigned long)-1);
+                 PyObject * obj;
+                 obj = PyLong_FromUnsignedLong((unsigned long)-1);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
         import sys
         assert module.from_unsignedlong() == 2 * sys.maxint + 1
 
     def test_fromlonglong(self):
         module = self.import_extension('foo', [
-            ("from_longlong", "METH_NOARGS",
+            ("from_longlong", "METH_VARARGS",
              """
-                 return PyLong_FromLongLong((long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """),
-            ("from_unsignedlonglong", "METH_NOARGS",
+            ("from_unsignedlonglong", "METH_VARARGS",
              """
-                 return PyLong_FromUnsignedLongLong((unsigned long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromUnsignedLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
-        assert module.from_longlong() == -1
-        assert module.from_unsignedlonglong() == (1<<64) - 1
+        assert module.from_longlong(-1) == -1
+        assert module.from_longlong(0) == 0
+        assert module.from_unsignedlonglong(0) == 0
+        assert module.from_unsignedlonglong(-1) == (1<<64) - 1
 
     def test_from_size_t(self):
         module = self.import_extension('foo', [
@@ -232,10 +274,15 @@
             ("has_sub", "METH_NOARGS",
              """
                 PyObject *ret, *obj = PyLong_FromLong(42);
-                if (obj->ob_type->tp_as_number->nb_subtract)
-                    ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                if (obj->ob_type != &PyLong_Type)
+                    ret = PyLong_FromLong(-2);
                 else
-                    ret = PyLong_FromLong(-1);
+                {
+                    if (obj->ob_type->tp_as_number->nb_subtract)
+                        ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                    else
+                        ret = PyLong_FromLong(-1);
+                }
                 Py_DECREF(obj);
                 return ret;
              """),
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -253,9 +253,28 @@
     from rpython.rtyper.lltypesystem import lltype, rffi
     return space.wrap(rffi.cast(lltype.Signed, handle))
 
+getsizeof_missing = """sys.getsizeof() is not implemented on PyPy.
+
+A memory profiler using this function is most likely to give results
+inconsistent with reality on PyPy.  It would be possible to have
+sys.getsizeof() return a number (with enough work), but that may or
+may not represent how much memory the object uses.  It doesn't even
+make really sense to ask how much *one* object uses, in isolation
+with the rest of the system.  For example, instances have maps,
+which are often shared across many instances; in this case the maps
+would probably be ignored by an implementation of sys.getsizeof(),
+but their overhead is important in some cases if they are many
+instances with unique maps.  Conversely, equal strings may share
+their internal string data even if they are different objects---or
+empty containers may share parts of their internals as long as they
+are empty.  Even stranger, some lists create objects as you read
+them; if you try to estimate the size in memory of range(10**6) as
+the sum of all items' size, that operation will by itself create one
+million integer objects that never existed in the first place.
+"""
+
 def getsizeof(space, w_object, w_default=None):
-    """Not implemented on PyPy."""
     if w_default is None:
-        raise oefmt(space.w_TypeError,
-                    "sys.getsizeof() not implemented on PyPy")
+        raise oefmt(space.w_TypeError, getsizeof_missing)
     return w_default
+getsizeof.__doc__ = getsizeof_missing
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -200,8 +200,12 @@
             or v.concretetype != lltype.Bool):
             return False
         for op in block.operations[::-1]:
-            if v in op.args:
-                return False   # variable is also used in cur block
+            # check if variable is used in block
+            for arg in op.args:
+                if arg == v:
+                    return False
+                if isinstance(arg, ListOfKind) and v in arg.content:
+                    return False
             if v is op.result:
                 if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne',
                                      'int_gt', 'int_ge',
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -243,6 +243,20 @@
         assert block.exitswitch == (opname, v1, '-live-before')
         assert block.exits == exits
 
+def test_optimize_goto_if_not__argument_to_call():
+    for opname in ['ptr_iszero', 'ptr_nonzero']:
+        v1 = Variable()
+        v3 = Variable(); v3.concretetype = lltype.Bool
+        v4 = Variable()
+        block = Block([v1])
+        callop = SpaceOperation('residual_call_r_i',
+                ["fake", ListOfKind('int', [v3])], v4)
+        block.operations = [SpaceOperation(opname, [v1], v3), callop]
+        block.exitswitch = v3
+        block.exits = exits = [FakeLink(False), FakeLink(True)]
+        res = Transformer().optimize_goto_if_not(block)
+        assert not res
+
 def test_symmetric():
     ops = {'int_add': 'int_add',
            'int_or': 'int_or',
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -90,18 +90,23 @@
         assert vs.make_inputargs(args, optimizer) == []
 
     def test_make_inputargs_2(self):
-        # Ensure that make_inputargs properly errors with VirtualStatesCantMatch
-        # when the type information for a virtual field conflicts. In practice the
-        # expected and given field always share a common subclass.
-        # This check is needed as not all paths to make_inputargs in unroll.py
-        # are guarded by a call to generate_guards.
+        # Ensure that make_inputargs does not error when the lengths of the fields
+        # for the runtime box does not match what the virtual state expected.
+        # This can occur in unroll.py, as not all paths to make_inputargs are
+        # guareded with a generalization_of check. The property is validated
+        # subsequently in all cases, so we just need to ensure that this case does
+        # not cause segfaults.
         optimizer = FakeOptimizer(self.cpu)
         classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
-        innervalue1 = info.InstancePtrInfo(known_class=classbox1, is_virtual=True, descr=self.valuedescr.get_parent_descr())
+        innervalue1 = info.InstancePtrInfo(
+                known_class=classbox1, is_virtual=True,
+                descr=self.valuedescr.get_parent_descr())
         for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
             innervalue1.setfield(field, None, ConstInt(42))
         classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3))
-        innervalue2 = info.InstancePtrInfo(known_class=classbox2, is_virtual=True, descr=self.valuedescr3.get_parent_descr())
+        innervalue2 = info.InstancePtrInfo(
+                known_class=classbox2, is_virtual=True,
+                descr=self.valuedescr3.get_parent_descr())
         for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
             innervalue2.setfield(field, None, ConstInt(42))
 
@@ -111,10 +116,14 @@
         nodebox2.set_forwarded(innervalue2)
 
         constr = VirtualStateConstructor(optimizer)
-        vs = constr.get_virtual_state([nodebox1])
+        vs1 = constr.get_virtual_state([nodebox1])
+        constr = VirtualStateConstructor(optimizer)
+        vs2 = constr.get_virtual_state([nodebox2])
 
-        with py.test.raises(VirtualStatesCantMatch):
-            args = vs.make_inputargs([nodebox2], optimizer, force_boxes=True)
+        # This should succeed with no exceptions
+        vs1.make_inputargs([nodebox2], optimizer, force_boxes=False)
+        assert not vs1.generalization_of(vs2, optimizer)
+        assert not vs2.generalization_of(vs1, optimizer)
 
     def test_position_generalization(self):
         def postest(info1, info2):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -167,7 +167,8 @@
                [self.get_box_replacement(x) for x in end_jump.getarglist()],
                self.optimizer, force_boxes=True)
             for arg in args:
-                self.optimizer.force_box(arg)
+                if arg is not None:
+                    self.optimizer.force_box(arg)
         except VirtualStatesCantMatch:
             raise InvalidLoop("Virtual states did not match "
                               "after picking the virtual state, when forcing"
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -177,14 +177,6 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
-    @staticmethod
-    def descr_issubclass(descr1, descr2, optimizer):
-        if not descr1.is_object() or not descr2.is_object():
-            return True
-        vtable1 = descr1.get_vtable()
-        vtable2 = descr2.get_vtable()
-        return optimizer._check_subclass(vtable1, vtable2)
-
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
@@ -193,13 +185,12 @@
         else:
             assert isinstance(info, AbstractStructPtrInfo)
 
-        for i in range(len(self.fielddescrs)):
+        # The min operation ensures we don't wander off either array, as not all
+        # to make_inputargs have validated their inputs with generate_guards.
+        for i in range(min(len(self.fielddescrs), len(info._fields))):
             state = self.fieldstate[i]
-            descr = self.fielddescrs[i].get_parent_descr()
             if not state:
                 continue
-            if not self.descr_issubclass(info.descr, descr, optimizer.optimizer):
-                raise VirtualStatesCantMatch()
             if state.position > self.position:
                 fieldbox = info._fields[i]
                 state.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)
diff --git a/rpython/translator/backendopt/merge_if_blocks.py b/rpython/translator/backendopt/merge_if_blocks.py
--- a/rpython/translator/backendopt/merge_if_blocks.py
+++ b/rpython/translator/backendopt/merge_if_blocks.py
@@ -20,6 +20,14 @@
         return False
     if isinstance(op.args[0], Constant) and isinstance(op.args[1], Constant):
         return False
+    # check that the constant is hashable (ie not a symbolic)
+    try:
+        if isinstance(op.args[0], Constant):
+            hash(op.args[0].value)
+        else:
+            hash(op.args[1].value)
+    except TypeError:
+        return False
     return True
 
 def merge_chain(chain, checkvar, varmap, graph):
diff --git a/rpython/translator/backendopt/test/test_merge_if_blocks.py b/rpython/translator/backendopt/test/test_merge_if_blocks.py
--- a/rpython/translator/backendopt/test/test_merge_if_blocks.py
+++ b/rpython/translator/backendopt/test/test_merge_if_blocks.py
@@ -2,11 +2,12 @@
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator.backendopt.all import backend_optimizations
 from rpython.translator.translator import TranslationContext, graphof as tgraphof
-from rpython.flowspace.model import Block
+from rpython.flowspace.model import Block, checkgraph
 from rpython.translator.backendopt.removenoops import remove_same_as
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, r_int
 from rpython.annotator.model import SomeChar, SomeUnicodeCodePoint
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 
 def do_test_merge(fn, testvalues):
     t = TranslationContext()
@@ -225,3 +226,29 @@
     malloc.remove_mallocs(t, t.graphs)
     from rpython.translator import simplify
     simplify.join_blocks(graph)
+
+def test_switch_on_symbolic():
+    symb1 = CDefinedIntSymbolic("1", 1)
+    symb2 = CDefinedIntSymbolic("2", 2)
+    symb3 = CDefinedIntSymbolic("3", 3)
+    def fn(x):
+        res = 0
+        if x == symb1:
+            res += x + 1
+        elif x == symb2:
+            res += x + 2
+        elif x == symb3:
+            res += x + 3
+        res += 1
+        return res
+    t = TranslationContext()
+    a = t.buildannotator()
+    a.build_types(fn, [int])
+    rtyper = t.buildrtyper()
+    rtyper.specialize()
+    graph = t.graphs[0]
+    remove_same_as(graph)
+    res = merge_if_blocks_once(graph)
+    assert not res
+    checkgraph(graph)
+
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -275,6 +275,7 @@
         if retval and self.translator.platform.name == 'msvc':
             raise ValueError('Cannot do profile based optimization on MSVC,'
                     'it is not supported in free compiler version')
+        return retval
 
     def getentrypointptr(self):
         # XXX check that the entrypoint has the correct

From pypy.commits at gmail.com  Sat Sep 17 17:13:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 14:13:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: translation fixes
Message-ID: <57ddb1ee.861b1c0a.6d3a0.ee46@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87199:10685d0bf57d
Date: 2016-09-17 22:45 +0200
http://bitbucket.org/pypy/pypy/changeset/10685d0bf57d/

Log:	translation fixes

diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -10,7 +10,7 @@
 CO_GENERATOR = 0x0020
 CO_NOFREE = 0x0040
 CO_COROUTINE = 0x0080
-CO_ITERABLE_COROUTINE = 0x0100
+CO_ITERABLE_COROUTINE = 0x0100    # set by @types.coroutine
 CO_GENERATOR_ALLOWED = 0x1000
 CO_FUTURE_DIVISION = 0x2000
 CO_FUTURE_ABSOLUTE_IMPORT = 0x4000
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -5,6 +5,7 @@
 from pypy.interpreter.astcompiler import consts
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import r_uint
 
 
 class GeneratorOrCoroutine(W_Root):
@@ -164,7 +165,7 @@
             # We reach this point if the iterable is exhausted.
             last_instr = jit.promote(frame.last_instr)
             assert last_instr >= 0
-            return last_instr + 1
+            return r_uint(last_instr + 1)
 
         if isinstance(w_arg_or_err, SApplicationException):
             ec = space.getexecutioncontext()
@@ -178,7 +179,7 @@
                             self.KIND)
         else:
             frame.pushvalue(w_arg_or_err)
-        return last_instr + 1
+        return r_uint(last_instr + 1)
 
     def next_yield_from(self, frame, w_inputvalue_or_err):
         """Fetch the next item of the current 'yield from', push it on
@@ -205,7 +206,7 @@
             except OperationError as e:
                 if not e.match(space, space.w_AttributeError):
                     raise
-                w_value = space.w_None
+                w_stop_value = space.w_None
             frame.pushvalue(w_stop_value)
             return
         else:
@@ -227,7 +228,7 @@
             e2.record_context(space, self.frame)
             raise e2
         else:
-            space.warn(space.wrap("generator '%s' raised StopIteration"
+            space.warn(space.wrap(u"generator '%s' raised StopIteration"
                                   % self.get_qualname()),
                        space.w_PendingDeprecationWarning)
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -309,11 +309,10 @@
             try:
                 if in_generator is None:
                     assert self.last_instr == -1
-                    next_instr = 0
+                    next_instr = r_uint(0)
                 else:
                     next_instr = in_generator.resume_execute_frame(
                                                     self, w_arg_or_err)
-                next_instr = r_uint(next_instr)
                 #
                 self.dispatch(self.pycode, next_instr, executioncontext)
             except pyopcode.Return:
@@ -961,6 +960,7 @@
 
 def get_block_class(opname):
     # select the appropriate kind of block
+    from pypy.interpreter.pyopcode import block_classes
     return block_classes[opname]
 
 def unpickle_block(space, w_tup):
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -1,6 +1,7 @@
 from rpython.rlib.rstacklet import StackletThread
 from rpython.rlib import jit
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
+from pypy.interpreter.error import oefmt
 from pypy.interpreter.executioncontext import ExecutionContext
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
@@ -115,7 +116,7 @@
         return self.space.newbool(valid)
 
     def descr__reduce__(self):
-        raise oefmt(space.w_NotImplementedError,
+        raise oefmt(self.space.w_NotImplementedError,
                     "continulet's pickle support is currently disabled")
         from pypy.module._continuation import interp_pickle
         return interp_pickle.reduce(self)
@@ -123,7 +124,7 @@
     def descr__setstate__(self, w_args):
         # XXX: review direct calls to frame.run(), notably when
         # unpickling generators (or coroutines!)
-        raise oefmt(space.w_NotImplementedError,
+        raise oefmt(self.space.w_NotImplementedError,
                     "continulet's pickle support is currently disabled")
         from pypy.module._continuation import interp_pickle
         interp_pickle.setstate(self, w_args)
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
--- a/pypy/module/_continuation/interp_pickle.py
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -1,6 +1,6 @@
 from pypy.tool import stdlib_opcode as pythonopcode
 from rpython.rlib import jit
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.pyframe import PyFrame
 from pypy.module._continuation.interp_continuation import State, global_state
 from pypy.module._continuation.interp_continuation import build_sthread

From pypy.commits at gmail.com  Sat Sep 17 17:13:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 14:13:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Test and fix
Message-ID: <57ddb1f0.8aabc20a.1d096.3922@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87200:8abd37f94fd0
Date: 2016-09-17 23:07 +0200
http://bitbucket.org/pypy/pypy/changeset/8abd37f94fd0/

Log:	Test and fix

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -499,9 +499,9 @@
 
     w_await = space.lookup(w_obj, "__await__")
     if w_await is None:
-        raise oefmt(space.w_AttributeError,
-                    "object %T can't be used in 'await' expression"
-                    " (no __await__ method)", w_obj)
+        raise oefmt(space.w_TypeError,
+                    "object %T can't be used in 'await' expression",
+                    w_obj)
     w_res = space.get_and_call_function(w_await, w_obj)
     if isinstance(w_res, Coroutine) or gen_is_coroutine(w_res):
         raise oefmt(space.w_TypeError,
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -104,3 +104,15 @@
         assert c.send(82) == 41
         raises(StopIteration, c.send, 93)
         """
+
+    def test_await_error(self): """
+        async def f():
+            await [42]
+        c = f()
+        try:
+            c.send(None)
+        except TypeError as e:
+            assert str(e) == "object list can't be used in 'await' expression"
+        else:
+            assert False, "should have raised"
+        """

From pypy.commits at gmail.com  Sat Sep 17 17:15:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Sep 2016 14:15:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Comment,
 test about StopAsyncIteration
Message-ID: <57ddb27f.e840c20a.b19b2.3311@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87201:b16d98337497
Date: 2016-09-17 23:15 +0200
http://bitbucket.org/pypy/pypy/changeset/b16d98337497/

Log:	Comment, test about StopAsyncIteration

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -107,6 +107,9 @@
         frame = self.frame
         if frame is None:
             if isinstance(self, Coroutine):
+                # NB. CPython checks a flag 'closing' here, but instead
+                # we can simply not be here at all if frame is None in
+                # this case
                 raise oefmt(space.w_RuntimeError,
                             "cannot reuse already awaited coroutine")
             # xxx a bit ad-hoc, but we don't want to go inside
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -32,6 +32,31 @@
             assert False, "should have raised"
         """
 
+    def test_StopAsyncIteration(self): """
+        class X:
+            def __aiter__(self):
+                return MyAIter()
+        class MyAIter:
+            count = 0
+            async def __anext__(self):
+                if self.count == 3:
+                    raise StopAsyncIteration
+                self.count += 1
+                return 42
+        async def f(x):
+            sum = 0
+            async for a in x:
+                sum += a
+            return sum
+        cr = f(X())
+        try:
+            cr.send(None)
+        except StopIteration as e:
+            assert e.value == 42 * 3
+        else:
+            assert False, "should have raised"
+        """
+
     def test_async_for_old_style(self): """
         class X:
             def __aiter__(self):

From pypy.commits at gmail.com  Sun Sep 18 06:18:38 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 18 Sep 2016 03:18:38 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: merge default
Message-ID: <57de69fe.89d11c0a.2d57f.a8e3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87202:9bde782e5ac8
Date: 2016-09-18 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/9bde782e5ac8/

Log:	merge default

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,8 @@
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -63,7 +63,7 @@
         """x.__iter__() <==> iter(x)"""
         return self.space.wrap(self)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -264,25 +264,22 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
-            else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
-            #
             try:
+                if operr is not None:
+                    ec = self.space.getexecutioncontext()
+                    next_instr = self.handle_operation_error(ec, operr)
+                    self.last_instr = intmask(next_instr - 1)
+                else:
+                    # Execution starts just after the last_instr.  Initially,
+                    # last_instr is -1.  After a generator suspends it points to
+                    # the YIELD_VALUE instruction.
+                    next_instr = r_uint(self.last_instr + 1)
+                    if next_instr != 0:
+                        self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
-            executioncontext.return_trace(self, w_exitvalue)
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,12 +57,14 @@
         def f():
             yield 2
         g = f()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
 
     def test_throw3(self):
@@ -221,7 +223,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -562,3 +562,21 @@
         res = f(10).g()
         sys.settrace(None)
         assert res == 10
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,16 +30,19 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
+    w_initializer_type = None
+    w_initializer = None
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if space.type(w_initializer) is space.w_str:
+            if w_initializer is not None:
+                if w_initializer_type is space.w_str:
                     a.descr_fromstring(space, w_initializer)
-                elif space.type(w_initializer) is space.w_list:
+                elif w_initializer_type is space.w_list:
                     a.descr_fromlist(space, w_initializer)
                 else:
                     a.extend(w_initializer, True)
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import widen
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -34,7 +34,7 @@
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
@@ -203,7 +203,7 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    return space.newlong(rffi.cast(ADDR, p))
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -8,18 +8,20 @@
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
-        value = api.PyLong_FromLong(3)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == 3
+        w_value = api.PyLong_FromLong(3)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == 3
 
-        value = api.PyLong_FromLong(sys.maxint)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == sys.maxint
+        w_value = api.PyLong_FromLong(sys.maxint)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == sys.maxint
 
     def test_aslong(self, space, api):
         w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+        assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsLong(w_value)
         assert value == (sys.maxint - 1)
 
@@ -35,12 +37,16 @@
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsSsize_t(w_value)
         assert value == 2
-        assert space.eq_w(w_value, api.PyLong_FromSsize_t(2))
+        w_val2 = api.PyLong_FromSsize_t(2)
+        assert isinstance(w_val2, W_LongObject)
+        assert space.eq_w(w_value, w_val2)
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
+        assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -103,6 +109,7 @@
 
     def test_as_voidptr(self, space, api):
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
+        assert isinstance(w_l, W_LongObject)
         assert space.unwrap(w_l) == 0L
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
 
@@ -128,23 +135,58 @@
         module = self.import_extension('foo', [
             ("from_unsignedlong", "METH_NOARGS",
              """
-                 return PyLong_FromUnsignedLong((unsigned long)-1);
+                 PyObject * obj;
+                 obj = PyLong_FromUnsignedLong((unsigned long)-1);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
         import sys
         assert module.from_unsignedlong() == 2 * sys.maxint + 1
 
     def test_fromlonglong(self):
         module = self.import_extension('foo', [
-            ("from_longlong", "METH_NOARGS",
+            ("from_longlong", "METH_VARARGS",
              """
-                 return PyLong_FromLongLong((long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """),
-            ("from_unsignedlonglong", "METH_NOARGS",
+            ("from_unsignedlonglong", "METH_VARARGS",
              """
-                 return PyLong_FromUnsignedLongLong((unsigned long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromUnsignedLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
-        assert module.from_longlong() == -1
-        assert module.from_unsignedlonglong() == (1<<64) - 1
+        assert module.from_longlong(-1) == -1
+        assert module.from_longlong(0) == 0
+        assert module.from_unsignedlonglong(0) == 0
+        assert module.from_unsignedlonglong(-1) == (1<<64) - 1
 
     def test_from_size_t(self):
         module = self.import_extension('foo', [
@@ -232,10 +274,15 @@
             ("has_sub", "METH_NOARGS",
              """
                 PyObject *ret, *obj = PyLong_FromLong(42);
-                if (obj->ob_type->tp_as_number->nb_subtract)
-                    ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                if (obj->ob_type != &PyLong_Type)
+                    ret = PyLong_FromLong(-2);
                 else
-                    ret = PyLong_FromLong(-1);
+                {
+                    if (obj->ob_type->tp_as_number->nb_subtract)
+                        ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                    else
+                        ret = PyLong_FromLong(-1);
+                }
                 Py_DECREF(obj);
                 return ret;
              """),
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -253,9 +253,28 @@
     from rpython.rtyper.lltypesystem import lltype, rffi
     return space.wrap(rffi.cast(lltype.Signed, handle))
 
+getsizeof_missing = """sys.getsizeof() is not implemented on PyPy.
+
+A memory profiler using this function is most likely to give results
+inconsistent with reality on PyPy.  It would be possible to have
+sys.getsizeof() return a number (with enough work), but that may or
+may not represent how much memory the object uses.  It doesn't even
+make really sense to ask how much *one* object uses, in isolation
+with the rest of the system.  For example, instances have maps,
+which are often shared across many instances; in this case the maps
+would probably be ignored by an implementation of sys.getsizeof(),
+but their overhead is important in some cases if they are many
+instances with unique maps.  Conversely, equal strings may share
+their internal string data even if they are different objects---or
+empty containers may share parts of their internals as long as they
+are empty.  Even stranger, some lists create objects as you read
+them; if you try to estimate the size in memory of range(10**6) as
+the sum of all items' size, that operation will by itself create one
+million integer objects that never existed in the first place.
+"""
+
 def getsizeof(space, w_object, w_default=None):
-    """Not implemented on PyPy."""
     if w_default is None:
-        raise oefmt(space.w_TypeError,
-                    "sys.getsizeof() not implemented on PyPy")
+        raise oefmt(space.w_TypeError, getsizeof_missing)
     return w_default
+getsizeof.__doc__ = getsizeof_missing
diff --git a/rpython/translator/backendopt/merge_if_blocks.py b/rpython/translator/backendopt/merge_if_blocks.py
--- a/rpython/translator/backendopt/merge_if_blocks.py
+++ b/rpython/translator/backendopt/merge_if_blocks.py
@@ -20,6 +20,14 @@
         return False
     if isinstance(op.args[0], Constant) and isinstance(op.args[1], Constant):
         return False
+    # check that the constant is hashable (ie not a symbolic)
+    try:
+        if isinstance(op.args[0], Constant):
+            hash(op.args[0].value)
+        else:
+            hash(op.args[1].value)
+    except TypeError:
+        return False
     return True
 
 def merge_chain(chain, checkvar, varmap, graph):
diff --git a/rpython/translator/backendopt/test/test_merge_if_blocks.py b/rpython/translator/backendopt/test/test_merge_if_blocks.py
--- a/rpython/translator/backendopt/test/test_merge_if_blocks.py
+++ b/rpython/translator/backendopt/test/test_merge_if_blocks.py
@@ -2,11 +2,12 @@
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator.backendopt.all import backend_optimizations
 from rpython.translator.translator import TranslationContext, graphof as tgraphof
-from rpython.flowspace.model import Block
+from rpython.flowspace.model import Block, checkgraph
 from rpython.translator.backendopt.removenoops import remove_same_as
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, r_int
 from rpython.annotator.model import SomeChar, SomeUnicodeCodePoint
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 
 def do_test_merge(fn, testvalues):
     t = TranslationContext()
@@ -225,3 +226,29 @@
     malloc.remove_mallocs(t, t.graphs)
     from rpython.translator import simplify
     simplify.join_blocks(graph)
+
+def test_switch_on_symbolic():
+    symb1 = CDefinedIntSymbolic("1", 1)
+    symb2 = CDefinedIntSymbolic("2", 2)
+    symb3 = CDefinedIntSymbolic("3", 3)
+    def fn(x):
+        res = 0
+        if x == symb1:
+            res += x + 1
+        elif x == symb2:
+            res += x + 2
+        elif x == symb3:
+            res += x + 3
+        res += 1
+        return res
+    t = TranslationContext()
+    a = t.buildannotator()
+    a.build_types(fn, [int])
+    rtyper = t.buildrtyper()
+    rtyper.specialize()
+    graph = t.graphs[0]
+    remove_same_as(graph)
+    res = merge_if_blocks_once(graph)
+    assert not res
+    checkgraph(graph)
+

From pypy.commits at gmail.com  Sun Sep 18 12:37:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 09:37:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: fix interp_jit
Message-ID: <57dec2d4.854bc20a.da871.5e9b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87203:84a0a35f73d2
Date: 2016-09-18 18:37 +0200
http://bitbucket.org/pypy/pypy/changeset/84a0a35f73d2/

Log:	fix interp_jit

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -15,7 +15,7 @@
 from pypy.interpreter.pycode import CO_GENERATOR, PyCode
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
-from pypy.interpreter.pyopcode import ExitFrame, Yield
+from pypy.interpreter.pyopcode import Yield
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app
@@ -60,7 +60,7 @@
             name, intmask(next_instr), opname)
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
-    return (bytecode.co_flags & CO_GENERATOR) != 0
+    return (bytecode.co_flags & (CO_COROUTINE | CO_GENERATOR)) != 0
 
 class PyPyJitDriver(JitDriver):
     reds = ['frame', 'ec']
@@ -90,15 +90,9 @@
                 self.valuestackdepth = hint(self.valuestackdepth, promote=True)
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
                 is_being_profiled = self.get_is_being_profiled()
-        XXX XXX   # please fix, the base dispatch() changed
         except Yield:
-            # preserve self.last_exception between generator yields
-            w_result = self.popvalue()
             jit.hint(self, force_virtualizable=True)
-            return w_result
-        except ExitFrame:
-            self.last_exception = None
-            return self.popvalue()
+            raise
 
     def jump_absolute(self, jumpto, ec):
         if we_are_jitted():

From pypy.commits at gmail.com  Sun Sep 18 13:11:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 10:11:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix PyLong_FromVoidPtr to return
 int/long like CPython.
Message-ID: <57decad4.29a9c20a.149f9.650d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87204:8bbafbedd0b2
Date: 2016-09-18 19:11 +0200
http://bitbucket.org/pypy/pypy/changeset/8bbafbedd0b2/

Log:	Fix PyLong_FromVoidPtr to return int/long like CPython. Hopefully
	fix a translation failure on 32-bit too.

diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,6 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import widen
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -28,25 +27,25 @@
     """Return a new PyLongObject object from a C size_t, or NULL on
     failure.
     """
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.LONGLONG], PyObject)
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.newlong(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
     """Return a new PyLongObject object from a C unsigned long, or
     NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONGLONG], PyObject)
 def PyLong_FromUnsignedLongLong(space, val):
     """Return a new PyLongObject object from a C unsigned long long,
     or NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([PyObject], rffi.ULONG, error=-1)
 def PyLong_AsUnsignedLong(space, w_long):
@@ -203,7 +202,10 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.newlong(rffi.cast(ADDR, p))
+    value = rffi.cast(ADDR, p)    # signed integer
+    if value < 0:
+        return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
+    return space.wrap(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -1,5 +1,6 @@
 import sys, py
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import maxint
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
@@ -108,10 +109,26 @@
         lltype.free(overflow, flavor='raw')
 
     def test_as_voidptr(self, space, api):
+        # CPython returns an int (not a long) depending on the value
+        # passed to PyLong_FromVoidPtr().  In all cases, NULL becomes
+        # the int 0.
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
-        assert isinstance(w_l, W_LongObject)
-        assert space.unwrap(w_l) == 0L
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == 0
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
+        # Positive values also return an int (assuming, like always in
+        # PyPy, that an int is big enough to store any pointer).
+        p = rffi.cast(rffi.VOIDP, maxint)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == maxint
+        assert api.PyLong_AsVoidPtr(w_l) == p
+        # Negative values always return a long.
+        p = rffi.cast(rffi.VOIDP, -maxint-1)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_long)
+        assert space.unwrap(w_l) == maxint+1
+        assert api.PyLong_AsVoidPtr(w_l) == p
 
     def test_sign_and_bits(self, space, api):
         if space.is_true(space.lt(space.sys.get('version_info'),
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -270,6 +270,9 @@
             return W_SmallLongObject.fromint(val)
         return W_LongObject.fromint(self, val)
 
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+        return W_LongObject.fromrarith_int(val)
+
     def newlong_from_rbigint(self, val):
         return newlong(self, val)
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -25,7 +25,6 @@
         space.raises_w(space.w_OverflowError, space.float_w, w_big)
 
     def test_rint_variants(self):
-        py.test.skip("XXX broken!")
         from rpython.rtyper.tool.rfficache import platform
         space = self.space
         for r in platform.numbertype_to_rclass.values():
@@ -36,8 +35,8 @@
             for x in values:
                 if not r.SIGNED:
                     x &= r.MASK
-                w_obj = space.wrap(r(x))
-                assert space.bigint_w(w_obj).eq(rbigint.fromint(x))
+                w_obj = space.newlong_from_rarith_int(r(x))
+                assert space.bigint_w(w_obj).eq(rbigint.fromlong(x))
 
 
 class AppTestLong:

From pypy.commits at gmail.com  Sun Sep 18 13:54:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 10:54:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: translation fix
Message-ID: <57ded4e0.6286c20a.3c7e6.750b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87207:00e7a9a3460c
Date: 2016-09-18 19:54 +0200
http://bitbucket.org/pypy/pypy/changeset/00e7a9a3460c/

Log:	translation fix

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -270,6 +270,7 @@
             return W_SmallLongObject.fromint(val)
         return W_LongObject.fromint(self, val)
 
+    @specialize.argtype(1)
     def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
         return W_LongObject.fromrarith_int(val)
 

From pypy.commits at gmail.com  Sun Sep 18 15:05:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 12:05:56 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: updates
Message-ID: <57dee594.0205c20a.b2543.9478@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5705:c602273d9953
Date: 2016-09-18 21:05 +0200
http://bitbucket.org/pypy/extradoc/changeset/c602273d9953/

Log:	updates

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -23,6 +23,15 @@
   threads concurrently.  It will make two stat objects and leak one of
   them.
 
+* _PyGen_yf() checks the opcode at [f_lasti + 1], which is the next
+  opcode that will run when we resume the generator: either YIELD or
+  YIELD_FROM.  But that is only true if the generator is not currently
+  running.  If it is (which probably doesn't occur in reasonable Python
+  code but can be constructed manually), then this checks for example
+  the byte/word that describes the argument of the currently running
+  opcode.  If we're very unlucky this byte has the value 72, which is
+  YIELD_FROM.  Total nonsense and crashes follow.
+
 
 Other bugs
 ----------
@@ -57,3 +66,7 @@
   invoked.  This can confuse coverage tools and profilers.  For example,
   in a stack ``f1()->f2()->f3()``, vmprof would show f3() as usually
   called via f2() from f1() but occasionally called directly from f1().
+
+* ceval.c: GET_AITER: calls _PyCoro_GetAwaitableIter(), which might
+  get an exception from calling the user-defined __await__() or checking
+  what it returns; such an exception is completely eaten.

From pypy.commits at gmail.com  Sun Sep 18 15:14:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 12:14:23 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: update
Message-ID: <57dee78f.c398c20a.f5f21.a2f6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5706:e4f3d504a0c6
Date: 2016-09-18 21:14 +0200
http://bitbucket.org/pypy/extradoc/changeset/e4f3d504a0c6/

Log:	update

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -24,13 +24,17 @@
   them.
 
 * _PyGen_yf() checks the opcode at [f_lasti + 1], which is the next
-  opcode that will run when we resume the generator: either YIELD or
-  YIELD_FROM.  But that is only true if the generator is not currently
-  running.  If it is (which probably doesn't occur in reasonable Python
-  code but can be constructed manually), then this checks for example
-  the byte/word that describes the argument of the currently running
-  opcode.  If we're very unlucky this byte has the value 72, which is
-  YIELD_FROM.  Total nonsense and crashes follow.
+  opcode that will run when we resume the generator: either it is the
+  opcode following the YIELD, or it is exactly YIELD_FROM.  It is not
+  possible at the moment to write Python code that compiles to a YIELD
+  immediately followed by YIELD_FROM, so by chance the two cases are
+  correctly distinguished.  *However,* the discussion so far assumes
+  that the generator is not currently running.  If it is (which probably
+  doesn't occur in reasonable Python code but can be constructed
+  manually), then this checks for example the byte/word that describes
+  the argument of the currently running opcode.  If we're very unlucky
+  this byte has the value 72, which is YIELD_FROM.  Total nonsense and
+  crashes follow.
 
 
 Other bugs

From pypy.commits at gmail.com  Sun Sep 18 15:17:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 12:17:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: comment
Message-ID: <57dee850.c549c20a.9e49f.c4e9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87208:1f643102d010
Date: 2016-09-18 21:17 +0200
http://bitbucket.org/pypy/pypy/changeset/1f643102d010/

Log:	comment

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1035,7 +1035,7 @@
         from pypy.interpreter.generator import Coroutine
         in_generator = self.get_generator()
         assert in_generator is not None
-        w_inputvalue = self.popvalue()
+        w_inputvalue = self.popvalue()    # that's always None, actually
         w_gen = self.popvalue()
         #
         in_generator.w_yielded_from = w_gen

From pypy.commits at gmail.com  Sun Sep 18 15:57:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 12:57:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Test and fix: must reset
 w_yielded_from to None when running
Message-ID: <57def193.85b81c0a.fd2ad.5758@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87209:53965720c0f3
Date: 2016-09-18 21:56 +0200
http://bitbucket.org/pypy/pypy/changeset/53965720c0f3/

Log:	Test and fix: must reset w_yielded_from to None when running

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -158,9 +158,11 @@
         # Called from execute_frame() just before resuming the bytecode
         # interpretation.
         space = self.space
-        if self.w_yielded_from is not None:
+        w_yf = self.w_yielded_from
+        if w_yf is not None:
+            self.w_yielded_from = None
             try:
-                self.next_yield_from(frame, w_arg_or_err)
+                self.next_yield_from(frame, w_yf, w_arg_or_err)
             except OperationError as operr:
                 ec = space.getexecutioncontext()
                 return frame.handle_operation_error(ec, operr)
@@ -184,13 +186,12 @@
             frame.pushvalue(w_arg_or_err)
         return r_uint(last_instr + 1)
 
-    def next_yield_from(self, frame, w_inputvalue_or_err):
+    def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
         """Fetch the next item of the current 'yield from', push it on
         the frame stack, and raises Yield.  If there isn't one, push
         w_stopiteration_value and returns.  May also just raise.
         """
         space = self.space
-        w_yf = self.w_yielded_from
         try:
             if isinstance(w_yf, GeneratorOrCoroutine):
                 w_retval = w_yf.send_ex(w_inputvalue_or_err)
@@ -199,7 +200,6 @@
             else:
                 w_retval = delegate_to_nongen(space, w_yf, w_inputvalue_or_err)
         except OperationError as e:
-            self.w_yielded_from = None
             if not e.match(space, space.w_StopIteration):
                 raise
             e.normalize_exception(space)
@@ -214,6 +214,7 @@
             return
         else:
             frame.pushvalue(w_retval)
+            self.w_yielded_from = w_yf
             raise Yield
 
     def _leak_stopiteration(self, e):
@@ -256,6 +257,7 @@
         operr = OperationError(w_type, w_val, tb)
         operr.normalize_exception(space)
 
+        # note: w_yielded_from is always None if 'self.running'
         w_yf = self.w_yielded_from
         if (w_yf is not None and
                     operr.match(space, space.w_GeneratorExit)):
@@ -278,6 +280,7 @@
             return     # nothing to do in this case
         space = self.space
         operr = get_generator_exit(space)
+        # note: w_yielded_from is always None if 'self.running'
         w_yf = self.w_yielded_from
         if w_yf is not None:
             self.w_yielded_from = None
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1035,11 +1035,10 @@
         from pypy.interpreter.generator import Coroutine
         in_generator = self.get_generator()
         assert in_generator is not None
-        w_inputvalue = self.popvalue()    # that's always None, actually
+        w_inputvalue = self.popvalue()    # that's always w_None, actually
         w_gen = self.popvalue()
         #
-        in_generator.w_yielded_from = w_gen
-        in_generator.next_yield_from(self, w_inputvalue)
+        in_generator.next_yield_from(self, w_gen, w_inputvalue)
         # Common case: the call above raises Yield.
         # If instead the iterable is empty, next_yield_from() pushed the
         # final result and returns.  In that case, we can just continue
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -397,10 +397,15 @@
 
     def test_gi_yieldfrom(self): """
         def g(x):
+            assert gen.gi_yieldfrom is None
             yield x
+            assert gen.gi_yieldfrom is None
         def f(x):
+            assert gen.gi_yieldfrom is None
             yield from g(x)
+            assert gen.gi_yieldfrom is None
             yield 42
+            assert gen.gi_yieldfrom is None
         gen = f(5)
         assert gen.gi_yieldfrom is None
         assert next(gen) == 5

From pypy.commits at gmail.com  Sun Sep 18 16:28:11 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 18 Sep 2016 13:28:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: PyPy creates a SOABI entry in confvars,
 Numpy needs only the correct SO entry
Message-ID: <57def8db.05371c0a.3fb14.56d8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87210:2b244ba62cf4
Date: 2016-09-18 23:27 +0300
http://bitbucket.org/pypy/pypy/changeset/2b244ba62cf4/

Log:	PyPy creates a SOABI entry in confvars, Numpy needs only the correct
	SO entry

	The duplication of the functionality is from CPYthon, and so is the
	lack of tests :(

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -13,6 +13,7 @@
 import sys
 import os
 import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -62,8 +63,8 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
+    g['SO'] = g['SOABI'][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -75,8 +76,8 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
+    g['SO'] = g['SOABI'][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -526,10 +526,7 @@
 
         # PyPy:
         import imp
-        for suffix, mode, type_ in imp.get_suffixes():
-            if type_ == imp.C_EXTENSION:
-                _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break
+        _CONFIG_VARS['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
 
     if args:
         vals = []

From pypy.commits at gmail.com  Sun Sep 18 17:10:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 14:10:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: fix the test
Message-ID: <57df02d1.6773c20a.b2ece.beea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87212:b15c04d5555f
Date: 2016-09-18 23:09 +0200
http://bitbucket.org/pypy/pypy/changeset/b15c04d5555f/

Log:	fix the test

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1153,20 +1153,21 @@
                 operr.w_type,
                 operr.get_w_value(self.space),
                 w_traceback)
-            self.last_exception = old_last_exception
+            self.pushvalue(SApplicationException(old_last_exception))
         else:
             w_res = self.call_contextmanager_exit_function(
                 w_exitfunc,
                 self.space.w_None,
                 self.space.w_None,
                 self.space.w_None)
-        self.pushvalue(w_unroller)
+            self.pushvalue(self.space.w_None)
         self.pushvalue(w_res)
 
     def WITH_CLEANUP_FINISH(self, oparg, next_instr):
         w_suppress = self.popvalue()
         w_unroller = self.popvalue()
         if isinstance(w_unroller, SApplicationException):
+            self.last_exception = w_unroller.operr
             if self.space.is_true(w_suppress):
                 # __exit__() returned True -> Swallow the exception.
                 self.settopvalue(self.space.w_None)

From pypy.commits at gmail.com  Sun Sep 18 17:10:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 14:10:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Two tests, one passing
Message-ID: <57df02cf.0d8d1c0a.4f0be.6dc5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87211:a9db1cd81bce
Date: 2016-09-18 22:06 +0200
http://bitbucket.org/pypy/pypy/changeset/a9db1cd81bce/

Log:	Two tests, one passing

diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -141,3 +141,22 @@
         else:
             assert False, "should have raised"
         """
+
+    def test_async_with_exception_context(self): """
+        class CM:
+            async def __aenter__(self):
+                pass
+            async def __aexit__(self, *e):
+                1/0
+        async def f():
+            async with CM():
+                raise ValueError
+        c = f()
+        try:
+            c.send(None)
+        except ZeroDivisionError as e:
+            assert e.__context__ is not None
+            assert isinstance(e.__context__, ValueError)
+        else:
+            assert False, "should have raised"
+        """
diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -460,6 +460,23 @@
                         1/0
         except IndexError as exc:
             assert exc.__context__ is None
+        else:
+            assert False, "should have raised"
+
+    def test_with_exception_context(self):
+        class Ctx:
+            def __enter__(self):
+                pass
+            def __exit__(self, *e):
+                1/0
+        try:
+            with Ctx():
+                raise ValueError
+        except ZeroDivisionError as e:
+            assert e.__context__ is not None
+            assert isinstance(e.__context__, ValueError)
+        else:
+            assert False, "should have raised"
 
 
 class AppTestTraceback:

From pypy.commits at gmail.com  Sun Sep 18 17:10:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 14:10:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: Another test and fix
Message-ID: <57df02d3.e97bc20a.d4c0e.b4e6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87213:14723a241c8a
Date: 2016-09-18 23:09 +0200
http://bitbucket.org/pypy/pypy/changeset/14723a241c8a/

Log:	Another test and fix

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -164,8 +164,7 @@
             try:
                 self.next_yield_from(frame, w_yf, w_arg_or_err)
             except OperationError as operr:
-                ec = space.getexecutioncontext()
-                return frame.handle_operation_error(ec, operr)
+                return frame.handle_generator_error(operr)
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
             last_instr = jit.promote(frame.last_instr)
@@ -173,8 +172,7 @@
             return r_uint(last_instr + 1)
 
         if isinstance(w_arg_or_err, SApplicationException):
-            ec = space.getexecutioncontext()
-            return frame.handle_operation_error(ec, w_arg_or_err.operr)
+            return frame.handle_generator_error(w_arg_or_err.operr)
 
         last_instr = jit.promote(frame.last_instr)
         if last_instr == -1:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -101,6 +101,12 @@
         operr = OperationError(w_type, w_value)
         return self.handle_operation_error(ec, operr)
 
+    def handle_generator_error(self, operr):
+        # for generator.py
+        ec = self.space.getexecutioncontext()
+        operr.record_context(self.space, self)
+        return self.handle_operation_error(ec, operr)
+
     def handle_operation_error(self, ec, operr, attach_tb=True):
         if attach_tb:
             if 1:
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -694,6 +694,24 @@
             2,
         ]
 
+    def test_exception_context(self): """
+        import operator
+        def f():
+            try:
+                raise ValueError
+            except ValueError:
+                yield from map(operator.truediv, [2, 3], [4, 0])
+        gen = f()
+        assert next(gen) == 0.5
+        try:
+            next(gen)
+        except ZeroDivisionError as e:
+            assert e.__context__ is not None
+            assert isinstance(e.__context__, ValueError)
+        else:
+            assert False, "should have raised"
+        """
+
 
 class AppTestGeneratorStop:
 

From pypy.commits at gmail.com  Sun Sep 18 17:19:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 14:19:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-corowrapper: comment
Message-ID: <57df04dc.861b1c0a.6d3a0.724f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-corowrapper
Changeset: r87214:491df214da00
Date: 2016-09-18 23:15 +0200
http://bitbucket.org/pypy/pypy/changeset/491df214da00/

Log:	comment

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1159,6 +1159,9 @@
                 operr.w_type,
                 operr.get_w_value(self.space),
                 w_traceback)
+            # push a marker that also contains the old_last_exception,
+            # which must be restored as 'self.last_exception' but only
+            # in WITH_CLEANUP_FINISH
             self.pushvalue(SApplicationException(old_last_exception))
         else:
             w_res = self.call_contextmanager_exit_function(
@@ -1171,9 +1174,9 @@
 
     def WITH_CLEANUP_FINISH(self, oparg, next_instr):
         w_suppress = self.popvalue()
-        w_unroller = self.popvalue()
-        if isinstance(w_unroller, SApplicationException):
-            self.last_exception = w_unroller.operr
+        w_marker = self.popvalue()
+        if isinstance(w_marker, SApplicationException):
+            self.last_exception = w_marker.operr    # may be None
             if self.space.is_true(w_suppress):
                 # __exit__() returned True -> Swallow the exception.
                 self.settopvalue(self.space.w_None)

From pypy.commits at gmail.com  Sun Sep 18 17:24:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 18 Sep 2016 14:24:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for
 test_interpreter.test_build_map_unpack
Message-ID: <57df0618.05d71c0a.f43d4.7604@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87215:bceb849c4008
Date: 2016-09-18 23:24 +0200
http://bitbucket.org/pypy/pypy/changeset/bceb849c4008/

Log:	Fix for test_interpreter.test_build_map_unpack

diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -343,7 +343,7 @@
         if self._len(node.keys) != self._len(node.values):
             raise ValidationError(
                 "Dict doesn't have the same number of keys as values")
-        self._validate_exprs(node.keys)
+        self._validate_exprs(node.keys, null_ok=True)
         self._validate_exprs(node.values)
 
     def visit_Set(self, node):

From pypy.commits at gmail.com  Mon Sep 19 03:15:10 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 00:15:10 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: int_is_true fixed
Message-ID: <57df907e.657bc20a.4520.469d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87216:28e71ef2f8ec
Date: 2016-09-19 09:11 +0200
http://bitbucket.org/pypy/pypy/changeset/28e71ef2f8ec/

Log:	int_is_true fixed

diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -329,6 +329,7 @@
     'VREPI': ('vri_a', ['\xE7','\x45']),
 
     'VCEQ':  ('vrr_b', ['\xE7','\xF8']),
+    'VCHL':  ('vrr_b', ['\xE7','\xF9']),
 
     # pack, merge, shift, ...
     'VMRL':  ('vrr_c', ['\xE7','\x60'], 'v,v,v,m'),
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -186,9 +186,8 @@
         size = sizeloc.value
         tmploc = regalloc.vrm.get_scratch_reg()
         self.mc.VX(tmploc, tmploc, tmploc) # all zero
-        self.mc.VNO(tmploc, tmploc, tmploc) # all one
-        self.mc.VCEQ(resloc, argloc, tmploc, l.itemsize_to_mask(size), 0b0001)
-        flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc)
+        self.mc.VCHL(resloc, argloc, tmploc, l.itemsize_to_mask(size), 0b0001)
+        flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
 
     def emit_vec_float_eq(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
@@ -574,8 +573,9 @@
     prepare_vec_cast_int_to_float = prepare_vec_cast_float_to_int
 
     def prepare_vec_guard_true(self, op):
-        self.assembler.guard_success_cc = c.VEQI
+        self.assembler.guard_success_cc = c.EQ
         return self._prepare_guard(op)
 
     def prepare_vec_guard_false(self, op):
-        self.assembler.guard_success_cc = c.VNEI
+        self.assembler.guard_success_cc = c.NE
+        return self._prepare_guard(op)

From pypy.commits at gmail.com  Mon Sep 19 03:15:14 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 00:15:14 -0700 (PDT)
Subject: [pypy-commit] pypy zarch-simd-support: close branch
Message-ID: <57df9082.a22fc20a.6ebfb.4567@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: zarch-simd-support
Changeset: r87218:ba39bd805638
Date: 2016-09-19 09:12 +0200
http://bitbucket.org/pypy/pypy/changeset/ba39bd805638/

Log:	close branch


From pypy.commits at gmail.com  Mon Sep 19 03:15:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 00:15:16 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: document branches
Message-ID: <57df9084.57981c0a.7bf0f.fd72@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87219:b5e36e08c898
Date: 2016-09-19 09:14 +0200
http://bitbucket.org/pypy/pypy/changeset/b5e36e08c898/

Log:	document branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,11 @@
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
+
+.. branch: zarch-simd-support
+
+s390x implementation for vector operations used in VecOpt
+
+.. branch: ppc-vsx-support
+
+PowerPC implementation for vector operations used in VecOpt

From pypy.commits at gmail.com  Mon Sep 19 03:44:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 00:44:16 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: expand gpr/fpr to a vector reg
Message-ID: <57df9750.4398c20a.ed3b3.561b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87220:a7ce3d0c0585
Date: 2016-09-19 09:42 +0200
http://bitbucket.org/pypy/pypy/changeset/a7ce3d0c0585/

Log:	expand gpr/fpr to a vector reg

diff --git a/rpython/jit/backend/zarch/instruction_builder.py b/rpython/jit/backend/zarch/instruction_builder.py
--- a/rpython/jit/backend/zarch/instruction_builder.py
+++ b/rpython/jit/backend/zarch/instruction_builder.py
@@ -546,6 +546,19 @@
         self.writechar(opcode2)
     return encode_vri_a
 
+def build_vri_c(mnemonic, (opcode1,opcode2), argtypes='v,v,i16,m'):
+    @builder.arguments(argtypes)
+    def encode_vri_c(self, v1, v3, i2, mask4):
+        self.writechar(opcode1)
+        rbx  = (v1 >= 16) << 3
+        rbx |= (v3 >= 16) << 2
+        byte = (v1 & BIT_MASK_4) << 4 | (v3 & BIT_MASK_4)
+        self.writechar(chr(byte))
+        self.write_i16(i2 & BIT_MASK_16)
+        self.writechar(chr((mask4 & BIT_MASK_4) << 4 | (rbx & BIT_MASK_4)))
+        self.writechar(opcode2)
+    return encode_vri_c
+
 def build_vrs_b(mnemonic, (opcode1,opcode2), argtypes='v,r,db,m'):
     @builder.arguments(argtypes)
     def encode_vrs_b(self, v1, r2, db3, m4):
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -300,6 +300,8 @@
 
     'VST':    ('vrx', ['\xE7','\x0E'], 'v,bid'),
 
+    'VREP':   ('vri_c', ['\xE7','\x4D']),
+
     # integral
     # -> arith
     'VA':     ('vrr_c', ['\xE7','\xF3'], 'v,v,v,m'),
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -244,9 +244,20 @@
         assert isinstance(op, VectorOp)
         resloc, loc0 = arglocs
         size = op.bytesize
-        self.mc.VLREP(resloc, loc0, l.itemsize_to_mask(size))
+        if loc0.is_core_reg():
+            self.mc.VLVG(resloc, loc0, l.addr(0), l.itemsize_to_mask(size))
+            self.mc.VREP(resloc, loc0, l.imm0, l.itemsize_to_mask(size))
+        else:
+            self.mc.VLREP(resloc, loc0, l.itemsize_to_mask(size))
 
-    emit_vec_expand_f = emit_vec_expand_i
+    def emit_vec_expand_f(self, op, arglocs, regalloc):
+        assert isinstance(op, VectorOp)
+        resloc, loc0 = arglocs
+        size = op.bytesize
+        if loc0.is_fp_reg():
+            self.mc.VREP(resloc, loc0, l.imm0, l.itemsize_to_mask(size))
+        else:
+            self.mc.VLREP(resloc, loc0, l.itemsize_to_mask(size))
 
     def _accum_reduce(self, op, arg, accumloc, targetloc):
         # Currently the accumulator can ONLY be 64 bit float/int

From pypy.commits at gmail.com  Mon Sep 19 03:44:18 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 00:44:18 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: catchup with default
Message-ID: <57df9752.079bc20a.97325.20f6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87221:950c5eb14f89
Date: 2016-09-19 09:43 +0200
http://bitbucket.org/pypy/pypy/changeset/950c5eb14f89/

Log:	catchup with default

diff too long, truncating to 2000 out of 2146 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -13,6 +13,7 @@
 import sys
 import os
 import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -62,8 +63,8 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
+    g['SO'] = g['SOABI'][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -75,8 +76,8 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
+    g['SO'] = g['SOABI'][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -526,10 +526,7 @@
 
         # PyPy:
         import imp
-        for suffix, mode, type_ in imp.get_suffixes():
-            if type_ == imp.C_EXTENSION:
-                _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break
+        _CONFIG_VARS['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
 
     if args:
         vals = []
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.2
+Version: 1.8.3
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.2"
-__version_info__ = (1, 8, 2)
+__version__ = "1.8.3"
+__version_info__ = (1, 8, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.2"
+                               "\ncompiled with cffi version: 1.8.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt
--- a/pypy/doc/config/translation.profopt.txt
+++ b/pypy/doc/config/translation.profopt.txt
@@ -3,3 +3,14 @@
 RPython program) to gather profile data. Example for pypy-c: "-c 'from
 richards import main;main(); from test import pystone;
 pystone.main()'"
+
+NOTE: be aware of what this does in JIT-enabled executables.  What it
+does is instrument and later optimize the C code that happens to run in
+the example you specify, ignoring any execution of the JIT-generated
+assembler.  That means that you have to choose the example wisely.  If
+it is something that will just generate assembler and stay there, there
+is little value.  If it is something that exercises heavily library
+routines that are anyway written in C, then it will optimize that.  Most
+interesting would be something that causes a lot of JIT-compilation,
+like running a medium-sized test suite several times in a row, in order
+to optimize the warm-up in general.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -17,6 +17,11 @@
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
 
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
+
 .. branch: zarch-simd-support
 
 s390x implementation for vector operations used in VecOpt
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -239,6 +239,10 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+            if config.translation.profopt is not None:
+                raise Exception("Cannot use the --profopt option "
+                                "when --shared is on (it is by default). "
+                                "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1974,6 +1974,7 @@
     'ZeroDivisionError',
     'RuntimeWarning',
     'PendingDeprecationWarning',
+    'UserWarning',
 ]
 
 if sys.platform.startswith("win"):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -63,7 +63,7 @@
         """x.__iter__() <==> iter(x)"""
         return self.space.wrap(self)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -264,25 +264,22 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
-            else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
-            #
             try:
+                if operr is not None:
+                    ec = self.space.getexecutioncontext()
+                    next_instr = self.handle_operation_error(ec, operr)
+                    self.last_instr = intmask(next_instr - 1)
+                else:
+                    # Execution starts just after the last_instr.  Initially,
+                    # last_instr is -1.  After a generator suspends it points to
+                    # the YIELD_VALUE instruction.
+                    next_instr = r_uint(self.last_instr + 1)
+                    if next_instr != 0:
+                        self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
-            executioncontext.return_trace(self, w_exitvalue)
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,12 +57,14 @@
         def f():
             yield 2
         g = f()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
 
     def test_throw3(self):
@@ -221,7 +223,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -562,3 +562,21 @@
         res = f(10).g()
         sys.settrace(None)
         assert res == 10
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.2"
+VERSION = "1.8.3"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer
 from pypy.module._cffi_backend import ctypeprim
 
 
@@ -22,6 +22,7 @@
     is_nonfunc_pointer_or_array = True
 
     def __init__(self, space, ctptr, length, arraysize, extra):
+        assert isinstance(ctptr, W_CTypePointer)
         W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
                                    ctptr.ctitem)
         self.length = length
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -35,8 +35,7 @@
         assert isinstance(ellipsis, bool)
         extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi)
         size = rffi.sizeof(rffi.VOIDP)
-        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult,
-                                could_cast_anything=False)
+        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult)
         self.fargs = fargs
         self.ellipsis = ellipsis
         self.abi = abi
@@ -59,6 +58,16 @@
                     lltype.free(self.cif_descr, flavor='raw')
                     self.cif_descr = lltype.nullptr(CIF_DESCRIPTION)
 
+    def is_unichar_ptr_or_array(self):
+        return False
+
+    def is_char_or_unichar_ptr_or_array(self):
+        return False
+
+    def string(self, cdataobj, maxlen):
+        # Can't use ffi.string() on a function pointer
+        return W_CType.string(self, cdataobj, maxlen)
+
     def new_ctypefunc_completing_argtypes(self, args_w):
         space = self.space
         nargs_declared = len(self.fargs)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -19,7 +19,6 @@
     # XXX this could be improved with an elidable method get_size()
     # that raises in case it's still -1...
 
-    cast_anything = False
     is_primitive_integer = False
     is_nonfunc_pointer_or_array = False
     is_indirect_arg_for_call_python = False
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -120,7 +120,6 @@
 
 class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar):
     _attrs_ = []
-    cast_anything = True
 
     def cast_to_int(self, cdata):
         return self.space.wrap(ord(cdata[0]))
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,12 +14,11 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _attrs_            = ['ctitem', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'accept_str', 'length']
     length = -1
 
-    def __init__(self, space, size, extra, extra_position, ctitem,
-                 could_cast_anything=True):
+    def __init__(self, space, size, extra, extra_position, ctitem):
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
@@ -27,10 +26,11 @@
         #  - for arrays, it is the array item type
         #  - for functions, it is the return type
         self.ctitem = ctitem
-        self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.accept_str = (self.can_cast_anything or
-                            (ctitem.is_primitive_integer and
-                             ctitem.size == rffi.sizeof(lltype.Char)))
+        self.accept_str = (self.is_nonfunc_pointer_or_array and
+                (isinstance(ctitem, ctypevoid.W_CTypeVoid) or
+                 isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or
+                 (ctitem.is_primitive_integer and
+                  ctitem.size == rffi.sizeof(lltype.Char))))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -137,7 +137,10 @@
 
 class W_CTypePtrBase(W_CTypePtrOrArray):
     # base class for both pointers and pointers-to-functions
-    _attrs_ = []
+    _attrs_ = ['is_void_ptr', 'is_voidchar_ptr']
+    _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr']
+    is_void_ptr = False
+    is_voidchar_ptr = False
 
     def convert_to_object(self, cdata):
         ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0]
@@ -154,7 +157,16 @@
             else:
                 raise self._convert_error("compatible pointer", w_ob)
         if self is not other:
-            if not (self.can_cast_anything or other.can_cast_anything):
+            if self.is_void_ptr or other.is_void_ptr:
+                pass     # cast from or to 'void *'
+            elif self.is_voidchar_ptr or other.is_voidchar_ptr:
+                space = self.space
+                msg = ("implicit cast from '%s' to '%s' "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" % (other.name, self.name))
+                space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1)
+            else:
                 raise self._convert_error("compatible pointer", w_ob)
 
         rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
@@ -165,8 +177,8 @@
 
 
 class W_CTypePointer(W_CTypePtrBase):
-    _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types']
-    _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr']
+    _attrs_ = ['is_file', 'cache_array_type', '_array_types']
+    _immutable_fields_ = ['is_file', 'cache_array_type?']
     kind = "pointer"
     cache_array_type = None
     is_nonfunc_pointer_or_array = True
@@ -181,6 +193,8 @@
         self.is_file = (ctitem.name == "struct _IO_FILE" or
                         ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
+        self.is_voidchar_ptr = (self.is_void_ptr or
+                           isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar))
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init, allocator):
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -7,7 +7,6 @@
 
 class W_CTypeVoid(W_CType):
     _attrs_ = []
-    cast_anything = True
     kind = "void"
 
     def __init__(self, space):
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -32,8 +32,8 @@
 @unwrap_spec(w_cdata=cdataobj.W_CData)
 def from_handle(space, w_cdata):
     ctype = w_cdata.ctype
-    if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or
-        not ctype.can_cast_anything):
+    if (not isinstance(ctype, ctypeptr.W_CTypePointer) or
+        not ctype.is_voidchar_ptr):
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3665,3 +3665,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -503,3 +503,10 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_bug_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        q = ffi.new("char[]", "abcd")
+        p = ffi.cast("char(*)(void)", q)
+        raises(TypeError, ffi.string, p)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,16 +30,19 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
+    w_initializer_type = None
+    w_initializer = None
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if space.type(w_initializer) is space.w_str:
+            if w_initializer is not None:
+                if w_initializer_type is space.w_str:
                     a.descr_fromstring(space, w_initializer)
-                elif space.type(w_initializer) is space.w_list:
+                elif w_initializer_type is space.w_list:
                     a.descr_fromlist(space, w_initializer)
                 else:
                     a.extend(w_initializer, True)
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,6 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -28,25 +27,25 @@
     """Return a new PyLongObject object from a C size_t, or NULL on
     failure.
     """
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.LONGLONG], PyObject)
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
     """Return a new PyLongObject object from a C unsigned long, or
     NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONGLONG], PyObject)
 def PyLong_FromUnsignedLongLong(space, val):
     """Return a new PyLongObject object from a C unsigned long long,
     or NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([PyObject], rffi.ULONG, error=-1)
 def PyLong_AsUnsignedLong(space, w_long):
@@ -203,7 +202,10 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    value = rffi.cast(ADDR, p)    # signed integer
+    if value < 0:
+        return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
+    return space.wrap(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -53,10 +53,7 @@
     else:
         n = len(fmt)
     for i in range(n):
-        if ord(fmt[i]) > 255:
-            view.c_format[i] = '*'
-        else:
-            view.c_format[i] = fmt[i]
+        view.c_format[i] = fmt[i]
     view.c_format[n] = '\x00'        
     shape = buf.getshape()
     strides = buf.getstrides()
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,9 +4,7 @@
 PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
 {
     PyObject* newType;
-    /*printf("in foo3type_tp_new, preprocessing...\n"); */
     newType = PyType_Type.tp_new(metatype, args, kwds);
-    /*printf("in foo3type_tp_new, postprocessing...\n"); */
     return newType;
 }
 
@@ -81,4 +79,5 @@
         return;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return;
+    Py_INCREF(&footype);
 }
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -1,5 +1,6 @@
 import sys, py
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import maxint
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
@@ -8,18 +9,20 @@
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
-        value = api.PyLong_FromLong(3)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == 3
+        w_value = api.PyLong_FromLong(3)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == 3
 
-        value = api.PyLong_FromLong(sys.maxint)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == sys.maxint
+        w_value = api.PyLong_FromLong(sys.maxint)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == sys.maxint
 
     def test_aslong(self, space, api):
         w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+        assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsLong(w_value)
         assert value == (sys.maxint - 1)
 
@@ -35,12 +38,16 @@
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsSsize_t(w_value)
         assert value == 2
-        assert space.eq_w(w_value, api.PyLong_FromSsize_t(2))
+        w_val2 = api.PyLong_FromSsize_t(2)
+        assert isinstance(w_val2, W_LongObject)
+        assert space.eq_w(w_value, w_val2)
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
+        assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -102,9 +109,26 @@
         lltype.free(overflow, flavor='raw')
 
     def test_as_voidptr(self, space, api):
+        # CPython returns an int (not a long) depending on the value
+        # passed to PyLong_FromVoidPtr().  In all cases, NULL becomes
+        # the int 0.
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
-        assert space.unwrap(w_l) == 0L
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == 0
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
+        # Positive values also return an int (assuming, like always in
+        # PyPy, that an int is big enough to store any pointer).
+        p = rffi.cast(rffi.VOIDP, maxint)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == maxint
+        assert api.PyLong_AsVoidPtr(w_l) == p
+        # Negative values always return a long.
+        p = rffi.cast(rffi.VOIDP, -maxint-1)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_long)
+        assert space.unwrap(w_l) == maxint+1
+        assert api.PyLong_AsVoidPtr(w_l) == p
 
     def test_sign_and_bits(self, space, api):
         if space.is_true(space.lt(space.sys.get('version_info'),
@@ -128,23 +152,58 @@
         module = self.import_extension('foo', [
             ("from_unsignedlong", "METH_NOARGS",
              """
-                 return PyLong_FromUnsignedLong((unsigned long)-1);
+                 PyObject * obj;
+                 obj = PyLong_FromUnsignedLong((unsigned long)-1);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
         import sys
         assert module.from_unsignedlong() == 2 * sys.maxint + 1
 
     def test_fromlonglong(self):
         module = self.import_extension('foo', [
-            ("from_longlong", "METH_NOARGS",
+            ("from_longlong", "METH_VARARGS",
              """
-                 return PyLong_FromLongLong((long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """),
-            ("from_unsignedlonglong", "METH_NOARGS",
+            ("from_unsignedlonglong", "METH_VARARGS",
              """
-                 return PyLong_FromUnsignedLongLong((unsigned long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromUnsignedLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
-        assert module.from_longlong() == -1
-        assert module.from_unsignedlonglong() == (1<<64) - 1
+        assert module.from_longlong(-1) == -1
+        assert module.from_longlong(0) == 0
+        assert module.from_unsignedlonglong(0) == 0
+        assert module.from_unsignedlonglong(-1) == (1<<64) - 1
 
     def test_from_size_t(self):
         module = self.import_extension('foo', [
@@ -232,10 +291,15 @@
             ("has_sub", "METH_NOARGS",
              """
                 PyObject *ret, *obj = PyLong_FromLong(42);
-                if (obj->ob_type->tp_as_number->nb_subtract)
-                    ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                if (obj->ob_type != &PyLong_Type)
+                    ret = PyLong_FromLong(-2);
                 else
-                    ret = PyLong_FromLong(-1);
+                {
+                    if (obj->ob_type->tp_as_number->nb_subtract)
+                        ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                    else
+                        ret = PyLong_FromLong(-1);
+                }
                 Py_DECREF(obj);
                 return ret;
              """),
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -961,7 +961,6 @@
 
     def test_tp_new_in_subclass_of_type(self):
         module = self.import_module(name='foo3')
-        #print('calling module.footype()...')
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,7 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(ll_strhash), p10, descr=<Calli . r EF=0>)
+            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -15,7 +15,7 @@
         if space.config.translating:
             del self.__class__.interpleveldefs['pypy_getudir']
         super(Module, self).__init__(space, w_name)
-        self.recursionlimit = 100
+        self.recursionlimit = 1000
         self.w_default_encoder = None
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -253,9 +253,28 @@
     from rpython.rtyper.lltypesystem import lltype, rffi
     return space.wrap(rffi.cast(lltype.Signed, handle))
 
+getsizeof_missing = """sys.getsizeof() is not implemented on PyPy.
+
+A memory profiler using this function is most likely to give results
+inconsistent with reality on PyPy.  It would be possible to have
+sys.getsizeof() return a number (with enough work), but that may or
+may not represent how much memory the object uses.  It doesn't even
+make really sense to ask how much *one* object uses, in isolation
+with the rest of the system.  For example, instances have maps,
+which are often shared across many instances; in this case the maps
+would probably be ignored by an implementation of sys.getsizeof(),
+but their overhead is important in some cases if they are many
+instances with unique maps.  Conversely, equal strings may share
+their internal string data even if they are different objects---or
+empty containers may share parts of their internals as long as they
+are empty.  Even stranger, some lists create objects as you read
+them; if you try to estimate the size in memory of range(10**6) as
+the sum of all items' size, that operation will by itself create one
+million integer objects that never existed in the first place.
+"""
+
 def getsizeof(space, w_object, w_default=None):
-    """Not implemented on PyPy."""
     if w_default is None:
-        raise oefmt(space.w_TypeError,
-                    "sys.getsizeof() not implemented on PyPy")
+        raise oefmt(space.w_TypeError, getsizeof_missing)
     return w_default
+getsizeof.__doc__ = getsizeof_missing
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -270,6 +270,10 @@
             return W_SmallLongObject.fromint(val)
         return W_LongObject.fromint(self, val)
 
+    @specialize.argtype(1)
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+        return W_LongObject.fromrarith_int(val)
+
     def newlong_from_rbigint(self, val):
         return newlong(self, val)
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -25,7 +25,6 @@
         space.raises_w(space.w_OverflowError, space.float_w, w_big)
 
     def test_rint_variants(self):
-        py.test.skip("XXX broken!")
         from rpython.rtyper.tool.rfficache import platform
         space = self.space
         for r in platform.numbertype_to_rclass.values():
@@ -36,8 +35,8 @@
             for x in values:
                 if not r.SIGNED:
                     x &= r.MASK
-                w_obj = space.wrap(r(x))
-                assert space.bigint_w(w_obj).eq(rbigint.fromint(x))
+                w_obj = space.newlong_from_rarith_int(r(x))
+                assert space.bigint_w(w_obj).eq(rbigint.fromlong(x))
 
 
 class AppTestLong:
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1002,6 +1002,9 @@
     prepare_op_cond_call_gc_wb_array = prepare_op_cond_call_gc_wb
 
     def prepare_op_cond_call(self, op, fcond):
+        # XXX don't force the arguments to be loaded in specific
+        # locations before knowing if we can take the fast path
+        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -326,6 +326,7 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
+    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_ext = VectorExt()
@@ -1318,6 +1319,16 @@
         # cond_call can't have a return value
         self.execute_call_n(calldescr, func, *args)
 
+    def execute_cond_call_value_i(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_i(calldescr, func, *args)
+        return value
+
+    def execute_cond_call_value_r(self, calldescr, value, func, *args):
+        if not value:
+            value = self.execute_call_r(calldescr, func, *args)
+        return value
+
     def _execute_call(self, calldescr, func, *args):
         effectinfo = calldescr.get_extra_info()
         if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'):
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -759,6 +759,8 @@
         if (opnum != rop.GUARD_TRUE and opnum != rop.GUARD_FALSE
                                     and opnum != rop.COND_CALL):
             return False
+        # NB: don't list COND_CALL_VALUE_I/R here, these two variants
+        # of COND_CALL don't accept a cc as input
         if next_op.getarg(0) is not op:
             return False
         if self.longevity[op][1] > i + 1:
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -11,7 +11,7 @@
 from rpython.jit.backend.llsupport.symbolic import (WORD,
         get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER
+     FLAG_POINTER, CallDescr
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -342,7 +342,9 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- call assembler -----------
+            # ---------- calls -----------
+            if OpHelpers.is_plain_call(op.getopnum()):
+                self.expand_call_shortcut(op)
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -588,6 +590,30 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
+    def expand_call_shortcut(self, op):
+        if not self.cpu.supports_cond_call_value:
+            return
+        descr = op.getdescr()
+        if descr is None:
+            return
+        assert isinstance(descr, CallDescr)
+        effectinfo = descr.get_extra_info()
+        if effectinfo is None or effectinfo.call_shortcut is None:
+            return
+        if op.type == 'r':
+            cond_call_opnum = rop.COND_CALL_VALUE_R
+        elif op.type == 'i':
+            cond_call_opnum = rop.COND_CALL_VALUE_I
+        else:
+            return
+        cs = effectinfo.call_shortcut
+        ptr_box = op.getarg(1 + cs.argnum)
+        value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
+                                       raw=(ptr_box.type == 'i'))
+        self.replace_op_with(op, ResOperation(cond_call_opnum,
+                                              [value_box] + op.getarglist(),
+                                              descr=descr))
+
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,7 +1,8 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr
+     SizeDescr, get_interiorfield_descr, get_call_descr
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -80,6 +81,14 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
+        call_shortcut = CallShortcut(0, tzdescr)
+        effectinfo = EffectInfo(None, None, None, None, None, None,
+                                EffectInfo.EF_RANDOM_EFFECTS,
+                                call_shortcut=call_shortcut)
+        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
+            [lltype.Ptr(T)], lltype.Signed,
+            effectinfo)
+        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -200,6 +209,7 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
+    supports_cond_call_value = True
 
     translate_support_code = None
 
@@ -1429,3 +1439,15 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
+
+    def test_handle_call_shortcut(self):
+        self.check_rewrite("""
+            [p0]
+            i1 = call_i(123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """, """
+            [p0]
+            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
+            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
+            jump(i1)
+        """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,6 +16,7 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
+    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2389,7 +2389,7 @@
             f2 = longlong.getfloatstorage(3.4)
             frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5, f1, f2)
             assert not called
-            for j in range(5):
+            for j in range(6):
                 assert self.cpu.get_int_value(frame, j) == j
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 6)) == 1.2
             assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 3.4
@@ -2447,6 +2447,54 @@
                                            67, 89)
             assert called == [(67, 89)]
 
+    def test_cond_call_value(self):
+        if not self.cpu.supports_cond_call_value:
+            py.test.skip("missing supports_cond_call_value")
+
+        def func_int(*args):
+            called.append(args)
+            return len(args) * 100 + 1000
+
+        for i in range(5):
+            called = []
+
+            FUNC = self.FuncType([lltype.Signed] * i, lltype.Signed)
+            func_ptr = llhelper(lltype.Ptr(FUNC), func_int)
+            calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+                                             EffectInfo.MOST_GENERAL)
+
+            ops = '''
+            [i0, i1, i2, i3, i4, i5, i6, f0, f1]
+            i15 = cond_call_value_i(i1, ConstClass(func_ptr), %s)
+            guard_false(i0, descr=faildescr) [i1,i2,i3,i4,i5,i6,i15, f0,f1]
+            finish(i15)
+            ''' % ', '.join(['i%d' % (j + 2) for j in range(i)] +
+                            ["descr=calldescr"])
+            loop = parse(ops, namespace={'faildescr': BasicFailDescr(),
+                                         'func_ptr': func_ptr,
+                                         'calldescr': calldescr})
+            looptoken = JitCellToken()
+            self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+            f1 = longlong.getfloatstorage(1.2)
+            f2 = longlong.getfloatstorage(3.4)
+            frame = self.cpu.execute_token(looptoken, 1, 50, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert not called
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        50, 1, 2, 3, 4, 5, 50]
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(
+                        self.cpu.get_float_value(frame, 8)) == 3.4
+            #
+            frame = self.cpu.execute_token(looptoken, 1, 0, 1, 2, 3, 4, 5,
+                                           f1, f2)
+            assert called == [(1, 2, 3, 4)[:i]]
+            assert [self.cpu.get_int_value(frame, j) for j in range(7)] == [
+                        0, 1, 2, 3, 4, 5, i * 100 + 1000]
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 7)) == 1.2
+            assert longlong.getrealfloat(self.cpu.get_float_value(frame, 8)) == 3.4
+
     def test_force_operations_returning_void(self):
         values = []
         def maybe_force(token, flag):
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -594,7 +594,7 @@
         return subset, d['f'], vtableptr
 
     def getresulttype(self):
-        if self.opnum == rop.CALL_I:
+        if self.opnum == rop.CALL_I or self.opnum == rop.COND_CALL_VALUE_I:
             return lltype.Signed
         elif self.opnum == rop.CALL_F:
             return lltype.Float
@@ -712,7 +712,12 @@
 class CondCallOperation(BaseCallOperation):
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
-        v_cond = builder.get_bool_var(r)
+        if self.opnum == rop.COND_CALL:
+            RESULT_TYPE = lltype.Void
+            v_cond = builder.get_bool_var(r)
+        else:
+            RESULT_TYPE = lltype.Signed
+            v_cond = r.choice(builder.intvars)
         subset = builder.subset_of_intvars(r)[:4]
         for i in range(len(subset)):
             if r.random() < 0.35:
@@ -724,8 +729,10 @@
                 seen.append(args)
             else:
                 assert seen[0] == args
+            if RESULT_TYPE is lltype.Signed:
+                return len(args) - 42000
         #
-        TP = lltype.FuncType([lltype.Signed] * len(subset), lltype.Void)
+        TP = lltype.FuncType([lltype.Signed] * len(subset), RESULT_TYPE)
         ptr = llhelper(lltype.Ptr(TP), call_me)
         c_addr = ConstAddr(llmemory.cast_ptr_to_adr(ptr), builder.cpu)
         args = [v_cond, c_addr] + subset
@@ -769,6 +776,7 @@
 for i in range(2):
     OPERATIONS.append(GuardClassOperation(rop.GUARD_CLASS))
     OPERATIONS.append(CondCallOperation(rop.COND_CALL))
+    OPERATIONS.append(CondCallOperation(rop.COND_CALL_VALUE_I))
     OPERATIONS.append(RaisingCallOperation(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationGuardNoException(rop.CALL_N))
     OPERATIONS.append(RaisingCallOperationWrongGuardException(rop.CALL_N))
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -186,8 +186,8 @@
         # copy registers to the frame, with the exception of the
         # 'cond_call_register_arguments' and eax, because these have already
         # been saved by the caller.  Note that this is not symmetrical:
-        # these 5 registers are saved by the caller but restored here at
-        # the end of this function.
+        # these 5 registers are saved by the caller but 4 of them are
+        # restored here at the end of this function.
         self._push_all_regs_to_frame(mc, cond_call_register_arguments + [eax],
                                      supports_floats, callee_only)
         # the caller already did push_gcmap(store=True)
@@ -210,7 +210,7 @@
             mc.ADD(esp, imm(WORD * 7))
         self.set_extra_stack_depth(mc, 0)
         self.pop_gcmap(mc)   # cancel the push_gcmap(store=True) in the caller
-        self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only)
+        self._pop_all_regs_from_frame(mc, [eax], supports_floats, callee_only)
         mc.RET()
         return mc.materialize(self.cpu, [])
 
@@ -1715,7 +1715,8 @@
         self.implement_guard(guard_token)
         # If the previous operation was a COND_CALL, overwrite its conditional
         # jump to jump over this GUARD_NO_EXCEPTION as well, if we can
-        if self._find_nearby_operation(-1).getopnum() == rop.COND_CALL:
+        if self._find_nearby_operation(-1).getopnum() in (
+                rop.COND_CALL, rop.COND_CALL_VALUE_I, rop.COND_CALL_VALUE_R):
             jmp_adr = self.previous_cond_call_jcond
             offset = self.mc.get_relative_pos() - jmp_adr
             if offset <= 127:
@@ -2393,7 +2394,7 @@
     def label(self):
         self._check_frame_depth_debug(self.mc)
 
-    def cond_call(self, op, gcmap, imm_func, arglocs):
+    def cond_call(self, gcmap, imm_func, arglocs, resloc=None):
         assert self.guard_success_cc >= 0
         self.mc.J_il8(rx86.invert_condition(self.guard_success_cc), 0)
                                                             # patched later
@@ -2406,11 +2407,14 @@
         # plus the register 'eax'
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         should_be_saved = self._regalloc.rm.reg_bindings.values()
+        restore_eax = False
         for gpr in cond_call_register_arguments + [eax]:
-            if gpr not in should_be_saved:
+            if gpr not in should_be_saved or gpr is resloc:
                 continue
             v = gpr_reg_mgr_cls.all_reg_indexes[gpr.value]
             self.mc.MOV_br(v * WORD + base_ofs, gpr.value)
+            if gpr is eax:
+                restore_eax = True
         #
         # load the 0-to-4 arguments into these registers
         from rpython.jit.backend.x86.jump import remap_frame_layout
@@ -2434,8 +2438,16 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.CALL(imm(follow_jump(cond_call_adr)))
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        if resloc is not None and resloc is not eax:
+            self.mc.MOV(resloc, eax)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  We must only restore eax, if
+        # needed.
+        if restore_eax:
+            v = gpr_reg_mgr_cls.all_reg_indexes[eax.value]
+            self.mc.MOV_rb(eax.value, v * WORD + base_ofs)
+        #
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -938,16 +938,45 @@
                     self.rm.force_spill_var(box)
                     assert box not in self.rm.reg_bindings
         #
-        assert op.type == 'v'
         args = op.getarglist()
         assert 2 <= len(args) <= 4 + 2     # maximum 4 arguments
-        v = args[1]
-        assert isinstance(v, Const)
-        imm_func = self.rm.convert_to_imm(v)
+        v_func = args[1]
+        assert isinstance(v_func, Const)
+        imm_func = self.rm.convert_to_imm(v_func)
+
+        # Delicate ordering here.  First get the argument's locations.
+        # If this also contains args[0], this returns the current
+        # location too.
         arglocs = [self.loc(args[i]) for i in range(2, len(args))]
         gcmap = self.get_gcmap()
-        self.load_condition_into_cc(op.getarg(0))
-        self.assembler.cond_call(op, gcmap, imm_func, arglocs)
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            resloc = None
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+
+            # Test the register for the result.
+            self.assembler.test_location(resloc)
+            self.assembler.guard_success_cc = rx86.Conditions['Z']
+
+        self.assembler.cond_call(gcmap, imm_func, arglocs, resloc)
+
+    consider_cond_call_value_i = consider_cond_call
+    consider_cond_call_value_r = consider_cond_call
 
     def consider_call_malloc_nursery(self, op):
         size_box = op.getarg(0)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -16,6 +16,7 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
+    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,9 +7,10 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection)
+    EffectInfo, CallInfoCollection, CallShortcut)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
+from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -214,6 +215,7 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
+        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -228,6 +230,12 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
+            if hasattr(funcobj, 'graph'):
+                call_shortcut = self.find_call_shortcut(funcobj.graph)
+            if getattr(func, "_call_shortcut_", False):
+                assert call_shortcut is not None, (
+                    "%r: marked as @jit.call_shortcut but shortcut not found"
+                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -242,6 +250,8 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
+                if hasattr(graph.func, '_call_shortcut_'):
+                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
@@ -298,6 +308,7 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
+            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -368,3 +379,65 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
+
+    def find_call_shortcut(self, graph):
+        """Identifies graphs that start like this:
+
+           def graph(x, y, z):         def graph(x, y, z):
+               if y.field:                 r = y.field
+                   return y.field          if r: return r
+        """
+        block = graph.startblock
+        if len(block.operations) == 0:
+            return
+        op = block.operations[0]
+        if op.opname != 'getfield':
+            return
+        [v_inst, c_fieldname] = op.args
+        if not isinstance(v_inst, Variable):
+            return
+        v_result = op.result
+        if v_result.concretetype != graph.getreturnvar().concretetype:
+            return
+        if v_result.concretetype == lltype.Void:
+            return
+        argnum = i = 0
+        while block.inputargs[i] is not v_inst:
+            if block.inputargs[i].concretetype != lltype.Void:
+                argnum += 1
+            i += 1
+        PSTRUCT = v_inst.concretetype
+        v_check = v_result
+        fastcase = True
+        for op in block.operations[1:]:
+            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
+                    and v_check is op.args[0]):
+                v_check = op.result
+            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
+                v_check = op.result
+                fastcase = not fastcase
+            elif (op.opname in ('int_eq', 'int_ne')
+                    and v_check is op.args[0]
+                    and isinstance(op.args[1], Constant)
+                    and op.args[1].value == 0):
+                v_check = op.result
+                if op.opname == 'int_eq':
+                    fastcase = not fastcase
+            else:
+                return
+        if v_check.concretetype is not lltype.Bool:
+            return
+        if block.exitswitch is not v_check:
+            return
+
+        links = [link for link in block.exits if link.exitcase == fastcase]
+        if len(links) != 1:
+            return
+        [link] = links
+        if link.args != [v_result]:
+            return
+        if not link.target.is_final_block():
+            return
+
+        fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,7 +117,8 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True):
+                can_collect=True,
+                call_shortcut=None):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -135,7 +136,8 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect)
+               can_collect,
+               call_shortcut)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -190,6 +192,7 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
+        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -275,7 +278,8 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True):
+                                 can_collect=True,
+                                 call_shortcut=None):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -364,7 +368,8 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect)
+                      can_collect,
+                      call_shortcut)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -387,6 +392,24 @@
 
 # ____________________________________________________________
 
+
+class CallShortcut(object):
+    def __init__(self, argnum, fielddescr):
+        self.argnum = argnum
+        self.fielddescr = fielddescr
+
+    def __eq__(self, other):
+        return (isinstance(other, CallShortcut) and
+                self.argnum == other.argnum and
+                self.fielddescr == other.fielddescr)
+    def __ne__(self, other):
+        return not (self == other)
+    def __hash__(self):
+        return hash((self.argnum, self.fielddescr))
+
+# ____________________________________________________________
+
+
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -200,8 +200,12 @@
             or v.concretetype != lltype.Bool):
             return False
         for op in block.operations[::-1]:
-            if v in op.args:
-                return False   # variable is also used in cur block
+            # check if variable is used in block
+            for arg in op.args:
+                if arg == v:
+                    return False
+                if isinstance(arg, ListOfKind) and v in arg.content:
+                    return False
             if v is op.result:
                 if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne',
                                      'int_gt', 'int_ge',
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl
-from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
 
 
 class FakePolicy:
@@ -368,3 +368,100 @@
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.check_can_collect() == expected
+
+def test_find_call_shortcut():
+    class FakeCPU:
+        def fielddescrof(self, TYPE, fieldname):
+            if isinstance(TYPE, lltype.GcStruct):
+                if fieldname == 'inst_foobar':
+                    return 'foobardescr'
+                if fieldname == 'inst_fooref':
+                    return 'foorefdescr'
+            if TYPE == RAW and fieldname == 'x':
+                return 'xdescr'
+            assert False, (TYPE, fieldname)
+    cc = CallControl(FakeCPU())
+
+    class B(object):
+        foobar = 0
+        fooref = None
+
+    def f1(a, b, c):
+        if b.foobar:
+            return b.foobar
+        b.foobar = a + c
+        return b.foobar
+
+    def f2(x, y, z, b):
+        r = b.fooref
+        if r is not None:
+            return r
+        r = b.fooref = B()
+        return r
+
+    class Space(object):
+        def _freeze_(self):
+            return True
+    space = Space()
+
+    def f3(space, b):
+        r = b.foobar
+        if not r:
+            r = b.foobar = 123
+        return r
+
+    def f4(raw):
+        r = raw.x
+        if r != 0:
+            return r
+        raw.x = 123
+        return 123
+    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
+
+    def f5(b):
+        r = b.foobar
+        if r == 0:
+            r = b.foobar = 123
+        return r
+
+    def f(a, c):
+        b = B()
+        f1(a, b, c)
+        f2(a, c, a, b)
+        f3(space, b)
+        r = lltype.malloc(RAW, flavor='raw')
+        f4(r)
+        f5(b)
+
+    rtyper = support.annotate(f, [10, 20])
+    f1_graph = rtyper.annotator.translator._graphof(f1)
+    assert cc.find_call_shortcut(f1_graph) == CallShortcut(1, "foobardescr")
+    f2_graph = rtyper.annotator.translator._graphof(f2)
+    assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
+    f3_graph = rtyper.annotator.translator._graphof(f3)
+    assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
+    f4_graph = rtyper.annotator.translator._graphof(f4)
+    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
+    f5_graph = rtyper.annotator.translator._graphof(f5)
+    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
+
+def test_cant_find_call_shortcut():
+    from rpython.jit.backend.llgraph.runner import LLGraphCPU
+
+    @jit.dont_look_inside
+    @jit.call_shortcut
+    def f1(n):
+        return n + 17   # no call shortcut found
+
+    def f(n):
+        return f1(n)
+
+    rtyper = support.annotate(f, [1])
+    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
+    cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
+    res = cc.find_all_graphs(FakePolicy())
+    [f_graph] = [x for x in res if x.func is f]
+    call_op = f_graph.startblock.operations[0]
+    assert call_op.opname == 'direct_call'
+    e = py.test.raises(AssertionError, cc.getcalldescr, call_op)
+    assert "shortcut not found" in str(e.value)
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -243,6 +243,20 @@
         assert block.exitswitch == (opname, v1, '-live-before')
         assert block.exits == exits
 
+def test_optimize_goto_if_not__argument_to_call():
+    for opname in ['ptr_iszero', 'ptr_nonzero']:
+        v1 = Variable()
+        v3 = Variable(); v3.concretetype = lltype.Bool
+        v4 = Variable()
+        block = Block([v1])
+        callop = SpaceOperation('residual_call_r_i',
+                ["fake", ListOfKind('int', [v3])], v4)
+        block.operations = [SpaceOperation(opname, [v1], v3), callop]
+        block.exitswitch = v3
+        block.exits = exits = [FakeLink(False), FakeLink(True)]
+        res = Transformer().optimize_goto_if_not(block)
+        assert not res
+
 def test_symmetric():
     ops = {'int_add': 'int_add',
            'int_or': 'int_or',
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -101,6 +101,18 @@
     if condbox.getint():
         do_call_n(cpu, metainterp, argboxes[1:], descr)
 
+def do_cond_call_value_i(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getint()
+    if value == 0:
+        value = do_call_i(cpu, metainterp, argboxes[1:], descr)
+    return value
+
+def do_cond_call_value_r(cpu, metainterp, argboxes, descr):
+    value = argboxes[0].getref_base()
+    if not value:
+        value = do_call_r(cpu, metainterp, argboxes[1:], descr)
+    return value
+
 def do_getarrayitem_gc_i(cpu, _, arraybox, indexbox, arraydescr):
     array = arraybox.getref_base()
     index = indexbox.getint()
@@ -366,6 +378,8 @@
                          rop.CALL_ASSEMBLER_I,
                          rop.CALL_ASSEMBLER_N,
                          rop.INCREMENT_DEBUG_COUNTER,
+                         rop.COND_CALL_VALUE_R,
+                         rop.COND_CALL_VALUE_I,
                          rop.COND_CALL_GC_WB,
                          rop.COND_CALL_GC_WB_ARRAY,
                          rop.ZERO_ARRAY,
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
 from rpython.jit.metainterp.executor import execute
 from rpython.jit.metainterp.optimize import InvalidLoop
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -11,6 +11,8 @@
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.debug import debug_print
+from rpython.rtyper import rclass
+from rpython.rtyper.lltypesystem import llmemory
 from rpython.jit.metainterp.optimize import SpeculativeError
 
 
@@ -799,6 +801,21 @@
         if not (0 <= index < arraylength):
             raise SpeculativeError
 
+    @staticmethod
+    def _check_subclass(vtable1, vtable2): # checks that vtable1 is a subclass of vtable2
+        known_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable1),
+            rclass.CLASSTYPE)
+        expected_class = llmemory.cast_adr_to_ptr(
+            llmemory.cast_int_to_adr(vtable2),
+            rclass.CLASSTYPE)
+        # note: the test is for a range including 'max', but 'max'
+        # should never be used for actual classes.  Including it makes
+        # it easier to pass artificial tests.
+        return (expected_class.subclassrange_min
+                <= known_class.subclassrange_min
+                <= expected_class.subclassrange_max)
+
     def is_virtual(self, op):
         if op.type == 'r':
             opinfo = self.getptrinfo(op)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -324,37 +324,24 @@
             return
         self.emit_operation(op)
 
-    def _check_subclass(self, vtable1, vtable2):
-        # checks that vtable1 is a subclass of vtable2
-        known_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable1),
-            rclass.CLASSTYPE)
-        expected_class = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(vtable2),
-            rclass.CLASSTYPE)
-        if (expected_class.subclassrange_min
-                <= known_class.subclassrange_min
-                <= expected_class.subclassrange_max):
-            return True
-        return False
-
     def optimize_GUARD_SUBCLASS(self, op):
         info = self.getptrinfo(op.getarg(0))
+        optimizer = self.optimizer
         if info and info.is_constant():
             c = self.get_box_replacement(op.getarg(0))
-            vtable = self.optimizer.cpu.ts.cls_of_box(c).getint()
-            if self._check_subclass(vtable, op.getarg(1).getint()):
+            vtable = optimizer.cpu.ts.cls_of_box(c).getint()
+            if optimizer._check_subclass(vtable, op.getarg(1).getint()):
                 return
             raise InvalidLoop("GUARD_SUBCLASS(const) proven to always fail")
         if info is not None and info.is_about_object():
-            known_class = info.get_known_class(self.optimizer.cpu)
+            known_class = info.get_known_class(optimizer.cpu)
             if known_class:
-                if self._check_subclass(known_class.getint(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(known_class.getint(),
+                                             op.getarg(1).getint()):
                     return
             elif info.get_descr() is not None:
-                if self._check_subclass(info.get_descr().get_vtable(),
-                                        op.getarg(1).getint()):
+                if optimizer._check_subclass(info.get_descr().get_vtable(),
+                                             op.getarg(1).getint()):
                     return
         self.emit_operation(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -17,12 +17,13 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
      VirtualStateInfo
-from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.optimizeopt import info, optimizer
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.tool import oparser
 
 class FakeOptimizer(object):
     optearlyforce = None
+    optimizer = optimizer.Optimizer
 
     class cpu:
         remove_gctypeptr = True
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -102,6 +102,8 @@
     node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2)
     node_vtable3 = lltype.malloc(OBJECT_VTABLE, immortal=True)
     node_vtable3.name = rclass.alloc_array_name('node3')
+    node_vtable3.subclassrange_min = 3
+    node_vtable3.subclassrange_max = 3
     node_vtable_adr3 = llmemory.cast_ptr_to_adr(node_vtable3)
     cpu = runner.LLGraphCPU(None)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -3,7 +3,8 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo,\
      VStructStateInfo, LEVEL_CONSTANT,\
      VArrayStateInfo, not_virtual, VirtualState,\
-     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo
+     GenerateGuardState, VirtualStatesCantMatch, VArrayStructStateInfo,\
+     VirtualStateConstructor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, TargetToken
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
      InputArgFloat
@@ -26,6 +27,7 @@
     def __init__(self, cpu):
         self.cpu = cpu
         self.optearlyforce = None
+        self.optimizer = Optimizer
 
 class BaseTestGenerateGuards(BaseTest):
     def setup_class(self):
@@ -87,6 +89,42 @@
         vs = VirtualState([info0])
         assert vs.make_inputargs(args, optimizer) == []
 
+    def test_make_inputargs_2(self):
+        # Ensure that make_inputargs does not error when the lengths of the fields
+        # for the runtime box does not match what the virtual state expected.
+        # This can occur in unroll.py, as not all paths to make_inputargs are
+        # guareded with a generalization_of check. The property is validated
+        # subsequently in all cases, so we just need to ensure that this case does
+        # not cause segfaults.
+        optimizer = FakeOptimizer(self.cpu)
+        classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        innervalue1 = info.InstancePtrInfo(
+                known_class=classbox1, is_virtual=True,
+                descr=self.valuedescr.get_parent_descr())
+        for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
+            innervalue1.setfield(field, None, ConstInt(42))
+        classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3))
+        innervalue2 = info.InstancePtrInfo(
+                known_class=classbox2, is_virtual=True,
+                descr=self.valuedescr3.get_parent_descr())
+        for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
+            innervalue2.setfield(field, None, ConstInt(42))
+
+        nodebox1 = InputArgRef(self.nodeaddr)
+        nodebox2 = InputArgRef(self.myptr3)
+        nodebox1.set_forwarded(innervalue1)
+        nodebox2.set_forwarded(innervalue2)
+
+        constr = VirtualStateConstructor(optimizer)
+        vs1 = constr.get_virtual_state([nodebox1])
+        constr = VirtualStateConstructor(optimizer)
+        vs2 = constr.get_virtual_state([nodebox2])
+
+        # This should succeed with no exceptions
+        vs1.make_inputargs([nodebox2], optimizer, force_boxes=False)
+        assert not vs1.generalization_of(vs2, optimizer)
+        assert not vs2.generalization_of(vs1, optimizer)
+
     def test_position_generalization(self):
         def postest(info1, info2):
             info1.position = 0
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -167,7 +167,8 @@
                [self.get_box_replacement(x) for x in end_jump.getarglist()],
                self.optimizer, force_boxes=True)
             for arg in args:
-                self.optimizer.force_box(arg)
+                if arg is not None:
+                    self.optimizer.force_box(arg)
         except VirtualStatesCantMatch:
             raise InvalidLoop("Virtual states did not match "
                               "after picking the virtual state, when forcing"
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -184,7 +184,10 @@
             raise VirtualStatesCantMatch()
         else:
             assert isinstance(info, AbstractStructPtrInfo)
-        for i in range(len(self.fielddescrs)):
+
+        # The min operation ensures we don't wander off either array, as not all
+        # to make_inputargs have validated their inputs with generate_guards.
+        for i in range(min(len(self.fielddescrs), len(info._fields))):
             state = self.fieldstate[i]
             if not state:
                 continue
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1146,8 +1146,8 @@
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',
     'CALL/*d/rfin',
-    'COND_CALL/*d/n',
-    # a conditional call, with first argument as a condition
+    'COND_CALL/*d/n',   # a conditional call, with first argument as a condition
+    'COND_CALL_VALUE/*d/ri',  # same but returns a result; emitted by rewrite
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',
     'CALL_LOOPINVARIANT/*d/rfin',
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -195,7 +195,8 @@
                            'new_with_vtable': 2, 'getinteriorfield_gc_i': 2,
                            'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
                            'call_i': 6, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
-                           'guard_no_exception': 8, 'new': 2})
+                           'guard_no_exception': 8, 'new': 2,
+                           'guard_nonnull': 2})
 
     def test_unrolling_of_dict_iter(self):
         driver = JitDriver(greens = [], reds = ['n'])
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -257,6 +257,28 @@
     func.oopspec = "jit.not_in_trace()"   # note that 'func' may take arguments
     return func
 
+def call_shortcut(func):
+    """A decorator to ensure that a function has a fast-path.
+    DOES NOT RELIABLY WORK ON METHODS, USE ONLY ON FUNCTIONS!
+
+    Only useful on functions that the JIT doesn't normally look inside.
+    It still replaces residual calls to that function with inline code
+    that checks for a fast path, and only does the call if not.  For
+    now, graphs made by the following kinds of functions are detected:
+
+           def func(x, y, z):         def func(x, y, z):
+               if y.field:                 r = y.field
+                   return y.field          if r is None:
+               ...                             ...
+                                           return r

From pypy.commits at gmail.com  Mon Sep 19 05:06:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Sep 2016 02:06:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2325/2361: __class__ assignment
 between two classes with the same
Message-ID: <57dfaa97.8a13c20a.e8505.5fb3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87222:4dd20155b7eb
Date: 2016-09-19 11:05 +0200
http://bitbucket.org/pypy/pypy/changeset/4dd20155b7eb/

Log:	Issue #2325/2361: __class__ assignment between two classes with the
	same slots. Adapted from PR #459 from mark wililams. Thanks!

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -797,9 +797,7 @@
         class AA(object):
             __slots__ = ('a',)
         aa = AA()
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "aa.__class__ = A")
+        aa.__class__ = A
         raises(TypeError, "aa.__class__ = object")
         class Z1(A):
             pass
@@ -861,9 +859,13 @@
             __slots__ = ['a', 'b']
         class Order2(object):
             __slots__ = ['b', 'a']
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "Order1().__class__ = Order2")
+        Order1().__class__ = Order2
+
+        # like CPython, the order of slot names doesn't matter
+        x = Order1()
+        x.a, x.b = 1, 2
+        x.__class__ = Order2
+        assert (x.a, x.b) == (1, 2)
 
         class U1(object):
             __slots__ = ['a', 'b']
@@ -873,10 +875,11 @@
             __slots__ = ['a', 'b']
         class V2(V1):
             __slots__ = ['c', 'd', 'e']
-        # the following line does not work on CPython >= 2.6 either.
-        # that's just obscure.  Really really.  So we just ignore
-        # the whole issue until someone comes complaining.  Then we'll
-        # just kill slots altogether apart from maybe doing a few checks.
+        # the following line does not work on CPython either: we can't
+        # change a class if the old and new class have different layouts
+        # that look compatible but aren't, because they don't have the
+        # same base-layout class (even if these base classes are
+        # themselves compatible)...  obscure.
         raises(TypeError, "U2().__class__ = V2")
 
     def test_name(self):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,9 +102,10 @@
     """
     _immutable_ = True
 
-    def __init__(self, typedef, nslots, base_layout=None):
+    def __init__(self, typedef, nslots, newslotnames=[], base_layout=None):
         self.typedef = typedef
         self.nslots = nslots
+        self.newslotnames = newslotnames[:]    # make a fixed-size list
         self.base_layout = base_layout
 
     def issublayout(self, parent):
@@ -114,6 +115,12 @@
                 return False
         return True
 
+    def expand(self, hasdict, weakrefable):
+        """Turn this Layout into a tuple.  If two classes get equal
+        tuples, it means their instances have a fully compatible layout."""
+        return (self.typedef, self.newslotnames, self.base_layout,
+                hasdict, weakrefable)
+
 
 # possible values of compares_by_identity_status
 UNKNOWN = 0
@@ -289,8 +296,7 @@
 
     # compute a tuple that fully describes the instance layout
     def get_full_instance_layout(self):
-        layout = self.layout
-        return (layout, self.hasdict, self.weakrefable)
+        return self.layout.expand(self.hasdict, self.weakrefable)
 
     def compute_default_mro(self):
         return compute_C3_mro(self.space, self)
@@ -1001,11 +1007,15 @@
         w_self.weakrefable = w_self.weakrefable or w_base.weakrefable
     return hasoldstylebase
 
+
 def create_all_slots(w_self, hasoldstylebase, w_bestbase, force_new_layout):
+    from pypy.objspace.std.listobject import StringSort
+
     base_layout = w_bestbase.layout
     index_next_extra_slot = base_layout.nslots
     space = w_self.space
     dict_w = w_self.dict_w
+    newslotnames = []
     if '__slots__' not in dict_w:
         wantdict = True
         wantweakref = True
@@ -1031,8 +1041,22 @@
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
             else:
-                index_next_extra_slot = create_slot(w_self, slot_name,
-                                                    index_next_extra_slot)
+                newslotnames.append(slot_name)
+        # Sort the list of names collected so far
+        sorter = StringSort(newslotnames, len(newslotnames))
+        sorter.sort()
+        # Try to create all slots in order.  The creation of some of
+        # them might silently fail; then we delete the name from the
+        # list.  At the end, 'index_next_extra_slot' has been advanced
+        # by the final length of 'newslotnames'.
+        i = 0
+        while i < len(newslotnames):
+            if create_slot(w_self, newslotnames[i], index_next_extra_slot):
+                index_next_extra_slot += 1
+                i += 1
+            else:
+                del newslotnames[i]
+    #
     wantdict = wantdict or hasoldstylebase
     if wantdict:
         create_dict_slot(w_self)
@@ -1041,11 +1065,12 @@
     if '__del__' in dict_w:
         w_self.hasuserdel = True
     #
+    assert index_next_extra_slot == base_layout.nslots + len(newslotnames)
     if index_next_extra_slot == base_layout.nslots and not force_new_layout:
         return base_layout
     else:
         return Layout(base_layout.typedef, index_next_extra_slot,
-                      base_layout=base_layout)
+                      newslotnames, base_layout=base_layout)
 
 def create_slot(w_self, slot_name, index_next_extra_slot):
     space = w_self.space
@@ -1058,9 +1083,10 @@
         slot_name = space.str_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
         member = Member(index_next_extra_slot, slot_name, w_self)
-        index_next_extra_slot += 1
         w_self.dict_w[slot_name] = space.wrap(member)
-    return index_next_extra_slot
+        return True
+    else:
+        return False
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -646,6 +646,9 @@
     def ll_str(self, x):
         return self.getstr()
 
+    def get_ll_eq_function(self):
+        return None
+
 
 class MultipleFrozenPBCReprBase(CanBeNull, Repr):
     def convert_const(self, pbc):
@@ -654,6 +657,9 @@
         frozendesc = self.rtyper.annotator.bookkeeper.getdesc(pbc)
         return self.convert_desc(frozendesc)
 
+    def get_ll_eq_function(self):
+        return None
+
 class MultipleUnrelatedFrozenPBCRepr(MultipleFrozenPBCReprBase):
     """For a SomePBC of frozen PBCs that have no common access set.
     The only possible operation on such a thing is comparison with 'is'."""
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -1729,6 +1729,23 @@
         res = self.interpret(f, [])
         assert res == 42
 
+    def test_equality_of_frozen_pbcs_inside_data_structures(self):
+        class A:
+            def _freeze_(self):
+                return True
+        a1 = A()
+        a2 = A()
+        def f():
+            return [a1] == [a1]
+        def g(i):
+            x1 = [a1, a2][i]
+            x2 = [a1, a2][i]
+            return (x1,) == (x2,)
+        res = self.interpret(f, [])
+        assert res == True
+        res = self.interpret(g, [1])
+        assert res == True
+
 # ____________________________________________________________
 
 def test_hlinvoke_simple():

From pypy.commits at gmail.com  Mon Sep 19 07:47:02 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Mon, 19 Sep 2016 04:47:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement default keyword on min & max
 with tests
Message-ID: <57dfd036.c186c20a.ff68f.ad71@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87224:85eef6bf8b2e
Date: 2016-09-19 12:28 +0100
http://bitbucket.org/pypy/pypy/changeset/85eef6bf8b2e/

Log:	Implement default keyword on min & max with tests

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -136,14 +136,24 @@
                         "%s() expects at least one argument",
                         implementation_of)
         w_key = None
+	has_default = False
         if any_kwds:
             kwds = args.keywords
-            if kwds[0] == "key" and len(kwds) == 1:
-                w_key = args.keywords_w[0]
-            else:
-                raise oefmt(space.w_TypeError,
-                            "%s() got unexpected keyword argument",
-                            implementation_of)
+            for n in range(len(kwds)):
+                if kwds[n] == "key":
+                    w_key = args.keywords_w[n]
+                elif kwds[n] == "default":
+                    w_default = args.keywords_w[n]
+                    has_default = True
+                else:
+                    raise oefmt(space.w_TypeError,
+                                "%s() got unexpected keyword argument",
+                                implementation_of)
+
+        if has_default and len(args_w) > 1:
+            raise oefmt(space.w_TypeError,
+                "Cannot specify a default for %s() with multiple positional arguments",
+                implementation_of)
 
         w_iter = space.iter(w_sequence)
         w_type = space.type(w_iter)
@@ -170,7 +180,10 @@
                 w_max_item = w_item
                 w_max_val = w_compare_with
         if w_max_item is None:
-            raise oefmt(space.w_ValueError, "arg is an empty sequence")
+            if has_default:
+                w_max_item = w_default
+            else:
+                raise oefmt(space.w_ValueError, "arg is an empty sequence")
         return w_max_item
     if unroll:
         min_max_impl = jit.unroll_safe(min_max_impl)
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -590,6 +590,12 @@
         assert type(min(1, 1.0, 1)) is int
         assert type(min(1.0, 1, 1)) is float
         assert type(min(1, 1, 1.0)) is int
+        assert min([], default=-1) == -1
+        assert min([1, 2], default=-1) == 1
+        raises(TypeError, min, 0, 1, default=-1)
+        assert min([], default=None) == None
+        raises(TypeError, min, 1, default=0)
+        raises(TypeError, min, default=1)
 
     def test_max(self):
         assert max(1, 2) == 2
@@ -602,3 +608,9 @@
         assert type(max(1, 1.0, 1)) is int
         assert type(max(1.0, 1, 1)) is float
         assert type(max(1, 1, 1.0)) is int
+        assert max([], default=-1) == -1
+        assert max([1, 2], default=3) == 2
+        raises(TypeError, min, 0, 1, default=-1)
+        assert max([], default=None) == None
+        raises(TypeError, max, 1, default=0)
+        raises(TypeError, max, default=1)

From pypy.commits at gmail.com  Mon Sep 19 08:30:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 05:30:02 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: translation issues
Message-ID: <57dfda4a.48cf1c0a.c5e18.5064@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87225:0cc906e34289
Date: 2016-09-19 14:29 +0200
http://bitbucket.org/pypy/pypy/changeset/0cc906e34289/

Log:	translation issues

diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py
--- a/rpython/jit/backend/zarch/assembler.py
+++ b/rpython/jit/backend/zarch/assembler.py
@@ -896,7 +896,7 @@
         ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap')
         mc.LG(r.SCRATCH, l.addr(ofs, r.SPP))
 
-    def break_long_loop(self):
+    def break_long_loop(self, regalloc):
         # If the loop is too long, the guards in it will jump forward
         # more than 32 KB.  We use an approximate hack to know if we
         # should break the loop here with an unconditional "b" that
@@ -904,7 +904,7 @@
         jmp_pos = self.mc.currpos()
         self.mc.reserve_cond_jump()
 
-        self.write_pending_failure_recoveries()
+        self.write_pending_failure_recoveries(regalloc)
 
         currpos = self.mc.currpos()
         pmc = OverwritingBuilder(self.mc, jmp_pos, 1)
diff --git a/rpython/jit/backend/zarch/instruction_builder.py b/rpython/jit/backend/zarch/instruction_builder.py
--- a/rpython/jit/backend/zarch/instruction_builder.py
+++ b/rpython/jit/backend/zarch/instruction_builder.py
@@ -628,7 +628,7 @@
             return 0
         elif argtype == 'r' or argtype == 'r/m' or \
              argtype == 'f' or argtype == 'eo' or \
-             argtype == 'v':
+             argtype == 'v' or argtype == 'm':
             return arg.value
         elif argtype.startswith('i') or argtype.startswith('u') or argtype.startswith('h'):
             return arg.value
diff --git a/rpython/jit/backend/zarch/locations.py b/rpython/jit/backend/zarch/locations.py
--- a/rpython/jit/backend/zarch/locations.py
+++ b/rpython/jit/backend/zarch/locations.py
@@ -247,6 +247,7 @@
     from rpython.jit.backend.zarch.registers import JITFRAME_FIXED_SIZE
     return base_ofs + WORD * (position + JITFRAME_FIXED_SIZE)
 
+imm3 = imm(1)
 imm1 = imm(1)
 imm0 = imm(0)
 
@@ -254,16 +255,17 @@
 MASK_VEC_HWORD = 1
 MASK_VEC_WORD = 2
 MASK_VEC_DWORD = 3
+MASK_VEC_QWORD = 4
 
 def itemsize_to_mask(v):
     if v == 16:
-        return MASK_VEC_QWORD
+        return imm(MASK_VEC_QWORD)
     elif v == 8:
-        return MASK_VEC_DWORD
+        return imm(MASK_VEC_DWORD)
     elif v == 4:
-        return MASK_VEC_WORD
+        return imm(MASK_VEC_WORD)
     elif v == 2:
-        return MASK_VEC_HWORD
+        return imm(MASK_VEC_HWORD)
     elif v == 1:
-        return MASK_VEC_BYTE
+        return imm(MASK_VEC_BYTE)
     assert 0, "not supported itemsize to mask!"
diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py
--- a/rpython/jit/backend/zarch/regalloc.py
+++ b/rpython/jit/backend/zarch/regalloc.py
@@ -615,7 +615,7 @@
             self.fprm._check_invariants()
             self.vrm._check_invariants()
             if self.assembler.mc.get_relative_pos() > self.limit_loop_break:
-                self.assembler.break_long_loop()
+                self.assembler.break_long_loop(self)
                 self.limit_loop_break = (self.assembler.mc.get_relative_pos() +
                                              LIMIT_LOOP_BREAK)
             i += 1
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -21,6 +21,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.jit.codewriter import longlong
 from rpython.rlib.objectmodel import always_inline
+from rpython.jit.backend.zarch.arch import WORD
 
 def not_implemented(msg):
     msg = '[zarch/vector_ext] %s\n' % msg
@@ -30,7 +31,7 @@
 
 @always_inline
 def permi(v1,v2):
-    return (v1 << 2 | v2) & 0xf
+    return l.imm((v1 << 2 | v2) & 0xf)
 
 def flush_vec_cc(asm, regalloc, condition, size, resultloc):
     # After emitting an instruction that leaves a boolean result in
@@ -91,7 +92,7 @@
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
         if itemsize == 8:
-            self.mc.VFA(resloc, loc0, loc1, 3, 0, 0)
+            self.mc.VFA(resloc, loc0, loc1, l.imm(3), l.imm(0), l.imm(0))
             return
         not_implemented("vec_float_add of size %d" % itemsize)
 
@@ -99,7 +100,7 @@
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
         if itemsize == 8:
-            self.mc.VFS(resloc, loc0, loc1, 3, 0, 0)
+            self.mc.VFS(resloc, loc0, loc1, l.imm(3), l.imm(0), l.imm(0))
             return
         not_implemented("vec_float_sub of size %d" % itemsize)
 
@@ -107,7 +108,7 @@
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
         if itemsize == 8:
-            self.mc.VFM(resloc, loc0, loc1, 3, 0, 0)
+            self.mc.VFM(resloc, loc0, loc1, l.imm(3), l.imm(0), l.imm(0))
             return
         not_implemented("vec_float_mul of size %d" % itemsize)
 
@@ -115,7 +116,7 @@
         resloc, loc0, loc1, itemsize_loc = arglocs
         itemsize = itemsize_loc.value
         if itemsize == 8:
-            self.mc.VFD(resloc, loc0, loc1, 3, 0, 0)
+            self.mc.VFD(resloc, loc0, loc1, l.imm(3), l.imm(0), l.imm(0))
             return
         not_implemented("vec_float_truediv of size %d" % itemsize)
 
@@ -141,17 +142,17 @@
         resloc, argloc, sizeloc = arglocs
         size = sizeloc.value
         if size == 8:
-            self.mc.VFPSO(resloc, argloc, 3, 0, 2)
+            self.mc.VFPSO(resloc, argloc, l.imm(3), l.imm(0), l.imm(2))
             return
-        not_implemented("vec_float_abs of size %d" % itemsize)
+        not_implemented("vec_float_abs of size %d" % size)
 
     def emit_vec_float_neg(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
         size = sizeloc.value
         if size == 8:
-            self.mc.VFPSO(resloc, argloc, 3, 0, 0)
+            self.mc.VFPSO(resloc, argloc, l.imm(3), l.imm(0), l.imm(0))
             return
-        not_implemented("vec_float_abs of size %d" % itemsize)
+        not_implemented("vec_float_abs of size %d" % size)
 
     def emit_vec_guard_true(self, guard_op, arglocs, regalloc):
         self._emit_guard(guard_op, arglocs)
@@ -189,7 +190,7 @@
         size = sizeloc.value
         tmploc = regalloc.vrm.get_scratch_reg()
         self.mc.VX(tmploc, tmploc, tmploc) # all zero
-        self.mc.VCHL(resloc, argloc, tmploc, l.itemsize_to_mask(size), 0b0001)
+        self.mc.VCHL(resloc, argloc, tmploc, l.itemsize_to_mask(size), l.imm(0b0001))
         flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
 
     def emit_vec_float_eq(self, op, arglocs, regalloc):
@@ -198,7 +199,7 @@
         size = sizeloc.value
         if size == 8:
             # bit 3 in last argument sets the condition code
-            self.mc.VFCE(resloc, loc0, loc1, 3, 0, 1)
+            self.mc.VFCE(resloc, loc0, loc1, l.imm(3), l.imm(0), l.imm(1))
         else:
             not_implemented("[zarch/assembler] float == for size %d" % size)
         flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
@@ -213,7 +214,7 @@
         size = sizeloc.value
         if size == 8:
             # bit 3 in last argument sets the condition code
-            self.mc.VFCE(resloc, loc0, loc1, 3, 0, 1)
+            self.mc.VFCE(resloc, loc0, loc1, l.imm(3), l.imm(0), l.imm(1))
             self.mc.VNO(resloc, resloc, resloc)
         else:
             not_implemented("[zarch/assembler] float != for size %d" % size)
@@ -221,27 +222,27 @@
 
     def emit_vec_cast_int_to_float(self, op, arglocs, regalloc):
         resloc, loc0 = arglocs
-        self.mc.VCDG(resloc, loc0, 3, 4, m.RND_TOZERO.value)
+        self.mc.VCDG(resloc, loc0, l.imm(3), l.imm(4), m.RND_TOZERO)
 
     def emit_vec_int_eq(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, loc0, loc1, sizeloc = arglocs
         size = sizeloc.value
-        self.mc.VCEQ(resloc, loc0, loc1, l.itemsize_to_mask(size), 1)
+        self.mc.VCEQ(resloc, loc0, loc1, l.itemsize_to_mask(size), l.imm(1))
         flush_vec_cc(self, regalloc, c.VEQI, op.bytesize, resloc)
 
     def emit_vec_int_ne(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, loc0, loc1, sizeloc = arglocs
         size = sizeloc.value
-        self.mc.VCEQ(resloc, loc0, loc1, l.itemsize_to_mask(size), 1)
+        self.mc.VCEQ(resloc, loc0, loc1, l.itemsize_to_mask(size), l.imm(1))
         self.mc.VNO(resloc, resloc, resloc)
-        flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, res)
+        flush_vec_cc(self, regalloc, c.VNEI, op.bytesize, resloc)
 
     def emit_vec_cast_float_to_int(self, op, arglocs, regalloc):
         resloc, loc0 = arglocs
         # 4 => bit 1 from the MSB: XxC
-        self.mc.VCGD(resloc, loc0, 3, 4, m.RND_TOZERO.value)
+        self.mc.VCGD(resloc, loc0, l.imm(3), l.imm(4), m.RND_TOZERO)
 
     def emit_vec_expand_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
@@ -267,10 +268,10 @@
         if arg.type == FLOAT:
             self.mc.VPDI(targetloc, accumloc, accumloc, permi(1,0))
             if op == '+':
-                self.mc.VFA(targetloc, targetloc, accumloc, 3, 0b1000, 0)
+                self.mc.VFA(targetloc, targetloc, accumloc, l.imm3, l.imm(0b1000), l.imm(0))
                 return
             elif op == '*':
-                self.mc.VFM(targetloc, targetloc, accumloc, 3, 0b1000, 0)
+                self.mc.VFM(targetloc, targetloc, accumloc, l.imm3, l.imm(0b1000), l.imm(0))
                 return
         else:
             assert arg.type == INT
@@ -314,7 +315,7 @@
                 # load from sourceloc into GP reg and store back into resloc
                 self.mc.VLGV(r.SCRATCH, sourceloc, sindex, l.itemsize_to_mask(size))
                 rindex = l.addr(j + residx)
-                self.mc.VLVG(resloc, sourceloc, rindex, l.itemsize_to_mask(size))
+                self.mc.VLVG(resloc, r.SCRATCH, rindex, l.itemsize_to_mask(size))
 
     emit_vec_unpack_i = emit_vec_pack_i
 
@@ -510,14 +511,16 @@
         arg = op.getarg(0)
         if arg.is_vector():
             srcloc = self.ensure_vector_reg(arg)
+            assert isinstance(arg, VectorOp)
+            size = arg.bytesize
         else:
             # unpack
-            srcloc = self.ensure_reg(arg0)
+            srcloc = self.ensure_reg(arg)
+            size = WORD
         if op.is_vector():
             resloc = self.force_allocate_vector_reg(op)
         else:
             resloc = self.force_allocate_reg(op)
-        size = arg.bytesize
         return [resloc, srcloc, srcloc, imm(0), imm(index.value), imm(count.value), imm(size)]
 
     def prepare_vec_pack_f(self, op):
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -40,15 +40,18 @@
 
 def s390x_detect_vx():
     with open("/proc/cpuinfo", "rb") as fd:
-        lines = fd.read().splitlines()
-        for line in lines:
-            if line.startswith("features"):
-                colonidx = line.find(':')
-                split = line[colonidx+1:].strip().split(' ')
-                if 'vx' in split:
-                    return True
-                break
-
+        content = fd.read()
+        start = content.find("features", 0)
+        if start >= 0:
+            after_colon = content.find(":", start)
+            if after_colon < 0:
+                return False
+            newline = content.find("\n", after_colon)
+            if newline < 0:
+                return False
+            split = content[after_colon+1:newline].strip().split(' ')
+            if 'vx' in split:
+                return True
     return False
 
 def s390x_cpu_revision():

From pypy.commits at gmail.com  Mon Sep 19 08:44:23 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 05:44:23 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: same patch for x86
Message-ID: <57dfdda7.29a9c20a.149f9.bffc@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87227:93b3c662735a
Date: 2016-09-19 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/93b3c662735a/

Log:	same patch for x86

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -16,6 +16,7 @@
 from rpython.rtyper import rclass
 from rpython.rlib.jit import AsmInfo
 from rpython.jit.backend.model import CompiledLoopToken
+from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
 from rpython.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs,
     gpr_reg_mgr_cls, xmm_reg_mgr_cls)
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
@@ -682,17 +683,25 @@
                 bridge_accum_info = bridge_accum_info.next()
             guard_accum_info = guard_accum_info.next()
 
-        # register mapping is most likely NOT valid, thus remap it in this
-        # short piece of assembler
+        # register mapping is most likely NOT valid, thus remap it
+        src_locations1 = []
+        dst_locations1 = []
+        src_locations2 = []
+        dst_locations2 = []
+
+        # Build the four lists
         assert len(guard_locs) == len(bridge_locs)
-        for i,gloc in enumerate(guard_locs):
-            bloc = bridge_locs[i]
-            bstack = bloc.location_code() == 'b'
-            gstack = gloc.location_code() == 'b'
-            if bstack and gstack:
-                pass
-            elif gloc is not bloc:
-                self.mov(gloc, bloc)
+        for i,src_loc in enumerate(guard_locs):
+            dst_loc = bridge_locs[i]
+            if not src_loc.is_fp_reg():
+                src_locations1.append(src_loc)
+                dst_locations1.append(dst_loc)
+            else:
+                src_locations2.append(src_loc)
+                dst_locations2.append(dst_loc)
+        remap_frame_layout_mixed(self, src_locations1, dst_locations1, r.SCRATCH,
+                                 src_locations2, dst_locations2, r.FP_SCRATCH)
+
         offset = self.mc.get_relative_pos()
         self.mc.JMP_l(0)
         self.mc.writeimm32(0)

From pypy.commits at gmail.com  Mon Sep 19 09:31:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 06:31:02 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: use x86 scratch regs instead
Message-ID: <57dfe896.262ec20a.d8afc.e315@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87228:d93cde0eb27e
Date: 2016-09-19 15:21 +0200
http://bitbucket.org/pypy/pypy/changeset/d93cde0eb27e/

Log:	use x86 scratch regs instead

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -699,8 +699,8 @@
             else:
                 src_locations2.append(src_loc)
                 dst_locations2.append(dst_loc)
-        remap_frame_layout_mixed(self, src_locations1, dst_locations1, r.SCRATCH,
-                                 src_locations2, dst_locations2, r.FP_SCRATCH)
+        remap_frame_layout_mixed(self, src_locations1, dst_locations1, X86_64_SCRATCH_REG,
+                                 src_locations2, dst_locations2, X86_64_XMM_SCRATCH_REG)
 
         offset = self.mc.get_relative_pos()
         self.mc.JMP_l(0)
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -39,8 +39,14 @@
     return ids
 
 def s390x_detect_vx():
+    contentlist = []
     with open("/proc/cpuinfo", "rb") as fd:
-        content = fd.read()
+        while True:
+            snippet = fd.read(4096)
+            if not snippet:
+                break
+            contentlist.append(snippet)
+        content = ''.join(contentlist)
         start = content.find("features", 0)
         if start >= 0:
             after_colon = content.find(":", start)

From pypy.commits at gmail.com  Mon Sep 19 09:31:05 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 19 Sep 2016 06:31:05 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: use detect_vsx in ppc,
 not detect_simd_z
Message-ID: <57dfe899.8759c20a.1522a.db3e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87229:a19724e734a3
Date: 2016-09-19 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/a19724e734a3/

Log:	use detect_vsx in ppc, not detect_simd_z

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -20,7 +20,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.jit.codewriter import longlong
-from rpython.jit.backend.zarch.detect_feature import detect_simd_z
+from rpython.jit.backend.ppc.detect_feature import detect_vsx
 from rpython.rlib.objectmodel import always_inline
 
 def not_implemented(msg):
@@ -85,7 +85,7 @@
 
 class AltiVectorExt(VectorExt):
     def setup_once(self, asm):
-        if detect_simd_z():
+        if detect_vsx():
             self.enable(16, accum=True)
             asm.setup_once_vector()
         self._setup = True
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -41,12 +41,7 @@
 def s390x_detect_vx():
     contentlist = []
     with open("/proc/cpuinfo", "rb") as fd:
-        while True:
-            snippet = fd.read(4096)
-            if not snippet:
-                break
-            contentlist.append(snippet)
-        content = ''.join(contentlist)
+        content = fd.read()
         start = content.find("features", 0)
         if start >= 0:
             after_colon = content.find(":", start)

From pypy.commits at gmail.com  Mon Sep 19 09:40:00 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 19 Sep 2016 06:40:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: backout bd9d1dd19c2c, per a8a4ffe56b74
Message-ID: <57dfeab0.07d01c0a.509d2.4d74@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87230:5c701a472a00
Date: 2016-09-19 14:39 +0100
http://bitbucket.org/pypy/pypy/changeset/5c701a472a00/

Log:	backout bd9d1dd19c2c, per a8a4ffe56b74

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -78,7 +78,6 @@
         return self._value
 
     @jit.elidable
-    @jit.call_shortcut
     def identifier_w(self, space):
         identifier = self._utf8
         if identifier is not None:

From pypy.commits at gmail.com  Mon Sep 19 12:34:53 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 19 Sep 2016 09:34:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: revert 2b244ba62cf4,
 redo the distutils part to be cpython compliant (no SOABI)
Message-ID: <57e013ad.09afc20a.54c0b.2737@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87233:71a05328d60c
Date: 2016-09-19 18:08 +0300
http://bitbucket.org/pypy/pypy/changeset/71a05328d60c/

Log:	revert 2b244ba62cf4, redo the distutils part to be cpython compliant
	(no SOABI)

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -63,8 +63,7 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
-    g['SO'] = g['SOABI'][0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -76,8 +75,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
-    g['SO'] = g['SOABI'][0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -526,7 +526,10 @@
 
         # PyPy:
         import imp
-        _CONFIG_VARS['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
+        for suffix, mode, type_ in imp.get_suffixes():
+            if type_ == imp.C_EXTENSION:
+                _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
+                break        
 
     if args:
         vals = []

From pypy.commits at gmail.com  Mon Sep 19 15:55:47 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 19 Sep 2016 12:55:47 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: a passing test
Message-ID: <57e042c3.c4f6c20a.807a3.723c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87234:5c15154ee248
Date: 2016-09-19 20:03 +0200
http://bitbucket.org/pypy/pypy/changeset/5c15154ee248/

Log:	a passing test

diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -335,6 +335,25 @@
             return res
         self.check(f, [int], getfield=0)
 
+    def test_loopinvariant_heap_merge_not_possible(self):
+        class A(object):
+            pass
+        def f(i):
+            res = 0
+            x = i
+            a = A()
+            if i == 0:
+                a.x = 1
+            else:
+                a.x = i
+            while x:
+                x -= 1
+                res += a.x
+                if x % 1000 == 1:
+                    a.x = 5
+            return res
+        self.check(f, [int], getfield=1)
+
     def test_direct_merge(self):
         def f(i):
             a = i + 1

From pypy.commits at gmail.com  Mon Sep 19 15:55:49 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 19 Sep 2016 12:55:49 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: loop_blocks was broken in
 complex cases
Message-ID: <57e042c5.2472c20a.2f30e.6de3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87235:c701f3c76e7c
Date: 2016-09-19 21:53 +0200
http://bitbucket.org/pypy/pypy/changeset/c701f3c76e7c/

Log:	loop_blocks was broken in complex cases

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -304,7 +304,46 @@
         result._clear_heapcache_for_loop_blocks(loop_blocks)
     return result
 
-def loop_blocks(graph):
+def compute_reachability_no_backedges(graph, backedges):
+    reachable = {}
+    blocks = list(graph.iterblocks())
+    # Reversed order should make the reuse path more likely.
+    for block in reversed(blocks):
+        reach = set()
+        scheduled = [block]
+        while scheduled:
+            current = scheduled.pop()
+            for link in current.exits:
+                if link in backedges:
+                    continue
+                if link.target in reachable:
+                    reach.add(link.target)
+                    reach = reach | reachable[link.target]
+                    continue
+                if link.target not in reach:
+                    reach.add(link.target)
+                    scheduled.append(link.target)
+        reachable[block] = reach
+    return reachable
+
+def loop_blocks(graph, backedges, entrymap):
+    reachable_no_backedges = compute_reachability_no_backedges(graph, backedges)
+    reachable = support.compute_reachability(graph)
+    result = {}
+    for block in graph.iterblocks():
+        entering_backedges = [link for link in entrymap[block]
+                if link in backedges]
+        if not entering_backedges:
+            # no backedge entries
+            continue
+        loop_blocks = {block}
+        for target in reachable_no_backedges[block]:
+            if any(link.prevblock in reachable[target]
+                       for link in entering_backedges):
+                loop_blocks.add(target)
+        result[block] = loop_blocks
+    return result
+
     loop_blocks = support.find_loop_blocks(graph)
     result = {}
     for loop_block, start in loop_blocks.iteritems():
@@ -319,8 +358,8 @@
     def transform(self, graph):
         variable_families = ssa.DataFlowFamilyBuilder(graph).get_variable_families()
         entrymap = mkentrymap(graph)
-        loops = loop_blocks(graph)
         backedges = support.find_backedges(graph)
+        loops = loop_blocks(graph, backedges, entrymap)
         todo = collections.deque([graph.startblock])
         caches_to_merge = collections.defaultdict(list)
         done = set()
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -1,6 +1,6 @@
 import pytest
 from rpython.translator.translator import TranslationContext, graphof
-from rpython.translator.backendopt.cse import CSE, Cache
+from rpython.translator.backendopt.cse import CSE, Cache, loop_blocks
 from rpython.flowspace.model import Variable, Constant
 from rpython.translator.backendopt import removenoops
 from rpython.flowspace.model import checkgraph, summary
@@ -354,6 +354,28 @@
             return res
         self.check(f, [int], getfield=1)
 
+    def test_loopinvariant_heap_merge_nested(self):
+        class A(object):
+            pass
+        def f(i):
+            res = 0
+            y = 0
+            while y < i:
+                y += 1
+                x = i
+                a = A()
+                if i == 0:
+                    a.x = 1
+                else:
+                    a.x = i
+                while x:
+                    x -= 1
+                    res += a.x
+                    if x % 1000 == 1:
+                        a.x = 5
+            return res
+        self.check(f, [int], getfield=1)
+
     def test_direct_merge(self):
         def f(i):
             a = i + 1
@@ -567,3 +589,45 @@
     assert not needs_adding
     assert res is v2
 
+
+def test_loop_blocks():
+    from rpython.flowspace.model import mkentrymap
+    from rpython.translator.backendopt import support
+    class A(object): pass
+    def f(i):
+        res = 0
+        y = 0
+        while y < i: # block1
+            y += 1 # block2
+            x = i
+            a = A()
+            if i == 0:
+                a.x = 1 # block3
+            else:
+                a.x = i # block4
+            while x: # block5
+                x -= 1 # block6
+                res += a.x
+                if x % 1000 == 1:
+                    a.x = 5 # block7
+        return res # block8
+
+    t = TranslationContext()
+    t.buildannotator().build_types(f, [int])
+    t.buildrtyper().specialize()
+    graph = graphof(t, f)
+    startblock = graph.startblock
+    block1 = startblock.exits[0].target
+    block2 = block1.exits[1].target
+    block3 = block2.exits[0].target
+    block4 = block2.exits[1].target
+    block5 = block4.exits[0].target
+    block6 = block5.exits[1].target
+    block7 = block6.exits[1].target
+
+    loops = loop_blocks(graph, support.find_backedges(graph),
+                        mkentrymap(graph))
+    assert loops == {
+        block1: {block1, block2, block3, block4, block5, block6, block7},
+        block5: {block5, block6, block7},
+    }

From pypy.commits at gmail.com  Mon Sep 19 15:55:51 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 19 Sep 2016 12:55:51 -0700 (PDT)
Subject: [pypy-commit] pypy better-storesink: merge
Message-ID: <57e042c7.8759c20a.1522a.7115@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r87236:ce6f32acf15c
Date: 2016-09-19 21:54 +0200
http://bitbucket.org/pypy/pypy/changeset/ce6f32acf15c/

Log:	merge

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,8 @@
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -63,7 +63,7 @@
         """x.__iter__() <==> iter(x)"""
         return self.space.wrap(self)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -264,25 +264,22 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
-            else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
-            #
             try:
+                if operr is not None:
+                    ec = self.space.getexecutioncontext()
+                    next_instr = self.handle_operation_error(ec, operr)
+                    self.last_instr = intmask(next_instr - 1)
+                else:
+                    # Execution starts just after the last_instr.  Initially,
+                    # last_instr is -1.  After a generator suspends it points to
+                    # the YIELD_VALUE instruction.
+                    next_instr = r_uint(self.last_instr + 1)
+                    if next_instr != 0:
+                        self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
-            executioncontext.return_trace(self, w_exitvalue)
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,12 +57,14 @@
         def f():
             yield 2
         g = f()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
 
     def test_throw3(self):
@@ -221,7 +223,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -562,3 +562,21 @@
         res = f(10).g()
         sys.settrace(None)
         assert res == 10
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,16 +30,19 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
+    w_initializer_type = None
+    w_initializer = None
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if space.type(w_initializer) is space.w_str:
+            if w_initializer is not None:
+                if w_initializer_type is space.w_str:
                     a.descr_fromstring(space, w_initializer)
-                elif space.type(w_initializer) is space.w_list:
+                elif w_initializer_type is space.w_list:
                     a.descr_fromlist(space, w_initializer)
                 else:
                     a.extend(w_initializer, True)
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import widen
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -34,7 +34,7 @@
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
@@ -203,7 +203,7 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    return space.newlong(rffi.cast(ADDR, p))
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -8,18 +8,20 @@
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
-        value = api.PyLong_FromLong(3)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == 3
+        w_value = api.PyLong_FromLong(3)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == 3
 
-        value = api.PyLong_FromLong(sys.maxint)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == sys.maxint
+        w_value = api.PyLong_FromLong(sys.maxint)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == sys.maxint
 
     def test_aslong(self, space, api):
         w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+        assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsLong(w_value)
         assert value == (sys.maxint - 1)
 
@@ -35,12 +37,16 @@
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsSsize_t(w_value)
         assert value == 2
-        assert space.eq_w(w_value, api.PyLong_FromSsize_t(2))
+        w_val2 = api.PyLong_FromSsize_t(2)
+        assert isinstance(w_val2, W_LongObject)
+        assert space.eq_w(w_value, w_val2)
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
+        assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -103,6 +109,7 @@
 
     def test_as_voidptr(self, space, api):
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
+        assert isinstance(w_l, W_LongObject)
         assert space.unwrap(w_l) == 0L
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
 
@@ -128,23 +135,58 @@
         module = self.import_extension('foo', [
             ("from_unsignedlong", "METH_NOARGS",
              """
-                 return PyLong_FromUnsignedLong((unsigned long)-1);
+                 PyObject * obj;
+                 obj = PyLong_FromUnsignedLong((unsigned long)-1);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
         import sys
         assert module.from_unsignedlong() == 2 * sys.maxint + 1
 
     def test_fromlonglong(self):
         module = self.import_extension('foo', [
-            ("from_longlong", "METH_NOARGS",
+            ("from_longlong", "METH_VARARGS",
              """
-                 return PyLong_FromLongLong((long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """),
-            ("from_unsignedlonglong", "METH_NOARGS",
+            ("from_unsignedlonglong", "METH_VARARGS",
              """
-                 return PyLong_FromUnsignedLongLong((unsigned long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromUnsignedLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
-        assert module.from_longlong() == -1
-        assert module.from_unsignedlonglong() == (1<<64) - 1
+        assert module.from_longlong(-1) == -1
+        assert module.from_longlong(0) == 0
+        assert module.from_unsignedlonglong(0) == 0
+        assert module.from_unsignedlonglong(-1) == (1<<64) - 1
 
     def test_from_size_t(self):
         module = self.import_extension('foo', [
@@ -232,10 +274,15 @@
             ("has_sub", "METH_NOARGS",
              """
                 PyObject *ret, *obj = PyLong_FromLong(42);
-                if (obj->ob_type->tp_as_number->nb_subtract)
-                    ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                if (obj->ob_type != &PyLong_Type)
+                    ret = PyLong_FromLong(-2);
                 else
-                    ret = PyLong_FromLong(-1);
+                {
+                    if (obj->ob_type->tp_as_number->nb_subtract)
+                        ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                    else
+                        ret = PyLong_FromLong(-1);
+                }
                 Py_DECREF(obj);
                 return ret;
              """),
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -253,9 +253,28 @@
     from rpython.rtyper.lltypesystem import lltype, rffi
     return space.wrap(rffi.cast(lltype.Signed, handle))
 
+getsizeof_missing = """sys.getsizeof() is not implemented on PyPy.
+
+A memory profiler using this function is most likely to give results
+inconsistent with reality on PyPy.  It would be possible to have
+sys.getsizeof() return a number (with enough work), but that may or
+may not represent how much memory the object uses.  It doesn't even
+make really sense to ask how much *one* object uses, in isolation
+with the rest of the system.  For example, instances have maps,
+which are often shared across many instances; in this case the maps
+would probably be ignored by an implementation of sys.getsizeof(),
+but their overhead is important in some cases if they are many
+instances with unique maps.  Conversely, equal strings may share
+their internal string data even if they are different objects---or
+empty containers may share parts of their internals as long as they
+are empty.  Even stranger, some lists create objects as you read
+them; if you try to estimate the size in memory of range(10**6) as
+the sum of all items' size, that operation will by itself create one
+million integer objects that never existed in the first place.
+"""
+
 def getsizeof(space, w_object, w_default=None):
-    """Not implemented on PyPy."""
     if w_default is None:
-        raise oefmt(space.w_TypeError,
-                    "sys.getsizeof() not implemented on PyPy")
+        raise oefmt(space.w_TypeError, getsizeof_missing)
     return w_default
+getsizeof.__doc__ = getsizeof_missing
diff --git a/rpython/translator/backendopt/merge_if_blocks.py b/rpython/translator/backendopt/merge_if_blocks.py
--- a/rpython/translator/backendopt/merge_if_blocks.py
+++ b/rpython/translator/backendopt/merge_if_blocks.py
@@ -20,6 +20,14 @@
         return False
     if isinstance(op.args[0], Constant) and isinstance(op.args[1], Constant):
         return False
+    # check that the constant is hashable (ie not a symbolic)
+    try:
+        if isinstance(op.args[0], Constant):
+            hash(op.args[0].value)
+        else:
+            hash(op.args[1].value)
+    except TypeError:
+        return False
     return True
 
 def merge_chain(chain, checkvar, varmap, graph):
diff --git a/rpython/translator/backendopt/test/test_merge_if_blocks.py b/rpython/translator/backendopt/test/test_merge_if_blocks.py
--- a/rpython/translator/backendopt/test/test_merge_if_blocks.py
+++ b/rpython/translator/backendopt/test/test_merge_if_blocks.py
@@ -2,11 +2,12 @@
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator.backendopt.all import backend_optimizations
 from rpython.translator.translator import TranslationContext, graphof as tgraphof
-from rpython.flowspace.model import Block
+from rpython.flowspace.model import Block, checkgraph
 from rpython.translator.backendopt.removenoops import remove_same_as
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, r_int
 from rpython.annotator.model import SomeChar, SomeUnicodeCodePoint
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 
 def do_test_merge(fn, testvalues):
     t = TranslationContext()
@@ -225,3 +226,29 @@
     malloc.remove_mallocs(t, t.graphs)
     from rpython.translator import simplify
     simplify.join_blocks(graph)
+
+def test_switch_on_symbolic():
+    symb1 = CDefinedIntSymbolic("1", 1)
+    symb2 = CDefinedIntSymbolic("2", 2)
+    symb3 = CDefinedIntSymbolic("3", 3)
+    def fn(x):
+        res = 0
+        if x == symb1:
+            res += x + 1
+        elif x == symb2:
+            res += x + 2
+        elif x == symb3:
+            res += x + 3
+        res += 1
+        return res
+    t = TranslationContext()
+    a = t.buildannotator()
+    a.build_types(fn, [int])
+    rtyper = t.buildrtyper()
+    rtyper.specialize()
+    graph = t.graphs[0]
+    remove_same_as(graph)
+    res = merge_if_blocks_once(graph)
+    assert not res
+    checkgraph(graph)
+

From pypy.commits at gmail.com  Mon Sep 19 16:07:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Sep 2016 13:07:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: untabify
Message-ID: <57e0459c.0aa71c0a.85b04.34c3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87237:434221ac191c
Date: 2016-09-19 22:05 +0200
http://bitbucket.org/pypy/pypy/changeset/434221ac191c/

Log:	untabify

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -136,7 +136,7 @@
                         "%s() expects at least one argument",
                         implementation_of)
         w_key = None
-	has_default = False
+        has_default = False
         if any_kwds:
             kwds = args.keywords
             for n in range(len(kwds)):

From pypy.commits at gmail.com  Mon Sep 19 16:08:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Sep 2016 13:08:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Test that loops automatically unroll if
 they are written anywhere in the
Message-ID: <57e045a0.09afc20a.54c0b.79fe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87238:3ebb2e0f8d2b
Date: 2016-09-19 22:07 +0200
http://bitbucket.org/pypy/pypy/changeset/3ebb2e0f8d2b/

Log:	Test that loops automatically unroll if they are written anywhere in
	the same function as one that contains a jit_merge_point()

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4558,3 +4558,20 @@
         self.meta_interp(f, [])
         self.check_resops(guard_nonnull=0)
 
+    def test_loop_before_main_loop(self):
+        fdriver = JitDriver(greens=[], reds='auto')
+        gdriver = JitDriver(greens=[], reds='auto')
+        def f(i, j):
+            while j > 0:   # this loop unrolls because it is in the same
+                j -= 1     # function as a jit_merge_point()
+            while i > 0:
+                fdriver.jit_merge_point()
+                i -= 1
+        def g(i, j, k):
+            while k > 0:
+                gdriver.jit_merge_point()
+                f(i, j)
+                k -= 1
+
+        self.meta_interp(g, [5, 5, 5])
+        self.check_resops(guard_true=10)   # 5 unrolled, plus 5 unrelated

From pypy.commits at gmail.com  Tue Sep 20 03:11:33 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 20 Sep 2016 00:11:33 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: use os.open instead of open()
 to read from /proc files at runtime
Message-ID: <57e0e125.0f1f1c0a.a894.bb26@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87239:d0e4fba9c670
Date: 2016-09-20 09:06 +0200
http://bitbucket.org/pypy/pypy/changeset/d0e4fba9c670/

Log:	use os.open instead of open() to read from /proc files at runtime

diff --git a/rpython/jit/backend/ppc/detect_feature.py b/rpython/jit/backend/ppc/detect_feature.py
--- a/rpython/jit/backend/ppc/detect_feature.py
+++ b/rpython/jit/backend/ppc/detect_feature.py
@@ -1,3 +1,4 @@
+import os
 import sys
 import struct
 import platform
@@ -13,19 +14,25 @@
 SYSTEM = platform.system()
 
 def detect_vsx_linux():
-    with open('/proc/self/auxv', 'rb') as fd:
-        while True:
-            buf = fd.read(8)
-            buf2 = fd.read(8)
-            if not buf or not buf2:
-                break
-            key = runpack("L", buf)
-            value = runpack("L", buf2)
-            if key == AT_HWCAP:
-                if value & PPC_FEATURE_HAS_ALTIVEC:
-                    return True
-            if key == AT_NULL:
-                return False
+    try:
+        fd = os.open("/proc/self/auxv", os.O_RDONLY, 0644)
+        try:
+            while True:
+                buf = os.read(fd, 8)
+                buf2 = os.read(fd, 8)
+                if not buf or not buf2:
+                    break
+                key = runpack("L", buf)
+                value = runpack("L", buf2)
+                if key == AT_HWCAP:
+                    if value & PPC_FEATURE_HAS_ALTIVEC:
+                        return True
+                if key == AT_NULL:
+                    return False
+        finally:
+            os.close(fd)
+    except OSError:
+        pass
     return False
 
 def detect_vsx():
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -693,7 +693,7 @@
         assert len(guard_locs) == len(bridge_locs)
         for i,src_loc in enumerate(guard_locs):
             dst_loc = bridge_locs[i]
-            if not src_loc.is_fp_reg():
+            if not src_loc.is_float():
                 src_locations1.append(src_loc)
                 dst_locations1.append(dst_loc)
             else:
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -128,7 +128,7 @@
             self.delay_emit(node)
             return
         # emit a now!
-        self.pre_emit(node)
+        self.pre_emit(node, True)
         self.mark_emitted(node)
         if not node.is_imaginary():
             op = node.getoperation()
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -1,4 +1,5 @@
 import re
+import os
 
 def extract_s390x_cpu_ids(lines):
     ids = []
@@ -39,20 +40,31 @@
     return ids
 
 def s390x_detect_vx():
-    contentlist = []
-    with open("/proc/cpuinfo", "rb") as fd:
-        content = fd.read()
-        start = content.find("features", 0)
-        if start >= 0:
-            after_colon = content.find(":", start)
-            if after_colon < 0:
-                return False
-            newline = content.find("\n", after_colon)
-            if newline < 0:
-                return False
-            split = content[after_colon+1:newline].strip().split(' ')
-            if 'vx' in split:
-                return True
+    chunks = []
+    try:
+        fd = os.open("/proc/self/auxv", os.O_RDONLY, 0644)
+        try:
+            while True:
+                chunk = os.read(fd, 4096)
+                if not chunk:
+                    break
+                chunks.append(chunk)
+        finally:
+            os.close(fd)
+    except OSError:
+        pass
+    content = ''.join(chunks)
+    start = content.find("features", 0)
+    if start >= 0:
+        after_colon = content.find(":", start)
+        if after_colon < 0:
+            return False
+        newline = content.find("\n", after_colon)
+        if newline < 0:
+            return False
+        split = content[after_colon+1:newline].strip().split(' ')
+        if 'vx' in split:
+            return True
     return False
 
 def s390x_cpu_revision():

From pypy.commits at gmail.com  Tue Sep 20 06:09:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 20 Sep 2016 03:09:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Issue #2400: Rename the interp-level
 module to '_operator'
Message-ID: <57e10ac8.c186c20a.ff68f.5590@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87240:6524139cfc98
Date: 2016-09-20 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/6524139cfc98/

Log:	Issue #2400: Rename the interp-level module to '_operator'

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -24,7 +24,7 @@
 default_modules = essential_modules.copy()
 default_modules.update([
     "_codecs", "atexit", "gc", "_weakref", "marshal", "errno", "imp",
-    "itertools", "math", "cmath", "_sre", "_pickle_support", "operator",
+    "itertools", "math", "cmath", "_sre", "_pickle_support", "_operator",
     "parser", "symbol", "token", "_ast", "_random", "__pypy__",
     "_string", "_testing", "time"
 ])
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -890,8 +890,8 @@
     import runpy
     if 'time' not in sys.builtin_module_names:
         import time; del time
-    if 'operator' not in sys.builtin_module_names:
-        import operator; del operator
+    if '_operator' not in sys.builtin_module_names:
+        import _operator; del _operator
 
     # no one should change to which lists sys.argv and sys.path are bound
     old_argv = sys.argv
diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -2,6 +2,7 @@
 
 class Module(MixedModule):
     """Operator Builtin Module. """
+    applevel_name = '_operator'
 
     # HACK! override loaders to be able to access different operations
     # under same name. I.e., operator.eq == operator.__eq__
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -1,12 +1,14 @@
 # -*- coding: utf-8 -*-
 
 class AppTestOperator:
+    spaceconfig = dict(usemodules=['operator'])
+
     def test_equality(self):
-        import operator
+        import _operator as operator
         assert operator.eq == operator.__eq__
 
     def test_getters_are_not_regular_functions(self):
-        import operator
+        import _operator as operator
         class A(object):
             getx = operator.attrgetter('x')
             get3 = operator.itemgetter(3)
@@ -21,7 +23,7 @@
         assert l == ["x"]
 
     def test_getter_multiple_gest(self):
-        import operator
+        import _operator as operator
 
         class A(object):
             pass
@@ -40,11 +42,11 @@
         raises(TypeError, operator.itemgetter(2, 'x', 5), data)
 
     def test_attrgetter(self):
-        import operator
+        import _operator as operator
         raises(TypeError, operator.attrgetter, 2)
 
     def test_dotted_attrgetter(self):
-        from operator import attrgetter
+        from _operator import attrgetter
         class A:
             pass
         a = A()
@@ -56,7 +58,7 @@
         assert attrgetter("child.name", "child.foo")(a) == ("world", "bar")
 
     def test_attrgetter_type(self):
-        from operator import attrgetter
+        from _operator import attrgetter
         assert type(attrgetter("child.name")) is attrgetter
 
     def test_concat(self):
@@ -88,7 +90,7 @@
             def __rmul__(self, other):
                 return other * self.lst
 
-        import operator
+        import _operator as operator
 
         raises(TypeError, operator.concat)
         raises(TypeError, operator.concat, None, None)
@@ -127,7 +129,7 @@
             def __rmul__(self, other):
                 return other * self.lst
 
-        import operator
+        import _operator as operator
 
         a = list(range(3))
         raises(TypeError, operator.mul)
@@ -153,14 +155,14 @@
         assert operator.mul(a, 0) == []
 
     def test_iadd(self):
-        import operator
+        import _operator as operator
 
         list = []
         assert operator.iadd(list, [1, 2]) is list
         assert list == [1, 2]
 
     def test_imul(self):
-        import operator
+        import _operator as operator
 
         class X(object):
             def __index__(self):
@@ -176,7 +178,7 @@
         assert a == [0, 1, 2, 0, 1, 2]
 
     def test_methodcaller(self):
-        from operator import methodcaller
+        from _operator import methodcaller
         class X(object):
             def method(self, arg1=2, arg2=3):
                 return arg1, arg2
@@ -187,7 +189,7 @@
         assert methodcaller("method", 4, arg2=42)(x) == (4, 42)
 
     def test_index(self):
-        import operator
+        import _operator as operator
         assert operator.index(42) == 42
         assert operator.__index__(42) == 42
         raises(TypeError, operator.index, "abc")
@@ -195,14 +197,14 @@
         assert str(exc.value) == "'str' object cannot be interpreted as an integer"
 
     def test_indexOf(self):
-        import operator
+        import _operator as operator
         raises(TypeError, operator.indexOf)
         raises(TypeError, operator.indexOf, None, None)
         assert operator.indexOf([4, 3, 2, 1], 3) == 1
         raises(ValueError, operator.indexOf, [4, 3, 2, 1], 0)
 
     def test_compare_digest(self):
-        import operator
+        import _operator as operator
 
         # Testing input type exception handling
         a, b = 100, 200
@@ -309,7 +311,7 @@
         assert not operator._compare_digest(a, b)
 
     def test_compare_digest_unicode(self):
-        import operator
+        import _operator as operator
         assert operator._compare_digest(u'asd', u'asd')
         assert not operator._compare_digest(u'asd', u'qwe')
         raises(TypeError, operator._compare_digest, u'asd', b'qwe')

From pypy.commits at gmail.com  Tue Sep 20 06:22:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 20 Sep 2016 03:22:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Merged in remarkablerocket/pypy/py3.5
 (pull request #480)
Message-ID: <57e10e01.c62f1c0a.e832a.0add@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.5
Changeset: r87243:5f9c690d2444
Date: 2016-09-20 12:22 +0200
http://bitbucket.org/pypy/pypy/changeset/5f9c690d2444/

Log:	Merged in remarkablerocket/pypy/py3.5 (pull request #480)

	 Implement round ndigits=None behaviour as per CPython 3.5 docs

diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -100,13 +100,13 @@
     """round(number[, ndigits]) -> number
 
 Round a number to a given precision in decimal digits (default 0 digits).
-This returns an int when called with one argument, otherwise the
-same type as the number. ndigits may be negative."""
+This returns an int when called with one argument or if ndigits=None,
+otherwise the same type as the number. ndigits may be negative."""
     round = space.lookup(w_number, '__round__')
     if round is None:
         raise oefmt(space.w_TypeError,
                     "type %T doesn't define __round__ method", w_number)
-    if w_ndigits is None:
+    if space.is_none(w_ndigits):
         return space.get_and_call_function(round, w_number)
     else:
         return space.get_and_call_function(round, w_number, w_ndigits)
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -814,8 +814,9 @@
         raises(TypeError, round, t)
         raises(TypeError, round, t, 0)
 
-        raises(TypeError, round, 3, None)
-        raises(TypeError, round, 3.0, None)
+        assert round(3, ndigits=None) == 3
+        assert round(3.0, ndigits=None) == 3
+        assert type(round(3.0, ndigits=None)) is int
 
     def test_vars_obscure_case(self):
         class C_get_vars(object):

From pypy.commits at gmail.com  Tue Sep 20 06:23:06 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Tue, 20 Sep 2016 03:23:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement round ndigits=None behaviour as
 per CPython 3.5 docs
Message-ID: <57e10e0a.c4f6c20a.807a3.5d0d@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87241:02d30675260e
Date: 2016-09-19 16:12 +0100
http://bitbucket.org/pypy/pypy/changeset/02d30675260e/

Log:	Implement round ndigits=None behaviour as per CPython 3.5 docs

diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -100,13 +100,13 @@
     """round(number[, ndigits]) -> number
 
 Round a number to a given precision in decimal digits (default 0 digits).
-This returns an int when called with one argument, otherwise the
-same type as the number. ndigits may be negative."""
+This returns an int when called with one argument or if ndigits=None,
+otherwise the same type as the number. ndigits may be negative."""
     round = space.lookup(w_number, '__round__')
     if round is None:
         raise oefmt(space.w_TypeError,
                     "type %T doesn't define __round__ method", w_number)
-    if w_ndigits is None:
+    if w_ndigits is None or w_ndigits is space.w_None:
         return space.get_and_call_function(round, w_number)
     else:
         return space.get_and_call_function(round, w_number, w_ndigits)
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -814,8 +814,9 @@
         raises(TypeError, round, t)
         raises(TypeError, round, t, 0)
 
-        raises(TypeError, round, 3, None)
-        raises(TypeError, round, 3.0, None)
+        assert round(3, ndigits=None) == 3
+        assert round(3.0, ndigits=None) == 3
+        assert type(round(3.0, ndigits=None)) is int
 
     def test_vars_obscure_case(self):
         class C_get_vars(object):

From pypy.commits at gmail.com  Tue Sep 20 06:23:08 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Tue, 20 Sep 2016 03:23:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Replace checks for None and space.w_None
 with space.is_none
Message-ID: <57e10e0c.2472c20a.2f30e.5739@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87242:3c6fe0885f72
Date: 2016-09-19 17:20 +0100
http://bitbucket.org/pypy/pypy/changeset/3c6fe0885f72/

Log:	Replace checks for None and space.w_None with space.is_none

diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -106,7 +106,7 @@
     if round is None:
         raise oefmt(space.w_TypeError,
                     "type %T doesn't define __round__ method", w_number)
-    if w_ndigits is None or w_ndigits is space.w_None:
+    if space.is_none(w_ndigits):
         return space.get_and_call_function(round, w_number)
     else:
         return space.get_and_call_function(round, w_number, w_ndigits)

From pypy.commits at gmail.com  Tue Sep 20 07:05:29 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 20 Sep 2016 04:05:29 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: apply translation fixes for ppc
Message-ID: <57e117f9.4398c20a.ed3b3.7a3f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87244:8dcfbc70c92d
Date: 2016-09-20 13:04 +0200
http://bitbucket.org/pypy/pypy/changeset/8dcfbc70c92d/

Log:	apply translation fixes for ppc

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -13,7 +13,7 @@
 from rpython.jit.backend.ppc.locations import imm, RegisterLocation
 from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN
 from rpython.jit.backend.llsupport.vector_ext import VectorExt
-from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET
+from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET, WORD
 import rpython.jit.backend.ppc.register as r
 import rpython.jit.backend.ppc.condition as c
 import rpython.jit.backend.ppc.locations as l
@@ -785,10 +785,12 @@
         arg = op.getarg(0)
         if arg.is_vector():
             srcloc = self.ensure_vector_reg(arg)
+            assert isinstance(arg, VectorOp)
+            size = arg.bytesize
         else:
             # unpack
-            srcloc = self.ensure_reg(arg0)
-        size = arg.bytesize
+            srcloc = self.ensure_reg(arg)
+            size = WORD
         if op.is_vector():
             resloc = self.force_allocate_vector_reg(op)
         else:

From pypy.commits at gmail.com  Tue Sep 20 07:06:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 20 Sep 2016 04:06:02 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57e1181a.861b1c0a.6d3a0.2ab5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87245:404ba62b0c60
Date: 2016-09-20 13:05 +0200
http://bitbucket.org/pypy/pypy/changeset/404ba62b0c60/

Log:	merge default

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -63,8 +63,7 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
-    g['SO'] = g['SOABI'][0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -76,8 +75,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
-    g['SO'] = g['SOABI'][0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -526,7 +526,10 @@
 
         # PyPy:
         import imp
-        _CONFIG_VARS['SOABI'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
+        for suffix, mode, type_ in imp.get_suffixes():
+            if type_ == imp.C_EXTENSION:
+                _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
+                break        
 
     if args:
         vals = []
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -797,9 +797,7 @@
         class AA(object):
             __slots__ = ('a',)
         aa = AA()
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "aa.__class__ = A")
+        aa.__class__ = A
         raises(TypeError, "aa.__class__ = object")
         class Z1(A):
             pass
@@ -861,9 +859,13 @@
             __slots__ = ['a', 'b']
         class Order2(object):
             __slots__ = ['b', 'a']
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "Order1().__class__ = Order2")
+        Order1().__class__ = Order2
+
+        # like CPython, the order of slot names doesn't matter
+        x = Order1()
+        x.a, x.b = 1, 2
+        x.__class__ = Order2
+        assert (x.a, x.b) == (1, 2)
 
         class U1(object):
             __slots__ = ['a', 'b']
@@ -873,10 +875,11 @@
             __slots__ = ['a', 'b']
         class V2(V1):
             __slots__ = ['c', 'd', 'e']
-        # the following line does not work on CPython >= 2.6 either.
-        # that's just obscure.  Really really.  So we just ignore
-        # the whole issue until someone comes complaining.  Then we'll
-        # just kill slots altogether apart from maybe doing a few checks.
+        # the following line does not work on CPython either: we can't
+        # change a class if the old and new class have different layouts
+        # that look compatible but aren't, because they don't have the
+        # same base-layout class (even if these base classes are
+        # themselves compatible)...  obscure.
         raises(TypeError, "U2().__class__ = V2")
 
     def test_name(self):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,9 +102,10 @@
     """
     _immutable_ = True
 
-    def __init__(self, typedef, nslots, base_layout=None):
+    def __init__(self, typedef, nslots, newslotnames=[], base_layout=None):
         self.typedef = typedef
         self.nslots = nslots
+        self.newslotnames = newslotnames[:]    # make a fixed-size list
         self.base_layout = base_layout
 
     def issublayout(self, parent):
@@ -114,6 +115,12 @@
                 return False
         return True
 
+    def expand(self, hasdict, weakrefable):
+        """Turn this Layout into a tuple.  If two classes get equal
+        tuples, it means their instances have a fully compatible layout."""
+        return (self.typedef, self.newslotnames, self.base_layout,
+                hasdict, weakrefable)
+
 
 # possible values of compares_by_identity_status
 UNKNOWN = 0
@@ -289,8 +296,7 @@
 
     # compute a tuple that fully describes the instance layout
     def get_full_instance_layout(self):
-        layout = self.layout
-        return (layout, self.hasdict, self.weakrefable)
+        return self.layout.expand(self.hasdict, self.weakrefable)
 
     def compute_default_mro(self):
         return compute_C3_mro(self.space, self)
@@ -1001,11 +1007,15 @@
         w_self.weakrefable = w_self.weakrefable or w_base.weakrefable
     return hasoldstylebase
 
+
 def create_all_slots(w_self, hasoldstylebase, w_bestbase, force_new_layout):
+    from pypy.objspace.std.listobject import StringSort
+
     base_layout = w_bestbase.layout
     index_next_extra_slot = base_layout.nslots
     space = w_self.space
     dict_w = w_self.dict_w
+    newslotnames = []
     if '__slots__' not in dict_w:
         wantdict = True
         wantweakref = True
@@ -1031,8 +1041,22 @@
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
             else:
-                index_next_extra_slot = create_slot(w_self, slot_name,
-                                                    index_next_extra_slot)
+                newslotnames.append(slot_name)
+        # Sort the list of names collected so far
+        sorter = StringSort(newslotnames, len(newslotnames))
+        sorter.sort()
+        # Try to create all slots in order.  The creation of some of
+        # them might silently fail; then we delete the name from the
+        # list.  At the end, 'index_next_extra_slot' has been advanced
+        # by the final length of 'newslotnames'.
+        i = 0
+        while i < len(newslotnames):
+            if create_slot(w_self, newslotnames[i], index_next_extra_slot):
+                index_next_extra_slot += 1
+                i += 1
+            else:
+                del newslotnames[i]
+    #
     wantdict = wantdict or hasoldstylebase
     if wantdict:
         create_dict_slot(w_self)
@@ -1041,11 +1065,12 @@
     if '__del__' in dict_w:
         w_self.hasuserdel = True
     #
+    assert index_next_extra_slot == base_layout.nslots + len(newslotnames)
     if index_next_extra_slot == base_layout.nslots and not force_new_layout:
         return base_layout
     else:
         return Layout(base_layout.typedef, index_next_extra_slot,
-                      base_layout=base_layout)
+                      newslotnames, base_layout=base_layout)
 
 def create_slot(w_self, slot_name, index_next_extra_slot):
     space = w_self.space
@@ -1058,9 +1083,10 @@
         slot_name = space.str_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
         member = Member(index_next_extra_slot, slot_name, w_self)
-        index_next_extra_slot += 1
         w_self.dict_w[slot_name] = space.wrap(member)
-    return index_next_extra_slot
+        return True
+    else:
+        return False
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4558,3 +4558,20 @@
         self.meta_interp(f, [])
         self.check_resops(guard_nonnull=0)
 
+    def test_loop_before_main_loop(self):
+        fdriver = JitDriver(greens=[], reds='auto')
+        gdriver = JitDriver(greens=[], reds='auto')
+        def f(i, j):
+            while j > 0:   # this loop unrolls because it is in the same
+                j -= 1     # function as a jit_merge_point()
+            while i > 0:
+                fdriver.jit_merge_point()
+                i -= 1
+        def g(i, j, k):
+            while k > 0:
+                gdriver.jit_merge_point()
+                f(i, j)
+                k -= 1
+
+        self.meta_interp(g, [5, 5, 5])
+        self.check_resops(guard_true=10)   # 5 unrolled, plus 5 unrelated
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -646,6 +646,9 @@
     def ll_str(self, x):
         return self.getstr()
 
+    def get_ll_eq_function(self):
+        return None
+
 
 class MultipleFrozenPBCReprBase(CanBeNull, Repr):
     def convert_const(self, pbc):
@@ -654,6 +657,9 @@
         frozendesc = self.rtyper.annotator.bookkeeper.getdesc(pbc)
         return self.convert_desc(frozendesc)
 
+    def get_ll_eq_function(self):
+        return None
+
 class MultipleUnrelatedFrozenPBCRepr(MultipleFrozenPBCReprBase):
     """For a SomePBC of frozen PBCs that have no common access set.
     The only possible operation on such a thing is comparison with 'is'."""
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -1729,6 +1729,23 @@
         res = self.interpret(f, [])
         assert res == 42
 
+    def test_equality_of_frozen_pbcs_inside_data_structures(self):
+        class A:
+            def _freeze_(self):
+                return True
+        a1 = A()
+        a2 = A()
+        def f():
+            return [a1] == [a1]
+        def g(i):
+            x1 = [a1, a2][i]
+            x2 = [a1, a2][i]
+            return (x1,) == (x2,)
+        res = self.interpret(f, [])
+        assert res == True
+        res = self.interpret(g, [1])
+        assert res == True
+
 # ____________________________________________________________
 
 def test_hlinvoke_simple():

From pypy.commits at gmail.com  Tue Sep 20 07:54:01 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 20 Sep 2016 04:54:01 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: somewhat obscure: if there is
 a condition on the "version" (ie second argument
Message-ID: <57e12359.d4301c0a.a2fa3.2b76@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r87246:70a901ab0437
Date: 2016-08-18 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/70a901ab0437/

Log:	somewhat obscure: if there is a condition on the "version" (ie
	second argument of a elidable_compatible call) then that condition
	was emmitted *after* the guard_compatible in the short preamble.
	that's nonsense, fix it with a hack for now.

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -146,11 +146,13 @@
         """ re-emit the conditions about variable op into the short preamble
         """
         from rpython.jit.metainterp.resoperation import rop, ResOperation
-        short.append(
+        localshort = []
+        localshort.append(
             ResOperation(rop.GUARD_COMPATIBLE, [
                 op, self.known_valid]))
         for cond in self.conditions:
-            cond.emit_condition(op, short, optimizer)
+            cond.emit_condition(op, localshort, short, optimizer)
+        short.extend(localshort)
 
     def emit_needed_conditions_if_const_matches(
             self, other, const, op, extra_guards, optimizer, cpu):
@@ -160,6 +162,7 @@
         at the end. """
         from rpython.jit.metainterp.resoperation import rop, ResOperation
         have_guard = False
+        local_extra_guards = []
         for cond in self.conditions:
             if other is None or not other.contains_condition(cond):
                 if const is None:
@@ -169,13 +172,16 @@
                     if not have_guard:
                         # NB: the guard_compatible here needs to use const,
                         # otherwise the optimizer will just complain
-                        extra_guards.append(ResOperation(
+                        local_extra_guards.append(ResOperation(
                             rop.GUARD_COMPATIBLE,
                                 [op, const]))
                         have_guard = True
-                    cond.emit_condition(op, extra_guards, optimizer, const)
+                    cond.emit_condition(
+                            op, local_extra_guards,
+                            extra_guards, optimizer, const)
                 else:
                     return False
+        extra_guards.extend(local_extra_guards)
         return True
 
     def attach_to_descr(self, descr, guard_value_op, optimizer):
@@ -263,7 +269,7 @@
     def repr(self):
         return ""
 
-    def emit_condition(self, op, short, optimizer, const=None):
+    def emit_condition(self, op, guards, pre_guards, optimizer, const=None):
         raise NotImplementedError("abstract base class")
 
     def _repr_const(self, arg):
@@ -341,7 +347,7 @@
                 return False
         return True
 
-    def emit_condition(self, op, short, optimizer, const=None):
+    def emit_condition(self, op, short, pre_short, optimizer, const=None):
         from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID
         from rpython.jit.metainterp.resoperation import rop, ResOperation
         # woah, mess
@@ -387,11 +393,14 @@
             res = self.debug_mp_str + "\n" + res
         return res
 
+class UnsupportedInfoInGuardCompatibleError(Exception):
+    pass
 
 class QuasiimmutGetfieldAndPureCallCondition(PureCallCondition):
     const_args_start_at = 3
 
     def __init__(self, op, qmutdescr, optimizer):
+        from rpython.jit.metainterp.optimizeopt import info
         PureCallCondition.__init__(self, op, optimizer)
         self.args[2] = None
         # XXX not 100% sure whether it's save to store the whole descr
@@ -399,6 +408,17 @@
         self.qmut = qmutdescr.qmut
         self.mutatefielddescr = qmutdescr.mutatefielddescr
         self.fielddescr = qmutdescr.fielddescr
+        self.need_nonnull_arg2 = False
+        if self.fielddescr.is_pointer_field():
+            fieldinfo = optimizer.getptrinfo(op.getarg(2))
+            if fieldinfo is not None:
+                if type(fieldinfo) != info.NonNullPtrInfo:
+                    # XXX PyPy only needs non-null versions. if another
+                    # interpreter needs something more specific we need to
+                    # generalize this code
+                    raise UnsupportedInfoInGuardCompatible()
+                else:
+                    self.need_nonnull_arg2 = True
 
     def activate(self, ref, optimizer):
         # record the quasi-immutable
@@ -458,7 +478,7 @@
                 return False
         return True
 
-    def emit_condition(self, op, short, optimizer, const=None):
+    def emit_condition(self, op, short, pre_short, optimizer, const=None):
         from rpython.jit.metainterp.resoperation import rop, ResOperation
         from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
         # more mess
@@ -466,6 +486,13 @@
         if fielddescr.is_pointer_field():
             getfield_op = ResOperation(
                 rop.GETFIELD_GC_R, [op], fielddescr)
+            if self.need_nonnull_arg2:
+                # XXX atm it's emmitted n times
+                getfield_op2 = ResOperation(
+                    rop.GETFIELD_GC_R, [op], fielddescr)
+                guard_nonnull = ResOperation(rop.GUARD_NONNULL, [getfield_op2])
+                pre_short.append(getfield_op2)
+                pre_short.append(guard_nonnull)
         elif fielddescr.is_float_field():
             getfield_op = ResOperation(
                 rop.GETFIELD_GC_F, [op], fielddescr)
@@ -485,7 +512,7 @@
                 rop.GUARD_NOT_INVALIDATED, []),
             getfield_op])
         index = len(short)
-        PureCallCondition.emit_condition(self, op, short, optimizer, const)
+        PureCallCondition.emit_condition(self, op, short, pre_short, optimizer, const)
         call_op = short[index]
         # puh, not pretty
         args = call_op.getarglist()[:]
diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py
--- a/rpython/jit/metainterp/test/test_compatible.py
+++ b/rpython/jit/metainterp/test/test_compatible.py
@@ -696,5 +696,7 @@
         main(False)
 
         self.meta_interp(main, [True], backendopt=True)
-        self.check_trace_count(2)
+         # XXX this should be 3, but hard to achieve without propagating more
+         # information across bridges
+        self.check_trace_count(4)
         self.check_resops(call_i=0)

From pypy.commits at gmail.com  Tue Sep 20 07:54:04 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 20 Sep 2016 04:54:04 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: add debug prints to at least
 have a chance to figure out what is going on
Message-ID: <57e1235c.d4301c0a.a2fa3.2b80@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r87248:4f02e3d3f6f9
Date: 2016-08-23 15:03 +0100
http://bitbucket.org/pypy/pypy/changeset/4f02e3d3f6f9/

Log:	add debug prints to at least have a chance to figure out what is
	going on

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -2,6 +2,8 @@
 from rpython.jit.metainterp.history import newconst
 from rpython.jit.codewriter import longlong
 from rpython.jit.metainterp.resoperation import rop
+from rpython.rlib.debug import (have_debug_prints, debug_start, debug_stop,
+    debug_print)
 
 def do_call(cpu, argboxes, descr):
     from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID
@@ -64,8 +66,11 @@
         self.last_quasi_immut_field_op = op
 
     def check_compat(self, cpu, ref):
-        for cond in self.conditions:
-            if not cond.check(cpu, ref):
+        for i, cond in enumerate(self.conditions):
+            res = cond.check_and_return_result_if_different(cpu, ref)
+            if res is not None:
+                if have_debug_prints():
+                    debug_print("incompatible condition", i, ", got", cond._repr_const(res), ":", cond.repr())
                 return False
         return True
 
@@ -168,7 +173,7 @@
                 if const is None:
                     return False
                 ref = const.getref_base()
-                if cond.check(cpu, ref):
+                if cond.check_and_return_result_if_different(cpu, ref) is None:
                     if not have_guard:
                         # NB: the guard_compatible here needs to use const,
                         # otherwise the optimizer will just complain
@@ -254,7 +259,9 @@
         self.debug_mp_str = s
         self.rpyfunc = None
 
-    def check(self, cpu, ref):
+    def check_and_return_result_if_different(self, cpu, ref):
+        """ checks the condition using ref as an argument to the function. if
+        the result is different, return the result. otherwise return None. """
         raise NotImplementedError
 
     def activate(self, ref, optimizer):
@@ -309,8 +316,7 @@
         self.descr = op.getdescr()
         self.rpyfunc = op.rpyfunc
 
-    def check(self, cpu, ref):
-        from rpython.rlib.debug import debug_print, debug_start, debug_stop
+    def check_and_return_result_if_different(self, cpu, ref):
         calldescr = self.descr
         # change exactly the first argument
         arglist = self.args
@@ -318,15 +324,17 @@
         try:
             res = do_call(cpu, arglist, calldescr)
         except Exception:
-            debug_start("jit-guard-compatible")
-            debug_print("call to elidable_compatible function raised")
-            debug_stop("jit-guard-compatible")
+            if have_debug_prints():
+                debug_start("jit-guard-compatible")
+                debug_print("call to elidable_compatible function raised")
+                debug_print(self.repr())
+                debug_stop("jit-guard-compatible")
             return False
         finally:
             arglist[1] = None
         if not res.same_constant(self.res):
-            return False
-        return True
+            return res
+        return None
 
     def same_cond(self, other, res=None):
         if type(other) is not PureCallCondition:
@@ -433,7 +441,7 @@
                                          self.mutatefielddescr)
         qmut.register_loop_token(loop_token.loop_token_wref)
 
-    def check(self, cpu, ref):
+    def check_and_return_result_if_different(self, cpu, ref):
         from rpython.rlib.debug import debug_print, debug_start, debug_stop
         from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
         calldescr = self.descr
@@ -444,15 +452,17 @@
         try:
             res = do_call(cpu, arglist, calldescr)
         except Exception:
-            debug_start("jit-guard-compatible")
-            debug_print("call to elidable_compatible function raised")
-            debug_stop("jit-guard-compatible")
+            if have_debug_prints():
+                debug_start("jit-guard-compatible")
+                debug_print("call to elidable_compatible function raised")
+                debug_print(self.repr())
+                debug_stop("jit-guard-compatible")
             return False
         finally:
             arglist[1] = arglist[2] = None
         if not res.same_constant(self.res):
-            return False
-        return True
+            return res
+        return None
 
     def same_cond(self, other, res=None):
         if type(other) is not QuasiimmutGetfieldAndPureCallCondition:
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -1,7 +1,7 @@
 import weakref
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, compute_unique_id
 from rpython.rlib.debug import debug_start, debug_stop, debug_print, have_debug_prints
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib import rstack
@@ -1110,18 +1110,31 @@
         """
         # need to do the checking oldest to newest, to check the most specific
         # condition first
-        if self._compatibility_conditions:
-            if self._compatibility_conditions.check_compat_and_activate(
-                    cpu, ref, self.rd_loop_token):
-                return self._compatibility_conditions.jump_target
-        for _compatibility_conditions in self.other_compat_conditions:
-            if _compatibility_conditions.check_compat_and_activate(
-                    cpu, ref, self.rd_loop_token):
-                return _compatibility_conditions.jump_target
-        # none of the other conditions matched. if we have a
-        # fallback_jump_target, go there (otherwise we run the risk of
-        # producing arbitrary amounts of code)
-        return self.fallback_jump_target
+        debug_start("jit-guard-compatible")
+        try:
+            if have_debug_prints():
+                debug_print("Guard0x%x check compatibility of" % (compute_unique_id(self), ), ref)
+            if self._compatibility_conditions:
+                if self._compatibility_conditions.check_compat_and_activate(
+                        cpu, ref, self.rd_loop_token):
+                    debug_print("first guard is compatible")
+                    return self._compatibility_conditions.jump_target
+            for i, _compatibility_conditions in enumerate(self.other_compat_conditions):
+                if _compatibility_conditions.check_compat_and_activate(
+                        cpu, ref, self.rd_loop_token):
+                    debug_print("attached guard", i, "is compatible")
+                    return _compatibility_conditions.jump_target
+            if have_debug_prints():
+                if self.fallback_jump_target:
+                    debug_print("all incompatible, going to fallback")
+                else:
+                    debug_print("all incompatible, failing")
+            # none of the other conditions matched. if we have a
+            # fallback_jump_target, go there (otherwise we run the risk of
+            # producing arbitrary amounts of code)
+            return self.fallback_jump_target
+        finally:
+            debug_stop("jit-guard-compatible")
 
     def compile_and_attach(self, metainterp, new_loop, orig_inputargs):
         # if new_loop starts with another guard_compatible on the same argument

From pypy.commits at gmail.com  Tue Sep 20 07:54:06 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 20 Sep 2016 04:54:06 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix translation
Message-ID: <57e1235e.0d8d1c0a.4f0be.3314@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r87249:3184d18e59b3
Date: 2016-08-26 15:02 +0100
http://bitbucket.org/pypy/pypy/changeset/3184d18e59b3/

Log:	fix translation

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -317,6 +317,7 @@
         self.rpyfunc = op.rpyfunc
 
     def check_and_return_result_if_different(self, cpu, ref):
+        from rpython.jit.metainterp.history import ConstInt
         calldescr = self.descr
         # change exactly the first argument
         arglist = self.args
@@ -329,7 +330,7 @@
                 debug_print("call to elidable_compatible function raised")
                 debug_print(self.repr())
                 debug_stop("jit-guard-compatible")
-            return False
+            return ConstInt(41)
         finally:
             arglist[1] = None
         if not res.same_constant(self.res):
@@ -442,6 +443,7 @@
         qmut.register_loop_token(loop_token.loop_token_wref)
 
     def check_and_return_result_if_different(self, cpu, ref):
+        from rpython.jit.metainterp.history import ConstInt
         from rpython.rlib.debug import debug_print, debug_start, debug_stop
         from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
         calldescr = self.descr
@@ -457,7 +459,7 @@
                 debug_print("call to elidable_compatible function raised")
                 debug_print(self.repr())
                 debug_stop("jit-guard-compatible")
-            return False
+            return ConstInt(41)
         finally:
             arglist[1] = arglist[2] = None
         if not res.same_constant(self.res):

From pypy.commits at gmail.com  Tue Sep 20 07:54:08 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 20 Sep 2016 04:54:08 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: just print the pointer constant
Message-ID: <57e12360.43e61c0a.64418.57fd@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r87250:f9845ecd2b54
Date: 2016-08-26 15:03 +0100
http://bitbucket.org/pypy/pypy/changeset/f9845ecd2b54/

Log:	just print the pointer constant

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -294,7 +294,7 @@
                     sid = self.metainterp_sd.cpu.get_actual_typeid(rffi.cast(llmemory.GCREF, llstr("abc")))
                     if sid == tid:
                         return hlstr(rffi.cast(lltype.Ptr(rstr.STR), arg.getref_base()))
-                return "<some const ptr>"
+                return "0x%x" % lltype.cast_ptr_to_int(arg.value)
             else:
                 return "None"
         elif isinstance(arg, ConstFloat):

From pypy.commits at gmail.com  Tue Sep 20 07:54:03 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 20 Sep 2016 04:54:03 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix translation
Message-ID: <57e1235b.8c9a1c0a.da309.3233@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r87247:a53c8229dad7
Date: 2016-08-19 13:42 +0200
http://bitbucket.org/pypy/pypy/changeset/a53c8229dad7/

Log:	fix translation

diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py
--- a/rpython/jit/metainterp/compatible.py
+++ b/rpython/jit/metainterp/compatible.py
@@ -393,7 +393,7 @@
             res = self.debug_mp_str + "\n" + res
         return res
 
-class UnsupportedInfoInGuardCompatibleError(Exception):
+class UnsupportedInfoInGuardCompatible(Exception):
     pass
 
 class QuasiimmutGetfieldAndPureCallCondition(PureCallCondition):
@@ -412,7 +412,7 @@
         if self.fielddescr.is_pointer_field():
             fieldinfo = optimizer.getptrinfo(op.getarg(2))
             if fieldinfo is not None:
-                if type(fieldinfo) != info.NonNullPtrInfo:
+                if type(fieldinfo) is not info.NonNullPtrInfo:
                     # XXX PyPy only needs non-null versions. if another
                     # interpreter needs something more specific we need to
                     # generalize this code

From pypy.commits at gmail.com  Tue Sep 20 07:54:13 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 20 Sep 2016 04:54:13 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: merge default
Message-ID: <57e12365.669fc20a.5756b.7ea6@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r87251:fc880b5612ed
Date: 2016-09-01 10:06 +0200
http://bitbucket.org/pypy/pypy/changeset/fc880b5612ed/

Log:	merge default

diff too long, truncating to 2000 out of 226692 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -27,3 +27,6 @@
 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2
 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3
 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
+77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -74,6 +74,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -117,7 +118,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -141,6 +141,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -211,6 +212,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -221,6 +223,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -229,12 +232,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -355,115 +360,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
-
-  Heinrich-Heine University, Germany 
-  Open End AB (formerly AB Strakt), Sweden
-  merlinux GmbH, Germany 
-  tismerysoft GmbH, Germany 
-  Logilab Paris, France 
-  DFKI GmbH, Germany 
-  Impara, Germany
-  Change Maker, Sweden 
-  University of California Berkeley, USA
-  Google Inc.
-  King's College London
-
-The PyPy Logo as used by http://speed.pypy.org and others was created
-by Samuel Reis and is distributed on terms of Creative Commons Share Alike
-License.
- 
-License for 'lib-python/2.7'
-============================
-
-Except when otherwise stated (look for LICENSE files or copyright/license
-information at the beginning of each file) the files in the 'lib-python/2.7'
-directory are all copyrighted by the Python Software Foundation and licensed
-under the terms that you can find here: https://docs.python.org/2/license.html
-
-License for 'pypy/module/unicodedata/'
-======================================
-
-The following files are from the website of The Unicode Consortium
-at http://www.unicode.org/.  For the terms of use of these files, see
-http://www.unicode.org/terms_of_use.html .  Or they are derived from
-files from the above website, and the same terms of use apply.
-
-    CompositionExclusions-*.txt
-    EastAsianWidth-*.txt
-    LineBreak-*.txt
-    UnicodeData-*.txt
-    UnihanNumeric-*.txt
-
-License for 'dotviewer/font/'
-=============================
-
-Copyright (C) 2008 The Android Open Source Project
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-  
-     http://www.apache.org/licenses/LICENSE-2.0
-  
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Detailed license information is contained in the NOTICE file in the
-directory.
-
-
-Licenses and Acknowledgements for Incorporated Software
-=======================================================
-
-This section is an incomplete, but growing list of licenses and
-acknowledgements for third-party software incorporated in the PyPy
-distribution.
-
-License for 'Tcl/Tk'
---------------------
-
-This copy of PyPy contains library code that may, when used, result in
-the Tcl/Tk library to be loaded.  PyPy also includes code that may be
-regarded as being a copy of some parts of the Tcl/Tk header files.
-You may see a copy of the License for Tcl/Tk in the file
-`lib_pypy/_tkinter/license.terms` included here.
-
-License for 'bzip2'
--------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
-
-    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
-
-License for 'openssl'
----------------------
-
-This copy of PyPy may be linked (dynamically or statically) with the
-openssl library.  You may see a copy of the License for OpenSSL at
-
-    https://www.openssl.org/source/license.html
-
-License for 'gdbm'
-------------------
-
-The gdbm module includes code from gdbm.h, which is distributed under
-the terms of the GPL license version 2 or any later version.  Thus the
-gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
-under the terms of the GPL license as well.
-
-License for 'rpython/rlib/rvmprof/src'
---------------------------------------
-
-The code is based on gperftools. You may see a copy of the License for it at
-
-    https://github.com/gperftools/gperftools/blob/master/COPYING
+  werat
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -85,10 +85,11 @@
     pass
 
 def splitline(line, re_word = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]')):
+    import ast
     result = []
     for word in re_word.findall(line):
         if word.startswith('"'):
-            word = eval(word)
+            word = ast.literal_eval(word)
         result.append(word)
     return result
 
diff --git a/include/PyPy.h b/include/PyPy.h
--- a/include/PyPy.h
+++ b/include/PyPy.h
@@ -2,7 +2,11 @@
 #define _PYPY_H_
 
 /* This header is meant to be included in programs that use PyPy as an
-   embedded library. */
+   embedded library.
+
+   NOTE: this is deprecated.  Instead, use cffi's embedding support:
+   http://cffi.readthedocs.org/en/latest/embedding.html
+*/
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -122,22 +122,24 @@
     """Dummy method to let some easy_install packages that have
     optional C speedup components.
     """
+    def customize(executable, flags):
+        command = compiler.executables[executable] + flags
+        setattr(compiler, executable, command)
+
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cppflags)
         if "CFLAGS" in os.environ:
             cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
+            for executable in ('compiler', 'compiler_so', 'linker_so'):
+                customize(executable, cflags)
         if "LDFLAGS" in os.environ:
             ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+            customize('linker_so', ldflags)
 
 
 from sysconfig_cpython import (
diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -174,7 +174,7 @@
 
 class StringlikeHashRandomizationTests(HashRandomizationTests):
     if check_impl_detail(pypy=True):
-        EMPTY_STRING_HASH = -1
+        EMPTY_STRING_HASH = -2
     else:
         EMPTY_STRING_HASH = 0
 
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -167,7 +167,7 @@
         else:
             return self.value
 
-    def __buffer__(self):
+    def __buffer__(self, flags):
         return buffer(self._buffer)
 
     def _get_b_base(self):
@@ -199,10 +199,13 @@
     return tp._alignmentofinstances()
 
 @builtinify
-def byref(cdata):
+def byref(cdata, offset=0):
     # "pointer" is imported at the end of this module to avoid circular
     # imports
-    return pointer(cdata)
+    ptr = pointer(cdata)
+    if offset != 0:
+        ptr._buffer[0] += offset
+    return ptr
 
 def cdata_from_address(self, address):
     # fix the address: turn it into as unsigned, in case it's a negative number
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -342,7 +342,7 @@
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
                 self._convert_args(argtypes, args[1:], kwargs))
-            newargs.insert(0, thisvalue.value)
+            newargs.insert(0, thisarg)
             argtypes.insert(0, c_void_p)
         else:
             thisarg = None
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -0,0 +1,91 @@
+# Note: uses the CFFI out-of-line ABI mode.  We can't use the API
+# mode because ffi.compile() needs to run the compiler, which
+# needs 'subprocess', which needs 'msvcrt' and '_subprocess',
+# which depend on '_pypy_winbase_cffi' already.
+#
+# Note that if you need to regenerate _pypy_winbase_cffi and
+# can't use a preexisting PyPy to do that, then running this
+# file should work as long as 'subprocess' is not imported
+# by cffi.  I had to hack in 'cffi._pycparser' to move an
+#'import subprocess' to the inside of a function.  (Also,
+# CPython+CFFI should work as well.)
+#
+# This module supports both msvcrt.py and _subprocess.py.
+
+from cffi import FFI
+
+ffi = FFI()
+
+ffi.set_source("_pypy_winbase_cffi", None)
+
+# ---------- MSVCRT ----------
+
+ffi.cdef("""
+typedef unsigned short wint_t;
+
+int _open_osfhandle(intptr_t osfhandle, int flags);
+intptr_t _get_osfhandle(int fd);
+int _setmode(int fd, int mode);
+int _locking(int fd, int mode, long nbytes);
+
+int _kbhit(void);
+int _getch(void);
+wint_t _getwch(void);
+int _getche(void);
+wint_t _getwche(void);
+int _putch(int);
+wint_t _putwch(wchar_t);
+int _ungetch(int);
+wint_t _ungetwch(wint_t);
+""")
+
+# ---------- SUBPROCESS ----------
+
+ffi.cdef("""
+typedef struct {
+    DWORD  cb;
+    char * lpReserved;
+    char * lpDesktop;
+    char * lpTitle;
+    DWORD  dwX;
+    DWORD  dwY;
+    DWORD  dwXSize;
+    DWORD  dwYSize;
+    DWORD  dwXCountChars;
+    DWORD  dwYCountChars;
+    DWORD  dwFillAttribute;
+    DWORD  dwFlags;
+    WORD   wShowWindow;
+    WORD   cbReserved2;
+    LPBYTE lpReserved2;
+    HANDLE hStdInput;
+    HANDLE hStdOutput;
+    HANDLE hStdError;
+} STARTUPINFO, *LPSTARTUPINFO;
+
+typedef struct {
+    HANDLE hProcess;
+    HANDLE hThread;
+    DWORD  dwProcessId;
+    DWORD  dwThreadId;
+} PROCESS_INFORMATION, *LPPROCESS_INFORMATION;
+
+DWORD WINAPI GetVersion(void);
+BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD);
+BOOL WINAPI CloseHandle(HANDLE);
+HANDLE WINAPI GetCurrentProcess(void);
+BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
+                            DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
+BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
+BOOL WINAPI TerminateProcess(HANDLE, UINT);
+HANDLE WINAPI GetStdHandle(DWORD);
+""")
+
+# --------------------
+
+if __name__ == "__main__":
+    ffi.compile()
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -0,0 +1,10 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
+    _version = 0x2601,
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+)
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -10,148 +10,99 @@
 
 # Declare external Win32 functions
 
-import ctypes
-
-_kernel32 = ctypes.WinDLL('kernel32')
-
-_CloseHandle = _kernel32.CloseHandle
-_CloseHandle.argtypes = [ctypes.c_int]
-_CloseHandle.restype = ctypes.c_int
-
-_CreatePipe = _kernel32.CreatePipe
-_CreatePipe.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int),
-                        ctypes.c_void_p, ctypes.c_int]
-_CreatePipe.restype = ctypes.c_int
-
-_GetCurrentProcess = _kernel32.GetCurrentProcess
-_GetCurrentProcess.argtypes = []
-_GetCurrentProcess.restype = ctypes.c_int
+from _pypy_winbase_cffi import ffi as _ffi
+_kernel32 = _ffi.dlopen('kernel32')
 
 GetVersion = _kernel32.GetVersion
-GetVersion.argtypes = []
-GetVersion.restype = ctypes.c_int
 
-_DuplicateHandle = _kernel32.DuplicateHandle
-_DuplicateHandle.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int,
-                             ctypes.POINTER(ctypes.c_int),
-                             ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_DuplicateHandle.restype = ctypes.c_int
-
-_WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
-_WaitForSingleObject.restype = ctypes.c_int
-
-_GetExitCodeProcess = _kernel32.GetExitCodeProcess
-_GetExitCodeProcess.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
-_GetExitCodeProcess.restype = ctypes.c_int
-
-_TerminateProcess = _kernel32.TerminateProcess
-_TerminateProcess.argtypes = [ctypes.c_int, ctypes.c_int]
-_TerminateProcess.restype = ctypes.c_int
-
-_GetStdHandle = _kernel32.GetStdHandle
-_GetStdHandle.argtypes = [ctypes.c_int]
-_GetStdHandle.restype = ctypes.c_int
-
-class _STARTUPINFO(ctypes.Structure):
-    _fields_ = [('cb',         ctypes.c_int),
-                ('lpReserved', ctypes.c_void_p),
-                ('lpDesktop',  ctypes.c_char_p),
-                ('lpTitle',    ctypes.c_char_p),
-                ('dwX',        ctypes.c_int),
-                ('dwY',        ctypes.c_int),
-                ('dwXSize',    ctypes.c_int),
-                ('dwYSize',    ctypes.c_int),
-                ('dwXCountChars', ctypes.c_int),
-                ('dwYCountChars', ctypes.c_int),
-                ("dwFillAttribute", ctypes.c_int),
-                ("dwFlags", ctypes.c_int),
-                ("wShowWindow", ctypes.c_short),
-                ("cbReserved2", ctypes.c_short),
-                ("lpReserved2", ctypes.c_void_p),
-                ("hStdInput", ctypes.c_int),
-                ("hStdOutput", ctypes.c_int),
-                ("hStdError", ctypes.c_int)
-                ]
-
-class _PROCESS_INFORMATION(ctypes.Structure):
-    _fields_ = [("hProcess", ctypes.c_int),
-                ("hThread", ctypes.c_int),
-                ("dwProcessID", ctypes.c_int),
-                ("dwThreadID", ctypes.c_int)]
-
-_CreateProcess = _kernel32.CreateProcessA
-_CreateProcess.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p,
-                           ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p,
-                           ctypes.POINTER(_STARTUPINFO), ctypes.POINTER(_PROCESS_INFORMATION)]
-_CreateProcess.restype = ctypes.c_int
-
-del ctypes
 
 # Now the _subprocess module implementation
 
-from ctypes import c_int as _c_int, byref as _byref, WinError as _WinError
+def _WinError():
+    code, message = _ffi.getwinerror()
+    raise WindowsError(code, message)
 
-class _handle:
-    def __init__(self, handle):
-        self.handle = handle
+_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+
+class _handle(object):
+    def __init__(self, c_handle):
+        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
+        self.c_handle = c_handle
+        if int(self) != -1:
+            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
 
     def __int__(self):
-        return self.handle
+        return int(_ffi.cast("intptr_t", self.c_handle))
 
-    def __del__(self):
-        if self.handle is not None:
-            _CloseHandle(self.handle)
+    def __repr__(self):
+        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
 
     def Detach(self):
-        handle, self.handle = self.handle, None
-        return handle
+        h = int(self)
+        if h != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+        return h
 
     def Close(self):
-        if self.handle not in (-1, None):
-            _CloseHandle(self.handle)
-            self.handle = None
+        if int(self) != -1:
+            c_handle = self.c_handle
+            self.c_handle = _INVALID_HANDLE_VALUE
+            _ffi.gc(c_handle, None)
+            _kernel32.CloseHandle(c_handle)
 
 def CreatePipe(attributes, size):
-    read = _c_int()
-    write = _c_int()
+    handles = _ffi.new("HANDLE[2]")
 
-    res = _CreatePipe(_byref(read), _byref(write), None, size)
+    res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size)
 
     if not res:
         raise _WinError()
 
-    return _handle(read.value), _handle(write.value)
+    return _handle(handles[0]), _handle(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_GetCurrentProcess())
+    return _handle(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
-    target = _c_int()
+    # CPython: the first three arguments are expected to be integers
+    target = _ffi.new("HANDLE[1]")
 
-    res = _DuplicateHandle(int(source_process), int(source), int(target_process),
-                           _byref(target),
-                           access, inherit, options)
+    res = _kernel32.DuplicateHandle(
+        _ffi.cast("HANDLE", source_process),
+        _ffi.cast("HANDLE", source),
+        _ffi.cast("HANDLE", target_process),
+        target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target.value)
+    return _handle(target[0])
+
+def _z(input):
+    if input is None:
+        return _ffi.NULL
+    if isinstance(input, basestring):
+        return str(input)
+    raise TypeError("string/unicode/None expected, got %r" % (
+        type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
                   inherit, flags, env, start_dir, startup_info):
-    si = _STARTUPINFO()
+    si = _ffi.new("STARTUPINFO *")
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
+        # CPython: these three handles are expected to be _handle objects
         if startup_info.hStdInput:
-            si.hStdInput = int(startup_info.hStdInput)
+            si.hStdInput = startup_info.hStdInput.c_handle
         if startup_info.hStdOutput:
-            si.hStdOutput = int(startup_info.hStdOutput)
+            si.hStdOutput = startup_info.hStdOutput.c_handle
         if startup_info.hStdError:
-            si.hStdError = int(startup_info.hStdError)
+            si.hStdError = startup_info.hStdError.c_handle
 
-    pi = _PROCESS_INFORMATION()
+    pi = _ffi.new("PROCESS_INFORMATION *")
 
     if env is not None:
         envbuf = ""
@@ -159,47 +110,55 @@
             envbuf += "%s=%s\0" % (k, v)
         envbuf += '\0'
     else:
-        envbuf = None
+        envbuf = _ffi.NULL
 
-    res = _CreateProcess(name, command_line, None, None, inherit, flags, envbuf,
-                        start_dir, _byref(si), _byref(pi))
+    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+                                   _ffi.NULL, inherit, flags, envbuf,
+                                   _z(start_dir), si, pi)
 
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessID, pi.dwThreadID
+    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
 
 def WaitForSingleObject(handle, milliseconds):
-    res = _WaitForSingleObject(int(handle), milliseconds)
-
+    # CPython: the first argument is expected to be an integer.
+    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
+                                        milliseconds)
     if res < 0:
         raise _WinError()
 
     return res
 
 def GetExitCodeProcess(handle):
-    code = _c_int()
+    # CPython: the first argument is expected to be an integer.
+    code = _ffi.new("DWORD[1]")
 
-    res = _GetExitCodeProcess(int(handle), _byref(code))
+    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
 
     if not res:
         raise _WinError()
 
-    return code.value
+    return code[0]
 
 def TerminateProcess(handle, exitcode):
-    res = _TerminateProcess(int(handle), exitcode)
+    # CPython: the first argument is expected to be an integer.
+    # The second argument is silently wrapped in a UINT.
+    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+                                     _ffi.cast("UINT", exitcode))
 
     if not res:
         raise _WinError()
 
 def GetStdHandle(stdhandle):
-    res = _GetStdHandle(stdhandle)
+    stdhandle = _ffi.cast("DWORD", stdhandle)
+    res = _kernel32.GetStdHandle(stdhandle)
 
     if not res:
         return None
     else:
-        return res
+        # note: returns integer, not handle object
+        return int(_ffi.cast("intptr_t", res))
 
 STD_INPUT_HANDLE = -10
 STD_OUTPUT_HANDLE = -11
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.7.0
+Version: 1.8.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
@@ -59,7 +61,7 @@
 
 #ifdef __cplusplus
 # ifndef _Bool
-#  define _Bool bool   /* semi-hackish: C++ has no _Bool; bool is builtin */
+   typedef bool _Bool;   /* semi-hackish: C++ has no _Bool; bool is builtin */
 # endif
 #endif
 
@@ -196,20 +198,6 @@
     return NULL;
 }
 
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
-                                    const char *fnname)
-{
-    if (PyTuple_GET_SIZE(args_tuple) != expected) {
-        PyErr_Format(PyExc_TypeError,
-                     "%.150s() takes exactly %zd arguments (%zd given)",
-                     fnname, expected, PyTuple_GET_SIZE(args_tuple));
-        return NULL;
-    }
-    return &PyTuple_GET_ITEM(args_tuple, 0);   /* pointer to the first item,
-                                                  the others follow */
-}
-
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.7.0"
+                               "\ncompiled with cffi version: 1.8.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/_pycparser/__init__.py b/lib_pypy/cffi/_pycparser/__init__.py
--- a/lib_pypy/cffi/_pycparser/__init__.py
+++ b/lib_pypy/cffi/_pycparser/__init__.py
@@ -10,7 +10,6 @@
 __all__ = ['c_lexer', 'c_parser', 'c_ast']
 __version__ = '2.14'
 
-from subprocess import Popen, PIPE
 from .c_parser import CParser
 
 
@@ -28,6 +27,7 @@
         When successful, returns the preprocessed file's contents.
         Errors from cpp will be printed out.
     """
+    from subprocess import Popen, PIPE
     path_list = [cpp_path]
     if isinstance(cpp_args, list):
         path_list += cpp_args
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            import warnings
-            warnings.warn("%r has no values explicitly defined; next version "
-                          "will refuse to guess which integer type it is "
-                          "meant to be (unsigned/signed, int/long)"
-                          % self._get_c_name())
-            smallest_value = largest_value = 0
+            raise api.CDefError("%r has no values explicitly defined: "
+                                "refusing to guess which integer type it is "
+                                "meant to be (unsigned/signed, int/long)"
+                % self._get_c_name())
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
+        if self.ffi._embedding is None:
+            prnt('#define Py_LIMITED_API')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -513,7 +515,7 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, (model.StructOrUnion, model.EnumType)):
+        elif isinstance(tp, model.StructOrUnionOrEnum):
             # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
@@ -570,7 +572,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
@@ -683,13 +685,11 @@
             rng = range(len(tp.args))
             for i in rng:
                 prnt('  PyObject *arg%d;' % i)
-            prnt('  PyObject **aa;')
             prnt()
-            prnt('  aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
-            prnt('  if (aa == NULL)')
+            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+                name, len(rng), len(rng),
+                ', '.join(['&arg%d' % i for i in rng])))
             prnt('    return NULL;')
-            for i in rng:
-                prnt('  arg%d = aa[%d];' % (i, i))
         prnt()
         #
         for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
             enumfields = list(tp.enumfields())
             for fldname, fldtype, fbitsize, fqual in enumfields:
                 fldtype = self._field_type(tp, fldname, fldtype)
+                self._check_not_opaque(fldtype,
+                                       "field '%s.%s'" % (tp.name, fldname))
                 # cname is None for _add_missing_struct_unions() only
                 op = OP_NOOP
                 if fbitsize >= 0:
@@ -911,6 +913,13 @@
                             first_field_index, c_fields))
         self._seen_struct_unions.add(tp)
 
+    def _check_not_opaque(self, tp, location):
+        while isinstance(tp, model.ArrayType):
+            tp = tp.item
+        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+            raise TypeError(
+                "%s is of an opaque type (not declared in cdef())" % location)
+
     def _add_missing_struct_unions(self):
         # not very nice, but some struct declarations might be missing
         # because they don't have any known C name.  Check that they are
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -308,7 +308,7 @@
         elif isinstance(tp, model.ArrayType):
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(model.PointerType(tp.item)))
-        elif isinstance(tp, model.StructType):
+        elif isinstance(tp, model.StructOrUnion):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
                     tp._get_c_name(), context))
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -839,7 +839,7 @@
             month = self._month
         if day is None:
             day = self._day
-        return date(year, month, day)
+        return date.__new__(type(self), year, month, day)
 
     # Comparisons of date objects with other.
 
@@ -1356,7 +1356,8 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return time(hour, minute, second, microsecond, tzinfo)
+        return time.__new__(type(self),
+                            hour, minute, second, microsecond, tzinfo)
 
     def __nonzero__(self):
         if self.second or self.microsecond:
@@ -1566,8 +1567,9 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return datetime(year, month, day, hour, minute, second, microsecond,
-                        tzinfo)
+        return datetime.__new__(type(self),
+                                year, month, day, hour, minute, second,
+                                microsecond, tzinfo)
 
     def astimezone(self, tz):
         if not isinstance(tz, tzinfo):
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -137,6 +137,8 @@
             lib.gdbm_sync(self.__ll_dbm)
 
 def open(filename, flags='r', mode=0666):
+    if isinstance(filename, unicode):
+        filename = filename.encode()
     if flags[0] == 'r':
         iflags = lib.GDBM_READER
     elif flags[0] == 'w':
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -7,26 +7,39 @@
 # XXX incomplete: implemented only functions needed by subprocess.py
 # PAC: 2010/08 added MS locking for Whoosh
 
-import ctypes
+# 07/2016: rewrote in CFFI
+
+import sys
+if sys.platform != 'win32':
+    raise ImportError("The 'msvcrt' module is only available on Windows")
+
+import _rawffi
+from _pypy_winbase_cffi import ffi as _ffi
+_lib = _ffi.dlopen(_rawffi.get_libc().name)
+
 import errno
-from ctypes_support import standard_c_lib as _c
-from ctypes_support import get_errno
-
-try:
-    open_osfhandle = _c._open_osfhandle
-except AttributeError: # we are not on windows
-    raise ImportError
 
 try: from __pypy__ import builtinify, validate_fd
 except ImportError: builtinify = validate_fd = lambda f: f
 
 
-open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int]
-open_osfhandle.restype = ctypes.c_int
+def _ioerr():
+    e = _ffi.errno
+    raise IOError(e, errno.errorcode[e])
 
-_get_osfhandle = _c._get_osfhandle
-_get_osfhandle.argtypes = [ctypes.c_int]
-_get_osfhandle.restype = ctypes.c_int
+
+ at builtinify
+def open_osfhandle(fd, flags):
+    """"open_osfhandle(handle, flags) -> file descriptor
+
+    Create a C runtime file descriptor from the file handle handle. The
+    flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,
+    and os.O_TEXT. The returned file descriptor may be used as a parameter
+    to os.fdopen() to create a file object."""
+    fd = _lib._open_osfhandle(fd, flags)
+    if fd == -1:
+        _ioerr()
+    return fd
 
 @builtinify
 def get_osfhandle(fd):
@@ -38,62 +51,74 @@
         validate_fd(fd)
     except OSError as e:
         raise IOError(*e.args)
-    return _get_osfhandle(fd)
+    result = _lib._get_osfhandle(fd)
+    if result == -1:
+        _ioerr()
+    return result
 
-setmode = _c._setmode
-setmode.argtypes = [ctypes.c_int, ctypes.c_int]
-setmode.restype = ctypes.c_int
+ at builtinify
+def setmode(fd, flags):
+    """setmode(fd, mode) -> Previous mode
+
+    Set the line-end translation mode for the file descriptor fd. To set
+    it to text mode, flags should be os.O_TEXT; for binary, it should be
+    os.O_BINARY."""
+    flags = _lib._setmode(fd, flags)
+    if flags == -1:
+        _ioerr()
+    return flags
 
 LK_UNLCK, LK_LOCK, LK_NBLCK, LK_RLCK, LK_NBRLCK = range(5)
 
-_locking = _c._locking
-_locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
-_locking.restype = ctypes.c_int
-
 @builtinify
 def locking(fd, mode, nbytes):
-    '''lock or unlock a number of bytes in a file.'''
-    rv = _locking(fd, mode, nbytes)
+    """"locking(fd, mode, nbytes) -> None
+
+    Lock part of a file based on file descriptor fd from the C runtime.
+    Raises IOError on failure. The locked region of the file extends from
+    the current file position for nbytes bytes, and may continue beyond
+    the end of the file. mode must be one of the LK_* constants listed
+    below. Multiple regions in a file may be locked at the same time, but
+    may not overlap. Adjacent regions are not merged; they must be unlocked
+    individually."""
+    rv = _lib._locking(fd, mode, nbytes)
     if rv != 0:
-        e = get_errno()
-        raise IOError(e, errno.errorcode[e])
+        _ioerr()
 
 # Console I/O routines
 
-kbhit = _c._kbhit
-kbhit.argtypes = []
-kbhit.restype = ctypes.c_int
+kbhit = _lib._kbhit
 
-getch = _c._getch
-getch.argtypes = []
-getch.restype = ctypes.c_char
+ at builtinify
+def getch():
+    return chr(_lib._getch())
 
-getwch = _c._getwch
-getwch.argtypes = []
-getwch.restype = ctypes.c_wchar
+ at builtinify
+def getwch():
+    return unichr(_lib._getwch())
 
-getche = _c._getche
-getche.argtypes = []
-getche.restype = ctypes.c_char
+ at builtinify
+def getche():
+    return chr(_lib._getche())
 
-getwche = _c._getwche
-getwche.argtypes = []
-getwche.restype = ctypes.c_wchar
+ at builtinify
+def getwche():
+    return unichr(_lib._getwche())
 
-putch = _c._putch
-putch.argtypes = [ctypes.c_char]
-putch.restype = None
+ at builtinify
+def putch(ch):
+    _lib._putch(ord(ch))
 
-putwch = _c._putwch
-putwch.argtypes = [ctypes.c_wchar]
-putwch.restype = None
+ at builtinify
+def putwch(ch):
+    _lib._putwch(ord(ch))
 
-ungetch = _c._ungetch
-ungetch.argtypes = [ctypes.c_char]
-ungetch.restype = None
+ at builtinify
+def ungetch(ch):
+    if _lib._ungetch(ord(ch)) == -1:   # EOF
+        _ioerr()
 
-ungetwch = _c._ungetwch
-ungetwch.argtypes = [ctypes.c_wchar]
-ungetwch.restype = None
-
-del ctypes
+ at builtinify
+def ungetwch(ch):
+    if _lib._ungetwch(ord(ch)) == -1:   # EOF
+        _ioerr()
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -86,7 +86,11 @@
     if len(limits) != 2:
         raise ValueError("expected a tuple of 2 integers")
 
-    if lib.my_setrlimit(resource, limits[0], limits[1]) == -1:
+    # accept and round down floats, like CPython does
+    limit0 = int(limits[0])
+    limit1 = int(limits[1])
+
+    if lib.my_setrlimit(resource, limit0, limit1) == -1:
         if ffi.errno == EINVAL:
             raise ValueError("current limit exceeds maximum limit")
         elif ffi.errno == EPERM:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -36,7 +36,7 @@
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
-    "_csv", "cppyy", "_pypyjson",
+    "_csv", "cppyy", "_pypyjson", "_jitlog"
 ])
 
 from rpython.jit.backend import detect_cpu
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -104,27 +104,24 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev liblzma-dev
-
-For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+    tk-dev libgc-dev \
+    liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     lib-sqlite3-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel
-
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+    gdbm-devel \
+    xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses
+    libexpat-devel libffi-devel python-curses \
+    xz-devel  # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
-For the optional lzma module on PyPy3 you will also need ``xz-devel``.
-
 On Mac OS X, most of these build-time dependencies are installed alongside
 the Developer Tools. However, note that in order for the installation to
 find them you may need to run::
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -58,16 +58,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2015, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -44,6 +44,7 @@
   Seo Sanghyeon
   Ronny Pfannschmidt
   Justin Peel
+  Raffael Tfirst
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
@@ -87,7 +88,6 @@
   Wenzhu Man
   John Witulski
   Laurence Tratt
-  Raffael Tfirst
   Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
@@ -111,6 +111,7 @@
   tav
   Taavi Burns
   Georg Brandl
+  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -181,6 +182,7 @@
   Vaibhav Sood
   Alan McIntyre
   Alexander Sedov
+  p_zieschang at yahoo.de
   Attila Gobi
   Jasper.Schulz
   Christopher Pope
@@ -191,6 +193,7 @@
   Arjun Naik
   Valentina Mukhamedzhanova
   Stefano Parmesan
+  touilleMan
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
@@ -199,12 +202,14 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
+  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Omer Katz
+  Mark Williams
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -325,9 +330,12 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  pizi
   Andrey Churin
   Dan Crosta
+  Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
+  werat
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
 
 The garbage collectors used or implemented by PyPy are not based on
 reference counting, so the objects are not freed instantly when they are no
-longer reachable.  The most obvious effect of this is that files are not
+longer reachable.  The most obvious effect of this is that files (and sockets, etc) are not
 promptly closed when they go out of scope.  For files that are opened for
 writing, data can be left sitting in their output buffers for a while, making
 the on-disk file appear empty or truncated.  Moreover, you might reach your
 OS's limit on the number of concurrently opened files.
 
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
 reference-counting approach to garbage collection.  The effect that you
 get in CPython has clearly been described as a side-effect of the
 implementation and not a language design decision: programs relying on
-this are basically bogus.  It would anyway be insane to try to enforce
+this are basically bogus.  It would be a too strong restriction to try to enforce
 CPython's behavior in a language spec, given that it has no chance to be
 adopted by Jython or IronPython (or any other port of Python to Java or
 .NET).
@@ -134,7 +141,7 @@
 
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython).  It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython).  It also means that
 **weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -335,3 +335,65 @@
 
 This will disable SELinux's protection and allow PyPy to configure correctly.
 Be sure to enable it again if you need it!
+
+
+How should I report a bug?
+--------------------------
+
+Our bug tracker is here: https://bitbucket.org/pypy/pypy/issues/
+
+Missing features or incompatibilities with CPython are considered
+bugs, and they are welcome.  (See also our list of `known
+incompatibilities`__.)
+
+.. __: http://pypy.org/compat.html
+
+For bugs of the kind "I'm getting a PyPy crash or a strange
+exception", please note that: **We can't do anything without
+reproducing the bug ourselves**.  We cannot do anything with
+tracebacks from gdb, or core dumps.  This is not only because the
+standard PyPy is compiled without debug symbols.  The real reason is
+that a C-level traceback is usually of no help at all in PyPy.
+Debugging PyPy can be annoying.
+
+`This is a clear and useful bug report.`__  (Admittedly, sometimes
+the problem is really hard to reproduce, but please try to.)
+
+.. __: https://bitbucket.org/pypy/pypy/issues/2363/segfault-in-gc-pinned-object-in
+
+In more details:
+
+* First, please give the exact PyPy version, and the OS.
+
+* It might help focus our search if we know if the bug can be
+  reproduced on a "``pypy --jit off``" or not.  If "``pypy --jit
+  off``" always works, then the problem might be in the JIT.
+  Otherwise, we know we can ignore that part.
+
+* If you got the bug using only Open Source components, please give a
+  step-by-step guide that we can follow to reproduce the problem
+  ourselves.  Don't assume we know anything about any program other
+  than PyPy.  We would like a guide that we can follow point by point
+  (without guessing or having to figure things out)
+  on a machine similar to yours, starting from a bare PyPy, until we
+  see the same problem.  (If you can, you can try to reduce the number
+  of steps and the time it needs to run, but that is not mandatory.)
+
+* If the bug involves Closed Source components, or just too many Open
+  Source components to install them all ourselves, then maybe you can
+  give us some temporary ssh access to a machine where the bug can be
+  reproduced.  Or, maybe we can download a VirtualBox or VMWare
+  virtual machine where the problem occurs.
+
+* If giving us access would require us to use tools other than ssh,
+  make appointments, or sign a NDA, then we can consider a commerical
+  support contract for a small sum of money.
+
+* If even that is not possible for you, then sorry, we can't help.
+
+Of course, you can try to debug the problem yourself, and we can help
+you get started if you ask on the #pypy IRC channel, but be prepared:
+debugging an annoying PyPy problem usually involves quite a lot of gdb
+in auto-generated C code, and at least some knowledge about the
+various components involved, from PyPy's own RPython source code to
+the GC and possibly the JIT.
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
     Defaults to 1/2 of your cache or ``4M``.
     Small values (like 1 or 1KB) are useful for debugging.
 
-``PYPY_GC_NURSERY_CLEANUP``
-    The interval at which nursery is cleaned up. Must
-    be smaller than the nursery size and bigger than the
-    biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+    If set to non-zero, will fill nursery with garbage, to help
+    debugging.
 
 ``PYPY_GC_INCREMENT_STEP``
     The size of memory marked during the marking step.  Default is size of
@@ -62,3 +61,8 @@
     use.
     Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
     on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+    The maximal number of pinned objects at any point in time.  Defaults
+    to a conservative value depending on nursery size and maximum object
+    size inside the nursery.  Useful for debugging by setting it to 0.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
    release-5.1.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
    whatsnew-5.1.0.rst
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -39,17 +39,16 @@
 library.
 
 If you want to install 3rd party libraries, the most convenient way is
-to install pip_ (unless you want to install virtualenv as explained
-below; then you can directly use pip inside virtualenvs):
+to install pip_ using ensurepip_ (unless you want to install virtualenv as 
+explained below; then you can directly use pip inside virtualenvs):
 
 .. code-block:: console
 
-    $ curl -O https://bootstrap.pypa.io/get-pip.py
-    $ ./pypy-2.1/bin/pypy get-pip.py
-    $ ./pypy-2.1/bin/pip install pygments  # for example
+    $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install pygments  # for example
 
-Third party libraries will be installed in ``pypy-2.1/site-packages``, and
-the scripts in ``pypy-2.1/bin``.
+Third party libraries will be installed in ``pypy-xxx/site-packages``, and
+the scripts in ``pypy-xxx/bin``.
 
 
 Installing using virtualenv
@@ -61,7 +60,7 @@
 checkout::
 
 	# from a tarball
-	$ virtualenv -p /opt/pypy-c-jit-41718-3fb486695f20-linux/bin/pypy my-pypy-env
+	$ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env
 
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
@@ -69,7 +68,7 @@
 Note that bin/python is now a symlink to bin/pypy.
 
 .. _pip: http://pypi.python.org/pypi/pip
-
+.. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
 
 Building PyPy yourself
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
  pypy
 ======
 
+.. note: this is turned into a regular man page "pypy.1" by
+   doing "make man" in pypy/doc/
+
 SYNOPSIS
 ========
 
@@ -48,6 +51,10 @@
 -B
     Disable writing bytecode (``.pyc``) files.
 
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
 --version
     Print the PyPy version.
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -57,7 +57,7 @@
 --------------
 
 Our cpyext C-API compatiblity layer can now run upstream NumPy unmodified.
-Release PyPy2.7-v5.3 still fails about 200 of the ~6000 test in the NumPy
+Release PyPy2.7-v5.4 still fails about 60 of the ~6000 test in the NumPy
 test suite. We could use help analyzing the failures and fixing them either
 as patches to upstream NumPy, or as fixes to PyPy.
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -0,0 +1,218 @@
+============
+PyPy2.7 v5.4
+============
+
+We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3.
+This new PyPy2.7 release includes incremental improvements to our C-API
+compatability layer (cpyext), enabling us to pass over 99% of the upstream
+numpy `test suite`_. We updated built-in cffi_ support to version 1.8,
+which now supports the "limited API" mode for c-extensions on 
+CPython >=3.2.
+
+We improved tooling for the PyPy JIT_, and expanded VMProf
+support to OpenBSD and Dragon Fly BSD
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the PyPy2.7 v5.4 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _JIT: https://morepypy.blogspot.com.au/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.3 released in June 2016)
+=========================================================
+
+* New features:
+
+  * Add `sys.{get,set}dlopenflags`
+
+  * Improve CPython compatibility of 'is' for small and empty strings
+
+  * Support for rgc.FinalizerQueue in the Boehm garbage collector
+
+  * (RPython) support spawnv() if it is called in C `_spawnv` on windows
+
+  * Fill in more slots when creating a PyTypeObject from a W_TypeObject,
+    like `__hex__`, `__sub__`, `__pow__`
+
+  * Copy CPython's logic more closely for `isinstance()` and
+    `issubclass()` as well as `type.__instancecheck__()` and
+    `type.__subclasscheck__()`
+
+  * Expose the name of CDLL objects
+
+  * Rewrite the win32 dependencies of `subprocess` to use cffi
+    instead of ctypes
+
+  * Improve the `JIT logging`_ facitilities
+
+  * (RPython) make int * string work
+
+  * Allocate all RPython strings with one extra byte, normally
+    unused. This now allows `ffi.from_buffer(string)` in CFFI with
+    no copy
+
+  * Adds a new commandline option `-X track-resources` that will
+    produce a `ResourceWarning` when the GC closes a file or socket.
+    The traceback for the place where the file or socket was allocated
+    is given as well, which aids finding places where `close()` is
+    missing
+
+  * Add missing `PyObject_Realloc`, `PySequence_GetSlice`
+
+  * `type.__dict__` now returns a `dict_proxy` object, like on CPython.
+    Previously it returned what looked like a regular dict object (but
+    it was already read-only)
+
+  * (RPython) add `rposix.{get,set}_inheritable()`, needed by Python 3.5
+
+  * (RPython) add `rposix_scandir` portably, needed for Python 3.5
+
+  * Increased but incomplete support for memoryview attributes (format, 
+    itemsize, ...) which also adds support for `PyMemoryView_FromObject`
+
+* Bug Fixes
+
+  * Reject `mkdir()` in read-only sandbox filesystems
+
+  * Add include guards to pymem.h to enable c++ compilation
+
+  * Fix build breakage on OpenBSD and FreeBSD
+
+  * Support OpenBSD, Dragon Fly BSD in VMProf
+
+  * Fix for `bytearray('').replace('a', 'ab')` for empty strings
+
+  * Sync internal state before calling `PyFile_AsFile()`
+
+  * Allow writing to a char* from `PyString_AsString()` until it is
+    forced, also refactor `PyStringObject` to look like CPython's
+    and allow subclassing `PyString_Type` and `PyUnicode_Type`
+
+  * Rpython rffi's socket(2) wrapper did not preserve errno
+
+  * Refactor `PyTupleObject` to look like CPython's and allow
+    subclassing `PyTuple_Type`
+
+  * Allow c-level assignment to a function pointer in a C-API
+    user-defined type after calling PyTypeReady by retrieving
+    a pointer to the function via offsets
+    rather than storing the function pointer itself
+
+  * Use `madvise(MADV_FREE)`, or if that doesn't exist
+    `MADV_DONTNEED` on freed arenas to release memory back to the
+    OS for resource monitoring
+
+  * Fix overflow detection in conversion of float to 64-bit integer
+    in timeout argument to various thread/threading primitives
+
+  * Fix win32 outputting `\r\r\n` in some cases
+
+  * Make `hash(-1)` return -2, as CPython does, and fix all the
+    ancilary places this matters
+
+  * Fix `PyNumber_Check()` to behave more like CPython
+
+  * (VMProf) Try hard to not miss any Python-level frame in the
+    captured stacks, even if there is metainterp or blackhole interp
+    involved.  Also fix the stacklet (greenlet) support
+
+  * Fix a critical JIT bug where `raw_malloc` -equivalent functions
+    lost the additional flags
+
+  * Fix the mapdict cache for subclasses of builtin types that
+    provide a dict
+
+  * Issues reported with our previous release were resolved_ after
+    reports from users on our issue tracker at
+    https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy
+
+* Performance improvements:
+
+  * Add a before_call()-like equivalent before a few operations like
+   `malloc_nursery`, to move values from registers into other registers
+    instead of to the stack.
+
+  * More tightly pack the stack when calling with `release gil`
+
+  * Support `int_floordiv()`, `int_mod()` in the JIT more efficiently
+    and add `rarithmetic.int_c_div()`, `rarithmetic.int_c_mod()` as
+    explicit interfaces. Clarify that `int_floordiv()` does python-style
+    rounding, unlike `llop.int_floordiv()`.
+
+  * Use `ll_assert` (more often) in incminimark
+
+  * (Testing) Simplify handling of interp-level tests and make it
+    more forward-compatible. Don't use interp-level RPython
+    machinery to test building app-level extensions in cpyext
+
+  * Constant-fold `ffi.offsetof("structname", "fieldname")` in cffi
+    backend
+
+  * Avoid a case in the JIT, where successive guard failures in
+    the same Python function end up as successive levels of
+    RPython functions, eventually exhausting the stack, while at
+    app-level the traceback is very short
+
+  * Check for NULL returns from calls to the raw-malloc and raise,
+    rather than a guard
+
+  * Improve `socket.recvfrom()` so that it copies less if possible
+
+  * When generating C code, inline `goto` to blocks with only one
+    predecessor, generating less lines of code
+
+  * When running the final backend-optimization phase before emitting
+    C code, constant-fold calls to we_are_jitted to return False. This
+    makes the generated C code a few percent smaller
+
+  * Refactor the `uid_t/gid_t` handling in `rlib.rposix` and in
+    `interp_posix.py`, based on the clean-up of CPython 2.7.x 
+
+.. _`JIT logging`: https://morepypy.blogspot.com/2016/08/pypy-tooling-upgrade-jitviewer-and.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.4.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,55 +1,9 @@
 ==========================
-What's new in PyPy2.7 5.3+
+What's new in PyPy2.7 5.4+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.3
-.. startrev: 873218a739f1
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
 
-.. 418b05f95db5
-Improve CPython compatibility for ``is``. Now code like ``if x is ():``
-works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
-
-.. pull request #455
-Add sys.{get,set}dlopenflags, for cpyext extensions.
-
-.. branch: fix-gen-dfa
-
-Resolves an issue with the generator script to build the dfa for Python syntax.
-
-.. branch: z196-support
-
-Fixes a critical issue in the register allocator and extends support on s390x.
-PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
-and z196 (released August 2010) in addition to zEC12 and z13.
-To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
-
-.. branch: s390x-5.3-catchup
-
-Implement the backend related changes for s390x.
-
-.. branch: incminimark-ll_assert
-.. branch: vmprof-openbsd
-
-.. branch: testing-cleanup
-
-Simplify handling of interp-level tests and make it more forward-
-compatible.
-
-.. branch: pyfile-tell
-Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
-
-.. branch: rw-PyString_AS_STRING
-Allow rw access to the char* returned from PyString_AS_STRING, also refactor
-PyStringObject to look like cpython's and allow subclassing PyString_Type and
-PyUnicode_Type
-
-.. branch: save_socket_errno
-
-Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
-the errno of the failing system call, but instead some random previous
-errno.
-
-.. branch: PyTuple_Type-subclass
-
-Refactor PyTupleObject to look like cpython's and allow subclassing 
-PyTuple_Type
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -0,0 +1,165 @@
+=========================
+What's new in PyPy2.7 5.4
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.3
+.. startrev: 873218a739f1
+
+.. 418b05f95db5
+Improve CPython compatibility for ``is``. Now code like ``if x is ():``
+works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
+
+.. pull request #455
+Add sys.{get,set}dlopenflags, for cpyext extensions.
+
+.. branch: fix-gen-dfa
+
+Resolves an issue with the generator script to build the dfa for Python syntax.
+
+.. branch: z196-support
+
+Fixes a critical issue in the register allocator and extends support on s390x.
+PyPy runs and translates on the s390x revisions z10 (released February 2008, experimental)
+and z196 (released August 2010) in addition to zEC12 and z13.
+To target e.g. z196 on a zEC12 machine supply CFLAGS="-march=z196" to your shell environment.
+
+.. branch: s390x-5.3-catchup
+
+Implement the backend related changes for s390x.
+
+.. branch: incminimark-ll_assert
+.. branch: vmprof-openbsd
+
+.. branch: testing-cleanup
+
+Simplify handling of interp-level tests and make it more forward-
+compatible.
+
+.. branch: pyfile-tell
+Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
+
+.. branch: rw-PyString_AS_STRING
+Allow rw access to the char* returned from PyString_AS_STRING, also refactor
+PyStringObject to look like cpython's and allow subclassing PyString_Type and
+PyUnicode_Type
+
+.. branch: save_socket_errno
+
+Bug fix: if ``socket.socket()`` failed, the ``socket.error`` did not show
+the errno of the failing system call, but instead some random previous
+errno.
+
+.. branch: PyTuple_Type-subclass
+
+Refactor PyTupleObject to look like cpython's and allow subclassing 
+PyTuple_Type
+
+.. branch: call-via-pyobj
+
+Use offsets from PyTypeObject to find actual c function to call rather than
+fixed functions, allows function override after PyType_Ready is called
+
+.. branch: issue2335
+
+Avoid exhausting the stack in the JIT due to successive guard
+failures in the same Python function ending up as successive levels of
+RPython functions, while at app-level the traceback is very short
+
+.. branch: use-madv-free
+
+Try harder to memory to the OS.  See e.g. issue #2336.  Note that it does
+not show up as a reduction of the VIRT column in ``top``, and the RES
+column might also not show the reduction, particularly on Linux >= 4.5 or
+on OS/X: it uses MADV_FREE, which only marks the pages as returnable to
+the OS if the memory is low.
+
+.. branch: cpyext-slotdefs2
+
+Fill in more slots when creating a PyTypeObject from a W_TypeObject
+More slots are still TBD, like tp_print and richcmp
+
+.. branch: json-surrogates
+
+Align json module decode with the cpython's impl, fixes issue 2345
+
+.. branch: issue2343
+
+Copy CPython's logic more closely for handling of ``__instancecheck__()``
+and ``__subclasscheck__()``.  Fixes issue 2343.
+
+.. branch: msvcrt-cffi
+
+Rewrite the Win32 dependencies of 'subprocess' to use cffi instead
+of ctypes. This avoids importing ctypes in many small programs and
+scripts, which in turn avoids enabling threads (because ctypes
+creates callbacks at import time, and callbacks need threads).
+
+.. branch: new-jit-log
+
+The new logging facility that integrates with and adds features to vmprof.com.
+
+.. branch: jitlog-32bit
+
+Resolve issues to use the new logging facility on a 32bit system
+
+.. branch: ep2016sprint
+
+Trying harder to make hash(-1) return -2, like it does on CPython
+
+.. branch: jitlog-exact-source-lines
+
+Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI.  Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
+
+.. branch: cpyext-realloc
+
+Implement PyObject_Realloc
+
+.. branch: inline-blocks
+
+Improve a little bit the readability of the generated C code
+
+.. branch: improve-vmprof-testing
+
+Improved vmprof support: now tries hard to not miss any Python-level
+frame in the captured stacks, even if there is the metainterp or
+blackhole interp involved.  Also fix the stacklet (greenlet) support.
+
+.. branch: py2-mappingproxy
+
+``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython.
+Previously it returned what looked like a regular dict object (but it
+was already read-only).
+
+
+.. branch: const-fold-we-are-jitted
+
+Reduce the size of the generated C code by constant-folding ``we_are_jitted``
+in non-jitcode.
+
+.. branch: memoryview-attributes
+
+Support for memoryview attributes (format, itemsize, ...).
+Extends the cpyext emulation layer.
+
+.. branch: redirect-assembler-jitlog
+
+Log more information to properly rebuild the redirected traces in jitviewer.
+
+.. branch: cpyext-subclass
+
+Copy Py_TPFLAGS_CHECKTYPES, Py_TPFLAGS_HAVE_INPLACEOPS when inheriting
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -24,11 +24,15 @@
 -V     : print the Python version number and exit (also --version)
 -W arg : warning control; arg is action:message:category:module:lineno
          also PYTHONWARNINGS=arg
+-X arg : set implementation-specific option
 file   : program read from script file
 -      : program read from stdin (default; interactive mode if a tty)
 arg ...: arguments passed to program in sys.argv[1:]
+
 PyPy options and arguments:
 --info : print translation information about this PyPy executable
+-X track-resources : track the creation of files and sockets and display
+                     a warning if they are not closed explicitly
 """
 # Missing vs CPython: PYTHONHOME, PYTHONCASEOK
 USAGE2 = """
@@ -229,6 +233,14 @@
         import pypyjit
         pypyjit.set_param(jitparam)
 
+def set_runtime_options(options, Xparam, *args):
+    if Xparam == 'track-resources':
+        sys.pypy_set_track_resources(True)
+    else:
+        print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),)
+        print >> sys.stderr, '[options] can be: track-resources'
+        raise SystemExit
+
 class CommandLineError(Exception):
     pass
 
@@ -404,6 +416,7 @@
     '--info':    (print_info,      None),

From pypy.commits at gmail.com  Tue Sep 20 07:54:15 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 20 Sep 2016 04:54:15 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: merge default
Message-ID: <57e12367.6740c20a.e15db.7a2a@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r87252:d983a5a6f8ed
Date: 2016-09-20 13:53 +0200
http://bitbucket.org/pypy/pypy/changeset/d983a5a6f8ed/

Log:	merge default

diff too long, truncating to 2000 out of 5948 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -30,3 +30,6 @@
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -369,3 +369,109 @@
   Roman Podoliaka
   Dan Loewenherz
   werat
+
+  Heinrich-Heine University, Germany 
+  Open End AB (formerly AB Strakt), Sweden
+  merlinux GmbH, Germany 
+  tismerysoft GmbH, Germany 
+  Logilab Paris, France 
+  DFKI GmbH, Germany 
+  Impara, Germany
+  Change Maker, Sweden 
+  University of California Berkeley, USA
+  Google Inc.
+  King's College London
+
+The PyPy Logo as used by http://speed.pypy.org and others was created
+by Samuel Reis and is distributed on terms of Creative Commons Share Alike
+License.
+ 
+License for 'lib-python/2.7'
+============================
+
+Except when otherwise stated (look for LICENSE files or copyright/license
+information at the beginning of each file) the files in the 'lib-python/2.7'
+directory are all copyrighted by the Python Software Foundation and licensed
+under the terms that you can find here: https://docs.python.org/2/license.html
+
+License for 'pypy/module/unicodedata/'
+======================================
+
+The following files are from the website of The Unicode Consortium
+at http://www.unicode.org/.  For the terms of use of these files, see
+http://www.unicode.org/terms_of_use.html .  Or they are derived from
+files from the above website, and the same terms of use apply.
+
+    CompositionExclusions-*.txt
+    EastAsianWidth-*.txt
+    LineBreak-*.txt
+    UnicodeData-*.txt
+    UnihanNumeric-*.txt
+
+License for 'dotviewer/font/'
+=============================
+
+Copyright (C) 2008 The Android Open Source Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+  
+     http://www.apache.org/licenses/LICENSE-2.0
+  
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Detailed license information is contained in the NOTICE file in the
+directory.
+
+
+Licenses and Acknowledgements for Incorporated Software
+=======================================================
+
+This section is an incomplete, but growing list of licenses and
+acknowledgements for third-party software incorporated in the PyPy
+distribution.
+
+License for 'Tcl/Tk'
+--------------------
+
+This copy of PyPy contains library code that may, when used, result in
+the Tcl/Tk library to be loaded.  PyPy also includes code that may be
+regarded as being a copy of some parts of the Tcl/Tk header files.
+You may see a copy of the License for Tcl/Tk in the file
+`lib_pypy/_tkinter/license.terms` included here.
+
+License for 'bzip2'
+-------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+bzip2 library.  You may see a copy of the License for bzip2/libbzip2 at
+
+    http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html
+
+License for 'openssl'
+---------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+openssl library.  You may see a copy of the License for OpenSSL at
+
+    https://www.openssl.org/source/license.html
+
+License for 'gdbm'
+------------------
+
+The gdbm module includes code from gdbm.h, which is distributed under
+the terms of the GPL license version 2 or any later version.  Thus the
+gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
+under the terms of the GPL license as well.
+
+License for 'rpython/rlib/rvmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://github.com/gperftools/gperftools/blob/master/COPYING
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
     """ Collector for test methods. """
     def collect(self):
         if hasinit(self.obj):
-            pytest.skip("class %s.%s with __init__ won't get collected" % (
+            # XXX used to be skip(), but silently skipping classes
+            # XXX just because they have been written long ago is
+            # XXX imho a very, very, very bad idea
+            pytest.fail("class %s.%s with __init__ won't get collected" % (
                 self.obj.__module__,
                 self.obj.__name__,
             ))
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -13,6 +13,7 @@
 import sys
 import os
 import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -62,8 +63,7 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -75,8 +75,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,7 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break
+                break        
 
     if args:
         vals = []
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.0
+Version: 1.8.3
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.3"
+__version_info__ = (1, 8, 3)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
 #define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+   Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+   Py_REF_DEBUG are not defined.  This is a best-effort approximation:
+   we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+   the same works for the other two macros.  Py_DEBUG implies them,
+   but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+#  include <pyconfig.h>
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#    define Py_LIMITED_API
+#  endif
+#endif
+
 #include <Python.h>
 #ifdef __cplusplus
 extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.0"
+                               "\ncompiled with cffi version: 1.8.3"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
         recompile(self, module_name, source,
                   c_file=filename, call_c_compiler=False, **kwds)
 
-    def compile(self, tmpdir='.', verbose=0, target=None):
+    def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
         """The 'target' argument gives the final file name of the
         compiled DLL.  Use '*' to force distutils' choice, suitable for
         regular CPython C API modules.  Use a file name ending in '.*'
@@ -669,7 +669,7 @@
         module_name, source, source_extension, kwds = self._assigned_source
         return recompile(self, module_name, source, tmpdir=tmpdir,
                          target=target, source_extension=source_extension,
-                         compiler_verbose=verbose, **kwds)
+                         compiler_verbose=verbose, debug=debug, **kwds)
 
     def init_once(self, func, tag):
         # Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,43 @@
         assert onerror is None   # XXX not implemented
         return BType(source, error)
 
+    _weakref_cache_ref = None
+
     def gcp(self, cdata, destructor):
-        BType = self.typeof(cdata)
+        if self._weakref_cache_ref is None:
+            import weakref
+            class MyRef(weakref.ref):
+                def __eq__(self, other):
+                    myref = self()
+                    return self is other or (
+                        myref is not None and myref is other())
+                def __ne__(self, other):
+                    return not (self == other)
+                def __hash__(self):
+                    try:
+                        return self._hash
+                    except AttributeError:
+                        self._hash = hash(self())
+                        return self._hash
+            self._weakref_cache_ref = {}, MyRef
+        weak_cache, MyRef = self._weakref_cache_ref
 
         if destructor is None:
-            if not (hasattr(BType, '_gcp_type') and
-                    BType._gcp_type is BType):
+            try:
+                del weak_cache[MyRef(cdata)]
+            except KeyError:
                 raise TypeError("Can remove destructor only on a object "
                                 "previously returned by ffi.gc()")
-            cdata._destructor = None
             return None
 
-        try:
-            gcp_type = BType._gcp_type
-        except AttributeError:
-            class CTypesDataGcp(BType):
-                __slots__ = ['_orig', '_destructor']
-                def __del__(self):
-                    if self._destructor is not None:
-                        self._destructor(self._orig)
-            gcp_type = BType._gcp_type = CTypesDataGcp
-        new_cdata = self.cast(gcp_type, cdata)
-        new_cdata._orig = cdata
-        new_cdata._destructor = destructor
+        def remove(k):
+            cdata, destructor = weak_cache.pop(k, (None, None))
+            if destructor is not None:
+                destructor(cdata)
+
+        new_cdata = self.cast(self.typeof(cdata), cdata)
+        assert new_cdata is not cdata
+        weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
         return new_cdata
 
     typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
         allsources.append(os.path.normpath(src))
     return Extension(name=modname, sources=allsources, **kwds)
 
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
     saved_environ = os.environ.copy()
     try:
-        outputfilename = _build(tmpdir, ext, compiler_verbose)
+        outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
         outputfilename = os.path.abspath(outputfilename)
     finally:
         # workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
                 os.environ[key] = value
     return outputfilename
 
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
     # XXX compact but horrible :-(
     from distutils.core import Distribution
     import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
     dist = Distribution({'ext_modules': [ext]})
     dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
+    if debug is None:
+        debug = sys.flags.debug
+    options['debug'] = ('ffiplatform', debug)
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
     options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
     def write_c_source_to_f(self, f, preamble):
         self._f = f
         prnt = self._prnt
-        if self.ffi._embedding is None:
-            prnt('#define Py_LIMITED_API')
+        if self.ffi._embedding is not None:
+            prnt('#define _CFFI_USE_EMBEDDING')
         #
         # first the '#include' (actually done by inlining the file's content)
         lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
 
 def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
               c_file=None, source_extension='.c', extradir=None,
-              compiler_verbose=1, target=None, **kwds):
+              compiler_verbose=1, target=None, debug=None, **kwds):
     if not isinstance(module_name, str):
         module_name = module_name.encode('ascii')
     if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
                 if target != '*':
                     _patch_for_target(patchlist, target)
                 os.chdir(tmpdir)
-                outputfilename = ffiplatform.compile('.', ext, compiler_verbose)
+                outputfilename = ffiplatform.compile('.', ext,
+                                                     compiler_verbose, debug)
             finally:
                 os.chdir(cwd)
                 _unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
     else:
         _add_c_module(dist, ffi, module_name, source, source_extension, kwds)
 
+def _set_py_limited_api(Extension, kwds):
+    """
+    Add py_limited_api to kwds if setuptools >= 26 is in use.
+    Do not alter the setting if it already exists.
+    Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+    """
+    if 'py_limited_api' not in kwds:
+        import setuptools
+        try:
+            setuptools_major_version = int(setuptools.__version__.partition('.')[0])
+            if setuptools_major_version >= 26:
+                kwds['py_limited_api'] = True
+        except ValueError:  # certain development versions of setuptools
+            # If we don't know the version number of setuptools, we
+            # try to set 'py_limited_api' anyway.  At worst, we get a
+            # warning.
+            kwds['py_limited_api'] = True
+    return kwds
 
 def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
     from distutils.core import Extension
-    from distutils.command.build_ext import build_ext
+    # We are a setuptools extension. Need this build_ext for py_limited_api.
+    from setuptools.command.build_ext import build_ext
     from distutils.dir_util import mkpath
     from distutils import log
     from cffi import recompiler
 
     allsources = ['$PLACEHOLDER']
     allsources.extend(kwds.pop('sources', []))
+    kwds = _set_py_limited_api(Extension, kwds)
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
     def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt
--- a/pypy/doc/config/translation.profopt.txt
+++ b/pypy/doc/config/translation.profopt.txt
@@ -3,3 +3,14 @@
 RPython program) to gather profile data. Example for pypy-c: "-c 'from
 richards import main;main(); from test import pystone;
 pystone.main()'"
+
+NOTE: be aware of what this does in JIT-enabled executables.  What it
+does is instrument and later optimize the C code that happens to run in
+the example you specify, ignoring any execution of the JIT-generated
+assembler.  That means that you have to choose the example wisely.  If
+it is something that will just generate assembler and stay there, there
+is little value.  If it is something that exercises heavily library
+routines that are anyway written in C, then it will optimize that.  Most
+interesting would be something that causes a lot of JIT-compilation,
+like running a medium-sized test suite several times in a row, in order
+to optimize the warm-up in general.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
    release-pypy2.7-v5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -0,0 +1,64 @@
+==========
+PyPy 5.4.1
+==========
+
+We have released a bugfix for PyPy2.7-v5.4.0, released last week,
+due to the following issues:
+
+  * Update list of contributors in documentation and LICENSE file,
+    this was unfortunately left out of 5.4.0. My apologies to the new
+    contributors
+
+  * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a
+    dynamically loadable file
+
+  * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython
+
+  * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
+
+  * Fix for an issue for translating the sandbox
+
+  * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
+    the last byte of a unicode string sometimes
+
+  * Update built-in cffi_ to version 1.8.1
+
+  * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
+    translation with a message asking for help porting it
+
+  * Fix a regression where a PyBytesObject was forced (converted to a RPython
+    object) when not required, reported as issue #2395
+
+Thanks to those who reported the issues.
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+This release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD),
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _cffi: https://cffi.readthedocs.io
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,17 @@
 
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -239,6 +239,10 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+            if config.translation.profopt is not None:
+                raise Exception("Cannot use the --profopt option "
+                                "when --shared is on (it is by default). "
+                                "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1430,6 +1430,9 @@
     BUF_FORMAT   = 0x0004
     BUF_ND       = 0x0008
     BUF_STRIDES  = 0x0010 | BUF_ND
+    BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+    BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+    BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
     BUF_INDIRECT = 0x0100 | BUF_STRIDES
 
     BUF_CONTIG_RO = BUF_ND
@@ -1978,6 +1981,7 @@
     'ZeroDivisionError',
     'RuntimeWarning',
     'PendingDeprecationWarning',
+    'UserWarning',
 ]
 
 if sys.platform.startswith("win"):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -63,7 +63,7 @@
         """x.__iter__() <==> iter(x)"""
         return self.space.wrap(self)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -264,25 +264,22 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
-            else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
-            #
             try:
+                if operr is not None:
+                    ec = self.space.getexecutioncontext()
+                    next_instr = self.handle_operation_error(ec, operr)
+                    self.last_instr = intmask(next_instr - 1)
+                else:
+                    # Execution starts just after the last_instr.  Initially,
+                    # last_instr is -1.  After a generator suspends it points to
+                    # the YIELD_VALUE instruction.
+                    next_instr = r_uint(self.last_instr + 1)
+                    if next_instr != 0:
+                        self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
-            executioncontext.return_trace(self, w_exitvalue)
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,12 +57,14 @@
         def f():
             yield 2
         g = f()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
 
     def test_throw3(self):
@@ -221,7 +223,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -562,3 +562,21 @@
         res = f(10).g()
         sys.settrace(None)
         assert res == 10
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.0"
+VERSION = "1.8.3"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer
 from pypy.module._cffi_backend import ctypeprim
 
 
@@ -22,6 +22,7 @@
     is_nonfunc_pointer_or_array = True
 
     def __init__(self, space, ctptr, length, arraysize, extra):
+        assert isinstance(ctptr, W_CTypePointer)
         W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
                                    ctptr.ctitem)
         self.length = length
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -35,8 +35,7 @@
         assert isinstance(ellipsis, bool)
         extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi)
         size = rffi.sizeof(rffi.VOIDP)
-        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult,
-                                could_cast_anything=False)
+        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult)
         self.fargs = fargs
         self.ellipsis = ellipsis
         self.abi = abi
@@ -59,6 +58,16 @@
                     lltype.free(self.cif_descr, flavor='raw')
                     self.cif_descr = lltype.nullptr(CIF_DESCRIPTION)
 
+    def is_unichar_ptr_or_array(self):
+        return False
+
+    def is_char_or_unichar_ptr_or_array(self):
+        return False
+
+    def string(self, cdataobj, maxlen):
+        # Can't use ffi.string() on a function pointer
+        return W_CType.string(self, cdataobj, maxlen)
+
     def new_ctypefunc_completing_argtypes(self, args_w):
         space = self.space
         nargs_declared = len(self.fargs)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -19,7 +19,6 @@
     # XXX this could be improved with an elidable method get_size()
     # that raises in case it's still -1...
 
-    cast_anything = False
     is_primitive_integer = False
     is_nonfunc_pointer_or_array = False
     is_indirect_arg_for_call_python = False
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -120,7 +120,6 @@
 
 class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar):
     _attrs_ = []
-    cast_anything = True
 
     def cast_to_int(self, cdata):
         return self.space.wrap(ord(cdata[0]))
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,12 +14,11 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _attrs_            = ['ctitem', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'accept_str', 'length']
     length = -1
 
-    def __init__(self, space, size, extra, extra_position, ctitem,
-                 could_cast_anything=True):
+    def __init__(self, space, size, extra, extra_position, ctitem):
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
@@ -27,10 +26,11 @@
         #  - for arrays, it is the array item type
         #  - for functions, it is the return type
         self.ctitem = ctitem
-        self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.accept_str = (self.can_cast_anything or
-                            (ctitem.is_primitive_integer and
-                             ctitem.size == rffi.sizeof(lltype.Char)))
+        self.accept_str = (self.is_nonfunc_pointer_or_array and
+                (isinstance(ctitem, ctypevoid.W_CTypeVoid) or
+                 isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or
+                 (ctitem.is_primitive_integer and
+                  ctitem.size == rffi.sizeof(lltype.Char))))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -137,7 +137,10 @@
 
 class W_CTypePtrBase(W_CTypePtrOrArray):
     # base class for both pointers and pointers-to-functions
-    _attrs_ = []
+    _attrs_ = ['is_void_ptr', 'is_voidchar_ptr']
+    _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr']
+    is_void_ptr = False
+    is_voidchar_ptr = False
 
     def convert_to_object(self, cdata):
         ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0]
@@ -154,7 +157,16 @@
             else:
                 raise self._convert_error("compatible pointer", w_ob)
         if self is not other:
-            if not (self.can_cast_anything or other.can_cast_anything):
+            if self.is_void_ptr or other.is_void_ptr:
+                pass     # cast from or to 'void *'
+            elif self.is_voidchar_ptr or other.is_voidchar_ptr:
+                space = self.space
+                msg = ("implicit cast from '%s' to '%s' "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" % (other.name, self.name))
+                space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1)
+            else:
                 raise self._convert_error("compatible pointer", w_ob)
 
         rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
@@ -165,8 +177,8 @@
 
 
 class W_CTypePointer(W_CTypePtrBase):
-    _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types']
-    _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr']
+    _attrs_ = ['is_file', 'cache_array_type', '_array_types']
+    _immutable_fields_ = ['is_file', 'cache_array_type?']
     kind = "pointer"
     cache_array_type = None
     is_nonfunc_pointer_or_array = True
@@ -181,6 +193,8 @@
         self.is_file = (ctitem.name == "struct _IO_FILE" or
                         ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
+        self.is_voidchar_ptr = (self.is_void_ptr or
+                           isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar))
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init, allocator):
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
                 return True
         return False
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        pass
-
     def convert_from_object(self, cdata, w_ob):
         if not self._copy_from_same(cdata, w_ob):
             self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
     )
     def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self.force_lazy_struct()
-        self._check_only_one_argument_for_union(w_ob)
 
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
-            if len(lst_w) > len(self._fields_list):
-                raise oefmt(space.w_ValueError,
-                            "too many initializers for '%s' (got %d)",
-                            self.name, len(lst_w))
-            for i in range(len(lst_w)):
-                optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+            j = 0
+            for w_obj in lst_w:
+                try:
+                    while (self._fields_list[j].flags &
+                               W_CField.BF_IGNORE_IN_CTOR):
+                        j += 1
+                except IndexError:
+                    raise oefmt(space.w_ValueError,
+                                "too many initializers for '%s' (got %d)",
+                                self.name, len(lst_w))
+                optvarsize = self._fields_list[j].write_v(cdata, w_obj,
                                                           optvarsize)
+                j += 1
             return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
 class W_CTypeUnion(W_CTypeStructOrUnion):
     kind = "union"
 
-    def _check_only_one_argument_for_union(self, w_ob):
-        space = self.space
-        n = space.int_w(space.len(w_ob))
-        if n > 1:
-            raise oefmt(space.w_ValueError,
-                        "initializer for '%s': %d items given, but only one "
-                        "supported (use a dict if needed)", self.name, n)
-
 
 class W_CField(W_Root):
     _immutable_ = True
@@ -200,18 +194,21 @@
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
 
-    def __init__(self, ctype, offset, bitshift, bitsize):
+    BF_IGNORE_IN_CTOR = 0x01
+
+    def __init__(self, ctype, offset, bitshift, bitsize, flags):
         self.ctype = ctype
         self.offset = offset
         self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
         self.bitsize = bitsize
+        self.flags = flags       # BF_xxx
 
     def is_bitfield(self):
         return self.bitshift >= 0
 
-    def make_shifted(self, offset):
+    def make_shifted(self, offset, fflags):
         return W_CField(self.ctype, offset + self.offset,
-                        self.bitshift, self.bitsize)
+                        self.bitshift, self.bitsize, self.flags | fflags)
 
     def read(self, cdata):
         cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
     offset = interp_attrproperty('offset', W_CField),
     bitshift = interp_attrproperty('bitshift', W_CField),
     bitsize = interp_attrproperty('bitsize', W_CField),
+    flags = interp_attrproperty('flags', W_CField),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -7,7 +7,6 @@
 
 class W_CTypeVoid(W_CType):
     _attrs_ = []
-    cast_anything = True
     kind = "void"
 
     def __init__(self, space):
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -32,8 +32,8 @@
 @unwrap_spec(w_cdata=cdataobj.W_CData)
 def from_handle(space, w_cdata):
     ctype = w_cdata.ctype
-    if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or
-        not ctype.can_cast_anything):
+    if (not isinstance(ctype, ctypeptr.W_CTypePointer) or
+        not ctype.is_voidchar_ptr):
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
         if alignment < falign and do_align:
             alignment = falign
         #
+        if is_union and i > 0:
+            fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+        else:
+            fflags = 0
+        #
         if fbitsize < 0:
             # not a bitfield: common case
 
@@ -372,7 +377,7 @@
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
-                    fld = srcfld.make_shifted(boffset // 8)
+                    fld = srcfld.make_shifted(boffset // 8, fflags)
                     fields_list.append(fld)
                     try:
                         fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
                 w_ctype._custom_field_pos = True
             else:
                 # a regular field
-                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+                fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+                                           fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
@@ -489,7 +495,7 @@
                     bitshift = 8 * ftype.size - fbitsize- bitshift
 
                 fld = ctypestruct.W_CField(ftype, field_offset_bytes,
-                                           bitshift, fbitsize)
+                                           bitshift, fbitsize, fflags)
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
     assert d[2][1].bitshift == -1
     assert d[2][1].bitsize == -1
 
+def test_nested_anonymous_struct_2():
+    BInt = new_primitive_type("int")
+    BStruct = new_struct_type("struct foo")
+    BInnerUnion = new_union_type("union bar")
+    complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+                                           ('a2', BInt, -1)])
+    complete_struct_or_union(BStruct, [('b1', BInt, -1),
+                                       ('', BInnerUnion, -1),
+                                       ('b2', BInt, -1)])
+    assert sizeof(BInnerUnion) == sizeof(BInt)
+    assert sizeof(BStruct) == sizeof(BInt) * 3
+    fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+    assert fields == [
+        ('b1', 0 * sizeof(BInt), 0),
+        ('a1', 1 * sizeof(BInt), 0),
+        ('a2', 1 * sizeof(BInt), 1),
+        ('b2', 2 * sizeof(BInt), 0),
+    ]
+
 def test_sizeof_union():
     # a union has the largest alignment of its members, and a total size
     # that is the largest of its items *possibly further aligned* if
@@ -3646,3 +3665,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -503,3 +503,10 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_bug_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        q = ffi.new("char[]", "abcd")
+        p = ffi.cast("char(*)(void)", q)
+        raises(TypeError, ffi.string, p)
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -450,7 +450,12 @@
                 # For compatibility
                 assert exc.value.errno == _ssl.SSL_ERROR_WANT_READ
             finally:
-                c.shutdown()
+                try:
+                    c.shutdown()
+                except _ssl.SSLError:
+                    # If the expected exception was raised, the SSLContext
+                    # can't be shut down yet
+                    pass
         finally:
             s.close()
 
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,16 +30,19 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
+    w_initializer_type = None
+    w_initializer = None
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if space.type(w_initializer) is space.w_str:
+            if w_initializer is not None:
+                if w_initializer_type is space.w_str:
                     a.descr_fromstring(space, w_initializer)
-                elif space.type(w_initializer) is space.w_list:
+                elif w_initializer_type is space.w_list:
                     a.descr_fromlist(space, w_initializer)
                 else:
                     a.extend(w_initializer, True)
@@ -597,6 +600,18 @@
     def getlength(self):
         return self.array.len * self.array.itemsize
 
+    def getformat(self):
+        return self.array.typecode
+
+    def getitemsize(self):
+        return self.array.itemsize
+
+    def getndim(self):
+        return 1
+
+    def getstrides(self):
+        return [self.getitemsize()]
+
     def getitem(self, index):
         array = self.array
         data = array._charbuf_start()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,10 +119,10 @@
 
 constant_names = """
 Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,6 +645,9 @@
         ('format', rffi.CCHARP),
         ('shape', Py_ssize_tP),
         ('strides', Py_ssize_tP),
+        ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
+        ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+        ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
         ('suboffsets', Py_ssize_tP),
         #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
         ('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,7 +1,6 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
+    cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
@@ -13,33 +12,4 @@
         return 1
     return 0  
 
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
-             rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
-    """Export obj into a Py_buffer, view.  These arguments must
-    never be NULL.  The flags argument is a bit field indicating what
-    kind of buffer the caller is prepared to deal with and therefore what
-    kind of buffer the exporter is allowed to return.  The buffer interface
-    allows for complicated memory sharing possibilities, but some caller may
-    not be able to handle all the complexity but may want to see if the
-    exporter will let them take a simpler view to its memory.
-
-    Some exporters may not be able to share memory in every possible way and
-    may need to raise errors to signal to some consumers that something is
-    just not possible. These errors should be a BufferError unless
-    there is another error that is actually causing the problem. The
-    exporter can use flags information to simplify how much of the
-    Py_buffer structure is filled in with non-default values and/or
-    raise an error if the object can't support a simpler view of its memory.
-
-    0 is returned on success and -1 on error."""
-    raise oefmt(space.w_TypeError,
-                "PyPy does not yet implement the new buffer interface")
-
- at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
-    """Return 1 if the memory defined by the view is C-style (fortran is
-    'C') or Fortran-style (fortran is 'F') contiguous or either one
-    (fortran is 'A').  Return 0 otherwise."""
-    # PyPy only supports contiguous Py_buffers for now.
-    return 1
+    
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -14,45 +14,33 @@
 ## Implementation of PyBytesObject
 ## ================================
 ##
-## The problem
-## -----------
+## PyBytesObject has its own ob_sval buffer, so we have two copies of a string;
+## one in the PyBytesObject returned from various C-API functions and another
+## in the corresponding RPython object.
 ##
-## PyString_AsString() must return a (non-movable) pointer to the underlying
-## ob_sval, whereas pypy strings are movable.  C code may temporarily store
-## this address and use it, as long as it owns a reference to the PyObject.
-## There is no "release" function to specify that the pointer is not needed
-## any more.
+## The following calls can create a PyBytesObject without a correspoinding
+## RPython object:
 ##
-## Also, the pointer may be used to fill the initial value of string. This is
-## valid only when the string was just allocated, and is not used elsewhere.
+## PyBytes_FromStringAndSize(NULL, n) / PyString_FromStringAndSize(NULL, n)
 ##
-## Solution
-## --------
+## In the PyBytesObject returned, the ob_sval buffer may be modified as
+## long as the freshly allocated PyBytesObject is not "forced" via a call
+## to any of the more sophisticated C-API functions. 
 ##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
-##
-## - A string allocated by pypy will be converted into a PyBytesObject with a
-##   NULL buffer.  The first time PyString_AsString() is called, memory is
-##   allocated (with flavor='raw') and content is copied.
-##
-## - A string allocated with PyString_FromStringAndSize(NULL, size) will
-##   allocate a PyBytesObject structure, and a buffer with the specified
-##   size+1, but the reference won't be stored in the global map; there is no
-##   corresponding object in pypy.  When from_ref() or Py_INCREF() is called,
-##   the pypy string is created, and added to the global map of tracked
-##   objects.  The buffer is then supposed to be immutable.
-##
-##-  A buffer obtained from PyString_AS_STRING() could be mutable iff
-##   there is no corresponding pypy object for the string
-##
-## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
-##   similar object.
-##
-## - PyString_Size() doesn't need to force the object.
+## Care has been taken in implementing the functions below, so that
+## if they are called with a non-forced PyBytesObject, they will not 
+## unintentionally force the creation of a RPython object. As long as only these
+## are used, the ob_sval buffer is still modifiable:
+## 
+## PyBytes_AsString / PyString_AsString 
+## PyBytes_AS_STRING / PyString_AS_STRING
+## PyBytes_AsStringAndSize / PyString_AsStringAndSize
+## PyBytes_Size / PyString_Size
+## PyBytes_Resize / PyString_Resize
+## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
 ##
 ## - There could be an (expensive!) check in from_ref() that the buffer still
-##   corresponds to the pypy gc-managed string.
+##   corresponds to the pypy gc-managed string, 
 ##
 
 PyBytesObjectStruct = lltype.ForwardReference()
@@ -156,9 +144,6 @@
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
     ref_str = rffi.cast(PyBytesObject, ref)
-    if not pyobj_has_w_obj(ref):
-        # XXX Force the ref?
-        bytes_realize(space, ref)
     return ref_str.c_ob_sval
 
 @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
@@ -182,9 +167,6 @@
             raise oefmt(space.w_TypeError,
                         "expected string or Unicode object, %T found",
                         from_ref(space, ref))
-    if not pyobj_has_w_obj(ref):
-        # force the ref
-        bytes_realize(space, ref)
     ref_str = rffi.cast(PyBytesObject, ref)
     data[0] = ref_str.c_ob_sval
     if length:
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -142,7 +142,9 @@
 typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
 typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
 typedef struct bufferinfo {
     void *buf;
     PyObject *obj;        /* owned reference */
@@ -156,12 +158,14 @@
     char *format;
     Py_ssize_t *shape;
     Py_ssize_t *strides;
-    Py_ssize_t *suboffsets;
-
+    Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+    unsigned char _format[Py_MAX_FMT];
+    Py_ssize_t _strides[Py_MAX_NDIMS];
+    Py_ssize_t _shape[Py_MAX_NDIMS];
     /* static store for shape and strides of
        mono-dimensional buffers. */
     /* Py_ssize_t smalltable[2]; */
-    void *internal;
+    void *internal; /* always NULL for app-level objects */
 } Py_buffer;
 
 
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.4.1-alpha0"
-#define PYPY_VERSION_NUM  0x05040100
+#define PYPY_VERSION "5.5.0-alpha0"
+#define PYPY_VERSION_NUM  0x05050000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,6 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -28,25 +27,25 @@
     """Return a new PyLongObject object from a C size_t, or NULL on
     failure.
     """
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.LONGLONG], PyObject)
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
     """Return a new PyLongObject object from a C unsigned long, or
     NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONGLONG], PyObject)
 def PyLong_FromUnsignedLongLong(space, val):
     """Return a new PyLongObject object from a C unsigned long long,
     or NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([PyObject], rffi.ULONG, error=-1)
 def PyLong_AsUnsignedLong(space, w_long):
@@ -203,7 +202,10 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    value = rffi.cast(ADDR, p)    # signed integer
+    if value < 0:
+        return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
+    return space.wrap(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,10 +1,122 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
-                                    build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+                         Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
+from pypy.objspace.std.memoryobject import W_MemoryView
 
 PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
 
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+             rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+    """Export obj into a Py_buffer, view.  These arguments must
+    never be NULL.  The flags argument is a bit field indicating what
+    kind of buffer the caller is prepared to deal with and therefore what
+    kind of buffer the exporter is allowed to return.  The buffer interface
+    allows for complicated memory sharing possibilities, but some caller may
+    not be able to handle all the complexity but may want to see if the
+    exporter will let them take a simpler view to its memory.
+
+    Some exporters may not be able to share memory in every possible way and
+    may need to raise errors to signal to some consumers that something is
+    just not possible. These errors should be a BufferError unless
+    there is another error that is actually causing the problem. The
+    exporter can use flags information to simplify how much of the
+    Py_buffer structure is filled in with non-default values and/or
+    raise an error if the object can't support a simpler view of its memory.
+
+    0 is returned on success and -1 on error."""
+    flags = widen(flags)
+    buf = space.buffer_w(w_obj, flags)
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+    except ValueError:
+        raise BufferError("could not create buffer from object")
+    view.c_obj = make_ref(space, w_obj)
+    return fill_Py_buffer(space, buf, view)
+
+def fill_Py_buffer(space, buf, view):    
+    # c_buf, c_obj have been filled in
+    ndim = buf.getndim()
+    view.c_len = buf.getlength()
+    view.c_itemsize = buf.getitemsize()
+    rffi.setintfield(view, 'c_ndim', ndim)
+    view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+    view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+    view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+    fmt = buf.getformat()
+    n = Py_MAX_FMT - 1 # NULL terminated buffer
+    if len(fmt) > n:
+        ### WARN?
+        pass
+    else:
+        n = len(fmt)
+    for i in range(n):
+        view.c_format[i] = fmt[i]
+    view.c_format[n] = '\x00'        
+    shape = buf.getshape()
+    strides = buf.getstrides()
+    for i in range(ndim):
+        view.c_shape[i] = shape[i]
+        view.c_strides[i] = strides[i]
+    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+    return 0
+
+def _IsFortranContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(view.c_ndim):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(view):
+    ndim = widen(view.c_ndim)
+    if ndim == 0:
+        return 1
+    if not view.c_strides:
+        return ndim == 1
+    sd = view.c_itemsize
+    if ndim == 1:
+        return view.c_shape[0] == 1 or sd == view.c_strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = view.c_shape[i]
+        if dim == 0:
+            return 1
+        if view.c_strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+ at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+    """Return 1 if the memory defined by the view is C-style (fortran is
+    'C') or Fortran-style (fortran is 'F') contiguous or either one
+    (fortran is 'A').  Return 0 otherwise."""
+    # traverse the strides, checking for consistent stride increases from
+    # right-to-left (c) or left-to-right (fortran). Copied from cpython
+    if not view.c_suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(view)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(view)
+    elif (fort == 'A'):
+        return (_IsCContiguous(view) or _IsFortranContiguous(view))
+    return 0
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_FromObject(space, w_obj):
     return space.call_method(space.builtin, "memoryview", w_obj)
@@ -12,6 +124,7 @@
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+    # XXX needed for numpy on py3k
     raise NotImplementedError('PyMemoryView_GET_BUFFER')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -20,21 +133,22 @@
     object.  The object must be a memoryview instance; this macro doesn't
     check its type, you must do it yourself or you will risk crashes."""
     view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
-    # TODO - fill in fields
-    '''
-    view.c_buf = buf
-    view.c_len = length
-    view.c_obj = obj
-    Py_IncRef(space, obj)
-    view.c_itemsize = 1
-    rffi.setintfield(view, 'c_readonly', readonly)
-    rffi.setintfield(view, 'c_ndim', 0)
-    view.c_format = lltype.nullptr(rffi.CCHARP.TO)
-    view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
-    view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-    ''' 
+    if not isinstance(w_obj, W_MemoryView):
+        return view
+    ndim = w_obj.buf.getndim()
+    if ndim >= Py_MAX_NDIMS:
+        # XXX warn?
+        return view
+    try:
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_obj = make_ref(space, w_obj)
+        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        isstr = False
+    except ValueError:
+        w_s = w_obj.descr_tobytes(space)
+        view.c_obj = make_ref(space, w_s)
+        rffi.setintfield(view, 'c_readonly', 1)
+        isstr = True
+    fill_Py_buffer(space, w_obj.buf, view)     
     return view
 
-
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -508,10 +508,9 @@
 @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
 def PyBuffer_Release(space, view):
     """
-    Releases a Py_buffer obtained from getbuffer ParseTuple's s*.
-
-    This is not a complete re-implementation of the CPython API; it only
-    provides a subset of CPython's behavior.
+    Release the buffer view. This should be called when the buffer is 
+    no longer being used as it may free memory from it
     """
     Py_DecRef(space, view.c_obj)
     view.c_obj = lltype.nullptr(PyObject.TO)
+    # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
 def PySequence_Fast(space, w_obj, m):
     """Returns the sequence o as a tuple, unless it is already a tuple or list, in
     which case o is returned.  Use PySequence_Fast_GET_ITEM() to access the
-    members of the result.  Returns NULL on failure.  If the object is not a
-    sequence, raises TypeError with m as the message text."""
+    members of the result.  Returns NULL on failure.  If the object cannot be
+    converted to a sequence, and raises a TypeError, raise a new TypeError with
+    m as the message text. If the conversion otherwise, fails, reraise the
+    original exception"""
     if isinstance(w_obj, W_ListObject):
         # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM    
         w_obj.convert_to_cpy_strategy(space)
         return w_obj
     try:
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
-    except OperationError:
-        raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+        raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
 def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
     def getshape(self):
         return self.shape
 
+    def getstrides(self):
+        return self.strides
+
     def getitemsize(self):
         return self.itemsize
 
+    def getndim(self):
+        return self.ndim
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
 PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
 {
   PyMyArray* self = (PyMyArray*)obj;
-  fprintf(stdout, "in PyMyArray_getbuffer\n");
   if (view == NULL) {
-    fprintf(stdout, "view is NULL\n");
     PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
     return -1;
   }
   if (flags == 0) {
-    fprintf(stdout, "flags is 0\n");
     PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
     return -1;
   }
@@ -188,7 +185,131 @@
     (initproc)PyMyArray_init,     /* tp_init */
 };
 
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+    Py_buffer* view = NULL;
+    PyObject* obj = PyTuple_GetItem(args, 0);
+    PyObject* memoryview = PyMemoryView_FromObject(obj);
+    if (memoryview == NULL)
+        return PyInt_FromLong(-1);
+    view = PyMemoryView_GET_BUFFER(memoryview);
+    Py_DECREF(memoryview);
+    return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG)                                        \
+    buf_flag = PyUnicode_FromString(#FLAG);                         \
+    flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ);  \
+    Py_DECREF(buf_flag);                                            \
+    if (flag_matches == 1) {                                        \
+        Py_DECREF(tmp);                                             \
+        flags |= PyBUF_##FLAG;                                      \
+        continue;                                                   \
+    }                                                               \
+    else if (flag_matches == -1) {                                  \
+        Py_DECREF(tmp);                                             \
+        return NULL;                                                \
+    }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+    PyObject *buffer_obj, *pyflags;
+    PyObject *tmp, *buf_flag;
+    Py_buffer buffer;
+    PyObject *shape, *strides;
+    Py_ssize_t i, n;
+    int flag_matches;
+    int flags = 0;
+
+    if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+        return NULL;
+    }
+
+    n = PySequence_Length(pyflags);
+    if (n < 0) {
+        return NULL;
+    }
+
+    for (i=0; i < n; i++) {
+        tmp = PySequence_GetItem(pyflags, i);
+        if (tmp == NULL) {
+            return NULL;
+        }
+
+        GET_PYBUF_FLAG(SIMPLE);
+        GET_PYBUF_FLAG(WRITABLE);
+        GET_PYBUF_FLAG(STRIDES);
+        GET_PYBUF_FLAG(ND);
+        GET_PYBUF_FLAG(C_CONTIGUOUS);
+        GET_PYBUF_FLAG(F_CONTIGUOUS);
+        GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+        GET_PYBUF_FLAG(INDIRECT);
+        GET_PYBUF_FLAG(FORMAT);
+        GET_PYBUF_FLAG(STRIDED);
+        GET_PYBUF_FLAG(STRIDED_RO);
+        GET_PYBUF_FLAG(RECORDS);
+        GET_PYBUF_FLAG(RECORDS_RO);
+        GET_PYBUF_FLAG(FULL);
+        GET_PYBUF_FLAG(FULL_RO);
+        GET_PYBUF_FLAG(CONTIG);
+        GET_PYBUF_FLAG(CONTIG_RO);
+
+        Py_DECREF(tmp);
+
+        /* One of the flags must match */
+        PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+        return NULL;
+    }
+
+    if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+        return NULL;
+    }
+
+    if (buffer.shape == NULL) {
+        Py_INCREF(Py_None);
+        shape = Py_None;
+    }
+    else {
+        shape = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+        }
+    }
+
+    if (buffer.strides == NULL) {
+        Py_INCREF(Py_None);
+        strides = Py_None;
+    }
+    else {
+        strides = PyTuple_New(buffer.ndim);
+        for (i=0; i < buffer.ndim; i++) {
+            PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+        }
+    }
+
+    PyBuffer_Release(&buffer);
+    return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
 static PyMethodDef buffer_functions[] = {
+    {"test_buffer",   (PyCFunction)test_buffer, METH_VARARGS, NULL},
+    {"get_buffer_info",   (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
     {NULL,        NULL}    /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,9 +4,7 @@
 PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
 {
     PyObject* newType;
-    /*printf("in foo3type_tp_new, preprocessing...\n"); */
     newType = PyType_Type.tp_new(metatype, args, kwds);
-    /*printf("in foo3type_tp_new, postprocessing...\n"); */
     return newType;
 }
 
@@ -81,4 +79,5 @@
         return;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return;
+    Py_INCREF(&footype);
 }
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
 import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.api import PyObject
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -183,8 +183,27 @@
                  Py_INCREF(Py_None);
                  return Py_None;
              """),
+            ("c_only", "METH_NOARGS",
+            """
+                int ret;
+                char * buf2;
+                PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+                if (!obj)
+                    return NULL;
+                buf2 = PyBytes_AsString(obj);
+                if (!buf2)
+                    return NULL;
+                /* buf should not have been forced, issue #2395 */
+                ret = _PyBytes_Resize(&obj, 512);
+                if (ret < 0)
+                    return NULL;
+                 Py_DECREF(obj);
+                 Py_INCREF(Py_None);
+                 return Py_None;
+            """),
             ])
         module.getbytes()
+        module.c_only()
 
     def test_py_string_as_string_Unicode(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
 
 from .support import c_compile
 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
     1/0
@@ -53,7 +55,48 @@
         libraries=libraries)
     return soname
 
-def compile_extension_module(space, modname, include_dirs=[],
+class SystemCompilationInfo(object):
+    """Bundles all the generic information required to compile extensions.
+
+    Note: here, 'system' means OS + target interpreter + test config + ...
+    """
+    def __init__(self, include_extra=None, compile_extra=None, link_extra=None,
+            extra_libs=None, ext=None):
+        self.include_extra = include_extra or []
+        self.compile_extra = compile_extra
+        self.link_extra = link_extra
+        self.extra_libs = extra_libs
+        self.ext = ext
+
+def get_cpyext_info(space):
+    from pypy.module.imp.importing import get_so_extension
+    state = space.fromcache(State)
+    api_library = state.api_lib
+    if sys.platform == 'win32':
+        libraries = [api_library]
+        # '%s' undefined; assuming extern returning int
+        compile_extra = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+            (space.int_w(w_maj), space.int_w(w_min))]
+    else:
+        libraries = []
+        if sys.platform.startswith('linux'):
+            compile_extra = [
+                "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
+    return SystemCompilationInfo(
+        include_extra=api.include_dirs,
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        extra_libs=libraries,
+        ext=get_so_extension(space))
+
+
+def compile_extension_module(sys_info, modname, include_dirs=[],
         source_files=None, source_strings=None):
     """
     Build an extension module and return the filename of the resulting native
@@ -65,35 +108,15 @@
     Any extra keyword arguments are passed on to ExternalCompilationInfo to
     build the module (so specify your source with one of those).

From pypy.commits at gmail.com  Tue Sep 20 07:58:00 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 20 Sep 2016 04:58:00 -0700 (PDT)
Subject: [pypy-commit] pypy guard-compatible: fix fast isinstance check with
 guard_compatible
Message-ID: <57e12448.e97bc20a.d4c0e.7a07@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: guard-compatible
Changeset: r87253:ccf32f0e0a8d
Date: 2016-09-20 13:57 +0200
http://bitbucket.org/pypy/pypy/changeset/ccf32f0e0a8d/

Log:	fix fast isinstance check with guard_compatible

diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -102,7 +102,8 @@
     # PyObject_IsInstance() logic.
 
     # Quick test for an exact match
-    if space.type(w_obj) is w_klass_or_tuple:
+    # disabled with jit to make it not impose a strong type condition
+    if not jit.we_are_jitted() and space.type(w_obj) is w_klass_or_tuple:
         return True
 
     # -- case (anything, tuple)

From pypy.commits at gmail.com  Tue Sep 20 09:12:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 20 Sep 2016 06:12:47 -0700 (PDT)
Subject: [pypy-commit] cffi default: typedef int foo_t[...];
Message-ID: <57e135cf.ca11c30a.b514c.a1bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2775:8c38fb219286
Date: 2016-09-20 15:07 +0200
http://bitbucket.org/cffi/cffi/changeset/8c38fb219286/

Log:	typedef int foo_t[...];

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -332,7 +332,7 @@
                         realtype = model.unknown_ptr_type(decl.name)
                     else:
                         realtype, quals = self._get_type_and_quals(
-                            decl.type, name=decl.name)
+                            decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
                 else:
                     raise api.CDefError("unrecognized construct", decl)
@@ -781,11 +781,14 @@
                 exprnode.name in self._int_constants):
             return self._int_constants[exprnode.name]
         #
-        if partial_length_ok:
-            if (isinstance(exprnode, pycparser.c_ast.ID) and
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
                 self._partial_length = True
                 return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
         #
         raise api.FFIError(":%d: unsupported expression: expected a "
                            "simple numeric constant" % exprnode.coord.line)
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -587,8 +587,11 @@
     # ----------
     # typedefs
 
+    def _typedef_type(self, tp, name):
+        return self._global_type(tp, "(*(%s *)0)" % (name,))
+
     def _generate_cpy_typedef_collecttype(self, tp, name):
-        self._do_collect_type(tp)
+        self._do_collect_type(self._typedef_type(tp, name))
 
     def _generate_cpy_typedef_decl(self, tp, name):
         pass
@@ -598,6 +601,7 @@
         self._lsts["typename"].append(TypenameExpr(name, type_index))
 
     def _generate_cpy_typedef_ctx(self, tp, name):
+        tp = self._typedef_type(tp, name)
         self._typedef_ctx(tp, name)
         if getattr(tp, "origin", None) == "unknown_type":
             self._struct_ctx(tp, tp.name, approxname=None)
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -479,3 +479,7 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_negative_array_size(self):
+        ffi = FFI()
+        py.test.raises(ValueError, ffi.cast, "int[-5]", 0)
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -502,3 +502,7 @@
     assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
     p = ffi.new("int[]", [-123456789])
     assert ffi.unpack(p, 1) == [-123456789]
+
+def test_negative_array_size():
+    ffi = _cffi1_backend.FFI()
+    py.test.raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1980,3 +1980,29 @@
         static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
+
+def test_typedef_array_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef int foo_t[...], bar_t[...];
+        int gv[...];
+        typedef int mat_t[...][...];
+        typedef int vmat_t[][...];
+        """)
+    lib = verify(ffi, "test_typedef_array_dotdotdot", """
+        typedef int foo_t[50], bar_t[50];
+        int gv[23];
+        typedef int mat_t[6][7];
+        typedef int vmat_t[][8];
+    """)
+    assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+    assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+    assert len(ffi.new("foo_t")) == 50
+    assert len(ffi.new("bar_t")) == 50
+    assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+    assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+    assert len(ffi.new("mat_t")) == 6
+    assert len(ffi.new("mat_t")[3]) == 7
+    py.test.raises(ffi.error, ffi.sizeof, "vmat_t")
+    p = ffi.new("vmat_t", 4)
+    assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")

From pypy.commits at gmail.com  Tue Sep 20 09:12:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 20 Sep 2016 06:12:49 -0700 (PDT)
Subject: [pypy-commit] cffi default: update to 1.8.4 and add whatsnew
Message-ID: <57e135d1.a468c20a.69aee.9eb4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2776:c61b49c67c83
Date: 2016-09-20 15:12 +0200
http://bitbucket.org/cffi/cffi/changeset/c61b49c67c83/

Log:	update to 1.8.4 and add whatsnew

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.8.3"
+#define CFFI_VERSION  "1.8.4"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.3"
-__version_info__ = (1, 8, 3)
+__version__ = "1.8.4"
+__version_info__ = (1, 8, 4)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.3"
+                               "\ncompiled with cffi version: 1.8.4"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.8'
 # The full version, including alpha/beta/rc tags.
-release = '1.8.3'
+release = '1.8.4'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.3.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.4.tar.gz
 
-   - MD5: c8e877fe0426a99d0cf5872cf2f95b27
+   - MD5: ...
 
-   - SHA: 2a5be10205bf3f6b21070a71ee2546d838868172
+   - SHA: ...
 
-   - SHA256: c321bd46faa7847261b89c0469569530cad5a41976bb6dba8202c0159f476568
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,14 @@
 ======================
 
 
+v1.8.4
+======
+
+* Add support in ``cdef()/set_source()`` for unspecified-length arrays
+  in typedefs: ``typedef int foo_t[...];``.  It was already supported
+  for global variables or structure fields.
+
+
 v1.8.3
 ======
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.8.3',
+        version='1.8.4',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Tue Sep 20 09:24:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 20 Sep 2016 06:24:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/c61b49c67c83
Message-ID: <57e13878.43e61c0a.64418.789c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87254:e0a099a466a1
Date: 2016-09-20 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/e0a099a466a1/

Log:	update to cffi/c61b49c67c83

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.3
+Version: 1.8.4
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.3"
-__version_info__ = (1, 8, 3)
+__version__ = "1.8.4"
+__version_info__ = (1, 8, 4)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.3"
+                               "\ncompiled with cffi version: 1.8.4"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -332,7 +332,7 @@
                         realtype = model.unknown_ptr_type(decl.name)
                     else:
                         realtype, quals = self._get_type_and_quals(
-                            decl.type, name=decl.name)
+                            decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
                 else:
                     raise api.CDefError("unrecognized construct", decl)
@@ -781,11 +781,14 @@
                 exprnode.name in self._int_constants):
             return self._int_constants[exprnode.name]
         #
-        if partial_length_ok:
-            if (isinstance(exprnode, pycparser.c_ast.ID) and
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
                 self._partial_length = True
                 return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
         #
         raise api.FFIError(":%d: unsupported expression: expected a "
                            "simple numeric constant" % exprnode.coord.line)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -587,8 +587,11 @@
     # ----------
     # typedefs
 
+    def _typedef_type(self, tp, name):
+        return self._global_type(tp, "(*(%s *)0)" % (name,))
+
     def _generate_cpy_typedef_collecttype(self, tp, name):
-        self._do_collect_type(tp)
+        self._do_collect_type(self._typedef_type(tp, name))
 
     def _generate_cpy_typedef_decl(self, tp, name):
         pass
@@ -598,6 +601,7 @@
         self._lsts["typename"].append(TypenameExpr(name, type_index))
 
     def _generate_cpy_typedef_ctx(self, tp, name):
+        tp = self._typedef_type(tp, name)
         self._typedef_ctx(tp, name)
         if getattr(tp, "origin", None) == "unknown_type":
             self._struct_ctx(tp, tp.name, approxname=None)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.3"
+VERSION = "1.8.4"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -510,3 +510,8 @@
         q = ffi.new("char[]", "abcd")
         p = ffi.cast("char(*)(void)", q)
         raises(TypeError, ffi.string, p)
+
+    def test_negative_array_size(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -8,7 +8,7 @@
 
 @unwrap_spec(cdef=str, module_name=str, source=str)
 def prepare(space, cdef, module_name, source, w_includes=None,
-            w_extra_source=None):
+            w_extra_source=None, w_min_version=None):
     try:
         import cffi
         from cffi import FFI            # <== the system one, which
@@ -16,8 +16,13 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 4, 0):
-        py.test.skip("system cffi module needs to be at least 1.4.0")
+    if w_min_version is None:
+        min_version = (1, 4, 0)
+    else:
+        min_version = tuple(space.unwrap(w_min_version))
+    if cffi.__version_info__ < min_version:
+        py.test.skip("system cffi module needs to be at least %s, got %s" % (
+            min_version, cffi.__version_info__))
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -1790,3 +1795,28 @@
                                 "void f(void) { }")
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
+
+    def test_typedef_array_dotdotdot(self):
+        ffi, lib = self.prepare("""
+            typedef int foo_t[...], bar_t[...];
+            int gv[...];
+            typedef int mat_t[...][...];
+            typedef int vmat_t[][...];
+            """,
+            "test_typedef_array_dotdotdot", """
+            typedef int foo_t[50], bar_t[50];
+            int gv[23];
+            typedef int mat_t[6][7];
+            typedef int vmat_t[][8];
+        """, min_version=(1, 8, 4))
+        assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+        assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+        assert len(ffi.new("foo_t")) == 50
+        assert len(ffi.new("bar_t")) == 50
+        assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+        assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+        assert len(ffi.new("mat_t")) == 6
+        assert len(ffi.new("mat_t")[3]) == 7
+        raises(ffi.error, ffi.sizeof, "vmat_t")
+        p = ffi.new("vmat_t", 4)
+        assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -480,3 +480,7 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_negative_array_size(self):
+        ffi = FFI()
+        py.test.raises(ValueError, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -503,3 +503,7 @@
     assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
     p = ffi.new("int[]", [-123456789])
     assert ffi.unpack(p, 1) == [-123456789]
+
+def test_negative_array_size():
+    ffi = _cffi1_backend.FFI()
+    py.test.raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1981,3 +1981,29 @@
         static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
+
+def test_typedef_array_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef int foo_t[...], bar_t[...];
+        int gv[...];
+        typedef int mat_t[...][...];
+        typedef int vmat_t[][...];
+        """)
+    lib = verify(ffi, "test_typedef_array_dotdotdot", """
+        typedef int foo_t[50], bar_t[50];
+        int gv[23];
+        typedef int mat_t[6][7];
+        typedef int vmat_t[][8];
+    """)
+    assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+    assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+    assert len(ffi.new("foo_t")) == 50
+    assert len(ffi.new("bar_t")) == 50
+    assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+    assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+    assert len(ffi.new("mat_t")) == 6
+    assert len(ffi.new("mat_t")[3]) == 7
+    py.test.raises(ffi.error, ffi.sizeof, "vmat_t")
+    p = ffi.new("vmat_t", 4)
+    assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")

From pypy.commits at gmail.com  Tue Sep 20 12:08:39 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 20 Sep 2016 09:08:39 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Start working on _value_class_
 annotation
Message-ID: <57e15f07.28eac20a.7b180.e4df@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87255:7c0752bdace0
Date: 2016-09-20 12:07 -0400
http://bitbucket.org/pypy/pypy/changeset/7c0752bdace0/

Log:	Start working on _value_class_ annotation

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -62,6 +62,9 @@
     """Raised when the _immutable_ or _immutable_fields_ hints are
     not consistent across a class hierarchy."""
 
+class ValueClassConflictError(Exception):
+    """Raise when the _value_class_ hints are not consistent across
+    the class heirarchy"""
 
 def getclassrepr(rtyper, classdef):
     if classdef is None:
@@ -517,6 +520,7 @@
             if hints is None:
                 hints = {}
             hints = self._check_for_immutable_hints(hints)
+            hints = self._check_for_value_class_hints(hints)
             kwds = {}
             if self.gcflavor == 'gc':
                 kwds['rtti'] = True
@@ -560,6 +564,38 @@
             hints['immutable_fields'] = accessor
         return hints
 
+    def _check_for_value_class_hints(self, hints):
+        """Look for value class hints in the class heirarchy to extract the proper
+        hints and ensure consistency of the _value_class_ annotation. This is
+        mostly equivalent to _check_for_immutable_hints except that
+        _value_class_=True requires _immutable_=True as well."""
+        hints = hints.copy()
+        classdesc = self.classdef.classdesc
+        value_class = classdesc.get_param('_value_class_', inherit=False)
+        if value_class is None:
+            if classdesc.get_param('_value_class_', inherit=True):
+                raise ValueClassConflictError(
+                    "class %r inherits from its parent _value_class_=True, "
+                    "so it should also declare _value_class_=True" % (
+                        self.classdef,))
+        elif value_class is not True:
+            raise TyperError(
+                "class %r: _value_class_ = something else than True" % (
+                    self.classdef,))
+        elif not hints.get('immutable', False):
+            raise ValueClassConflictError(
+                "class %r: _value_class_ = True requires that "
+                "_immutable_ = True as well")
+        else:
+            # Value classes may only be subclasses of other value classes
+            basedesc = classdesc.basedesc
+            if basedesc and not basedesc.get_param('_value_class_', False):
+                raise ValueClassConflictError(
+                    "class %r: _value_class_ = True, but its parent "
+                    "class %r does not")
+            hints['value_class'] = True
+        return hints
+
     def __repr__(self):
         if self.classdef is None:
             clsname = 'object'
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -1305,3 +1305,53 @@
         def f():
             return a.next.next.next.next is not None
         assert self.interpret(f, []) == True
+
+    def test_value_class(self):
+
+        class I(object):
+            _immutable_   = True
+            _value_class_ = True
+
+            def __init__(self, v):
+                self.v = v
+
+        i = I(3)
+        def f():
+            return i.v
+
+        t, typer, graph = self.gengraph(f, [], backendopt=True)
+        assert summary(graph) == {}
+
+    def test_value_class_not_immutable(self):
+        from rpython.rtyper.rclass import ValueClassConflictError
+
+        class I(object):
+            _value_class_ = True
+
+            def __init__(self, v):
+                self.v = v
+
+        i = I(3)
+        def f():
+            return i.v
+
+        py.test.raises(ValueClassConflictError, self.gengraph, f, [])
+
+    def test_value_class_subclass_not_value_class(self):
+        from rpython.rtyper.rclass import ValueClassConflictError
+
+        class Base(object):
+            _immutable_ = True
+
+        class I(Base):
+            _immutable_   = True
+            _value_class_ = True
+
+            def __init__(self, v):
+                self.v = v
+
+        i = I(3)
+        def f():
+            return i.v
+
+        py.test.raises(ValueClassConflictError, self.gengraph, f, [])

From pypy.commits at gmail.com  Tue Sep 20 12:28:57 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 20 Sep 2016 09:28:57 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Fix test
Message-ID: <57e163c9.0cce1c0a.792f7.9115@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87256:2c44eef6d8bd
Date: 2016-09-20 12:24 -0400
http://bitbucket.org/pypy/pypy/changeset/2c44eef6d8bd/

Log:	Fix test

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -587,12 +587,6 @@
                 "class %r: _value_class_ = True requires that "
                 "_immutable_ = True as well")
         else:
-            # Value classes may only be subclasses of other value classes
-            basedesc = classdesc.basedesc
-            if basedesc and not basedesc.get_param('_value_class_', False):
-                raise ValueClassConflictError(
-                    "class %r: _value_class_ = True, but its parent "
-                    "class %r does not")
             hints['value_class'] = True
         return hints
 
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -1341,11 +1341,11 @@
         from rpython.rtyper.rclass import ValueClassConflictError
 
         class Base(object):
-            _immutable_ = True
+            _immutable_   = True
+            _value_class_ = True
 
         class I(Base):
             _immutable_   = True
-            _value_class_ = True
 
             def __init__(self, v):
                 self.v = v
@@ -1355,3 +1355,4 @@
             return i.v
 
         py.test.raises(ValueClassConflictError, self.gengraph, f, [])
+

From pypy.commits at gmail.com  Tue Sep 20 12:58:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 20 Sep 2016 09:58:42 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: update
Message-ID: <57e16ac2.8f081c0a.bd04e.aae7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5707:2e0b239248d6
Date: 2016-09-20 18:58 +0200
http://bitbucket.org/pypy/extradoc/changeset/2e0b239248d6/

Log:	update

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -17,7 +17,7 @@
 * arigo: look at test failures relaced to os.scandir() or the pathlib
   module, or the enum module
 
-* arigo: review coroutines, add missing types like coroutine_wrapper
+* arigo: (py3k issue #2278) add full precision for st_mtime_ns
 
 
 

From pypy.commits at gmail.com  Tue Sep 20 13:02:07 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 20 Sep 2016 10:02:07 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: _value_class_=True implies
 _immutable_=True for now
Message-ID: <57e16b8f.c19d1c0a.8cbaa.b0c9@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87257:261d2a991cda
Date: 2016-09-20 13:01 -0400
http://bitbucket.org/pypy/pypy/changeset/261d2a991cda/

Log:	_value_class_=True implies _immutable_=True for now May wish to do
	away with _immutable_ in the future

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -545,6 +545,14 @@
         hints = hints.copy()
         classdesc = self.classdef.classdesc
         immut = classdesc.get_param('_immutable_', inherit=False)
+        value_class = classdesc.get_param('_value_class_', inherit=False)
+
+        if immut is None:
+            immut = value_class
+        elif value_class is not None and value_class and not immut:
+            raise ImmutableConflictError(
+                "class %r: _immutable_ != True and _value_class_ = True")
+
         if immut is None:
             if classdesc.get_param('_immutable_', inherit=True):
                 raise ImmutableConflictError(
@@ -568,7 +576,7 @@
         """Look for value class hints in the class heirarchy to extract the proper
         hints and ensure consistency of the _value_class_ annotation. This is
         mostly equivalent to _check_for_immutable_hints except that
-        _value_class_=True requires _immutable_=True as well."""
+        _value_class_=True imples _immutable_=True as well."""
         hints = hints.copy()
         classdesc = self.classdef.classdesc
         value_class = classdesc.get_param('_value_class_', inherit=False)
@@ -582,12 +590,14 @@
             raise TyperError(
                 "class %r: _value_class_ = something else than True" % (
                     self.classdef,))
-        elif not hints.get('immutable', False):
-            raise ValueClassConflictError(
-                "class %r: _value_class_ = True requires that "
-                "_immutable_ = True as well")
+        # elif not hints.get('immutable', False):
+            # raise ValueClassConflictError(
+                # "class %r: _value_class_ = True requires that "
+                # "_immutable_ = True as well")
         else:
+            # _value_class_ = True implies _immutable_ = True
             hints['value_class'] = True
+            hints['immutable'] = True
         return hints
 
     def __repr__(self):
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -1309,7 +1309,6 @@
     def test_value_class(self):
 
         class I(object):
-            _immutable_   = True
             _value_class_ = True
 
             def __init__(self, v):
@@ -1322,8 +1321,24 @@
         t, typer, graph = self.gengraph(f, [], backendopt=True)
         assert summary(graph) == {}
 
-    def test_value_class_not_immutable(self):
-        from rpython.rtyper.rclass import ValueClassConflictError
+    def test_value_class_conflicts_with_immut(self):
+        from rpython.rtyper.rclass import ImmutableConflictError
+
+        class I(object):
+            _immutable_   = False
+            _value_class_ = True
+
+            def __init__(self, v):
+                self.v = v
+
+        i = I(3)
+        def f():
+            return i.v
+
+        py.test.raises(ImmutableConflictError, self.gengraph, f, [])
+
+    def test_value_class_implies_immutable(self):
+        from rpython.jit.metainterp.typesystem import deref
 
         class I(object):
             _value_class_ = True
@@ -1333,15 +1348,17 @@
 
         i = I(3)
         def f():
-            return i.v
+            return i
 
-        py.test.raises(ValueClassConflictError, self.gengraph, f, [])
+        t, typer, graph = self.gengraph(f, [])
+        I_TYPE = deref(graph.getreturnvar().concretetype)
+        assert I_TYPE._hints['immutable']
+        assert I_TYPE._hints['value_class']
 
     def test_value_class_subclass_not_value_class(self):
         from rpython.rtyper.rclass import ValueClassConflictError
 
         class Base(object):
-            _immutable_   = True
             _value_class_ = True
 
         class I(Base):

From pypy.commits at gmail.com  Tue Sep 20 15:04:56 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 20 Sep 2016 12:04:56 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Remove dead code
Message-ID: <57e18858.aa52c20a.f1460.22bd@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87258:d6ed7f0b5471
Date: 2016-09-20 13:06 -0400
http://bitbucket.org/pypy/pypy/changeset/d6ed7f0b5471/

Log:	Remove dead code

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -576,7 +576,7 @@
         """Look for value class hints in the class heirarchy to extract the proper
         hints and ensure consistency of the _value_class_ annotation. This is
         mostly equivalent to _check_for_immutable_hints except that
-        _value_class_=True imples _immutable_=True as well."""
+        _value_class_=True imples _immutable_=True."""
         hints = hints.copy()
         classdesc = self.classdef.classdesc
         value_class = classdesc.get_param('_value_class_', inherit=False)
@@ -590,10 +590,6 @@
             raise TyperError(
                 "class %r: _value_class_ = something else than True" % (
                     self.classdef,))
-        # elif not hints.get('immutable', False):
-            # raise ValueClassConflictError(
-                # "class %r: _value_class_ = True requires that "
-                # "_immutable_ = True as well")
         else:
             # _value_class_ = True implies _immutable_ = True
             hints['value_class'] = True

From pypy.commits at gmail.com  Tue Sep 20 15:04:58 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 20 Sep 2016 12:04:58 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Add value_class information to
 SizeDescr
Message-ID: <57e1885a.a2ddc20a.af9ef.257f@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87259:ad12c5eab511
Date: 2016-09-20 14:52 -0400
http://bitbucket.org/pypy/pypy/changeset/ad12c5eab511/

Log:	Add value_class information to SizeDescr

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -139,6 +139,9 @@
     def is_immutable(self):
         return heaptracker.is_immutable_struct(self.S)
 
+    def is_value_class(self):
+        return heaptracker.is_value_class(self.S)
+
     def get_type_id(self):
         assert isinstance(self.S, lltype.GcStruct)
         return TypeIDSymbolic(self.S)     # integer-like symbolic
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -61,16 +61,19 @@
     tid = llop.combine_ushort(lltype.Signed, 0, 0)
     vtable = lltype.nullptr(rclass.OBJECT_VTABLE)
     immutable_flag = False
+    value_class_flag = False
 
     def __init__(self, size, gc_fielddescrs=None, all_fielddescrs=None,
                  vtable=lltype.nullptr(rclass.OBJECT_VTABLE),
-                 immutable_flag=False):
+                 immutable_flag=False,
+                 value_class_flag=False):
         assert lltype.typeOf(vtable) == lltype.Ptr(rclass.OBJECT_VTABLE)
         self.size = size
         self.gc_fielddescrs = gc_fielddescrs
         self.all_fielddescrs = all_fielddescrs
         self.vtable = vtable
         self.immutable_flag = immutable_flag
+        self.value_class_flag = value_class_flag
 
     def get_all_fielddescrs(self):
         return self.all_fielddescrs
@@ -94,6 +97,9 @@
     def is_immutable(self):
         return self.immutable_flag
 
+    def is_value_class(self):
+        return self.value_class_flag
+
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self.vtable))
 
@@ -109,12 +115,14 @@
     except KeyError:
         size = symbolic.get_size(STRUCT, gccache.translate_support_code)
         immutable_flag = heaptracker.is_immutable_struct(STRUCT)
+        value_class_flag = heaptracker.is_value_class(STRUCT)
         if vtable:
             assert heaptracker.has_gcstruct_a_vtable(STRUCT)
         else:
             assert not heaptracker.has_gcstruct_a_vtable(STRUCT)
         sizedescr = SizeDescr(size, vtable=vtable,
-                              immutable_flag=immutable_flag)
+                              immutable_flag=immutable_flag,
+                              value_class_flag=value_class_flag)
         gccache.init_size_descr(STRUCT, sizedescr)
         cache[STRUCT] = sizedescr
         # XXX do we really need gc_fielddescrs if we also have
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -52,6 +52,26 @@
             descr_s = get_size_descr(c0, STRUCT)
             assert descr_s.is_immutable() == True
 
+def test_get_size_descr_value_class():
+    hints = {'immutable': True, 'value_class': True}
+    S = lltype.GcStruct('S', hints=hints)
+    T = lltype.GcStruct('T', ('parent', S),
+                        ('x', lltype.Char),
+                        hints=hints)
+    U = lltype.GcStruct('U', ('parent', T),
+                        ('u', lltype.Ptr(T)),
+                        ('v', lltype.Signed),
+                        hints=hints)
+    V = lltype.GcStruct('V', ('parent', U),
+                        ('miss1', lltype.Void),
+                        ('miss2', lltype.Void),
+                        hints=hints)
+    for STRUCT in [S, T, U, V]:
+        for translated in [False, True]:
+            c0 = GcCache(translated)
+            descr_s = get_size_descr(c0, STRUCT)
+            assert descr_s.is_value_class() == True
+
 def test_get_field_descr():
     U = lltype.Struct('U')
     T = lltype.GcStruct('T')
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -23,6 +23,9 @@
 def is_immutable_struct(S):
     return isinstance(S, lltype.GcStruct) and S._hints.get('immutable', False)
 
+def is_value_class(S):
+    return isinstance(S, lltype.GcStruct) and S._hints.get('value_class', False)
+
 # ____________________________________________________________
 
 def has_gcstruct_a_vtable(GCSTRUCT):

From pypy.commits at gmail.com  Tue Sep 20 15:04:59 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 20 Sep 2016 12:04:59 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: GcStruct support for value_class
 hint
Message-ID: <57e1885b.04141c0a.25833.cc8d@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87260:c37f0db2879f
Date: 2016-09-20 15:03 -0400
http://bitbucket.org/pypy/pypy/changeset/c37f0db2879f/

Log:	GcStruct support for value_class hint

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -370,7 +370,8 @@
         return _struct(self, n, initialization='example')
 
     def _immutable_field(self, field):
-        if self._hints.get('immutable'):
+        if (self._hints.get('immutable') or
+            self._hints.get('value_class')):
             return True
         if 'immutable_fields' in self._hints:
             try:
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -787,6 +787,11 @@
                  hints={'immutable_fields': FieldListAccessor({'x': 1234})})
     assert S._immutable_field('x') == 1234
 
+def test_value_class_hint():
+    S = GcStruct('S', ('x', lltype.Signed), hints={'value_class': True})
+    assert S._immutable_field('x') == True
+    assert S._hints.get('value_class', False) == True
+
 def test_typedef():
     T = Typedef(Signed, 'T')
     assert T == Signed

From pypy.commits at gmail.com  Tue Sep 20 16:52:33 2016
From: pypy.commits at gmail.com (ntruessel)
Date: Tue, 20 Sep 2016 13:52:33 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc
Message-ID: <57e1a191.0205c20a.b2543.5235@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r87261:d78ddf89fd0b
Date: 2016-09-20 22:51 +0200
http://bitbucket.org/pypy/pypy/changeset/d78ddf89fd0b/

Log:	Update qcgc

diff --git a/rpython/translator/c/src/qcgc/config.h b/rpython/translator/c/src/qcgc/config.h
--- a/rpython/translator/c/src/qcgc/config.h
+++ b/rpython/translator/c/src/qcgc/config.h
@@ -11,7 +11,8 @@
 #define EVENT_LOG 1							// Enable event log
 #define LOGFILE "./qcgc_events.log"			// Default logfile
 #define LOG_ALLOCATION 0					// Enable allocation log
-#define LOG_DUMP_FREELIST_STATS 1			// Dump freelist stats
+#define LOG_DUMP_FREELIST_STATS 0			// Dump freelist stats
+#define LOG_ALLOCATOR_SWITCH 0				// Log allocator switches
 
 #define QCGC_SHADOWSTACK_SIZE 163840		// Total shadowstack size
 #define QCGC_ARENA_BAG_INIT_SIZE 16			// Initial size of the arena bag
@@ -19,7 +20,7 @@
 #define QCGC_LARGE_ALLOC_THRESHOLD_EXP 14	// Less than QCGC_ARENA_SIZE_EXP
 #define QCGC_MARK_LIST_SEGMENT_SIZE 64		// TODO: Tune for performance
 #define QCGC_GRAY_STACK_INIT_SIZE 128		// TODO: Tune for performance
-#define QCGC_INC_MARK_MIN 64				// TODO: Tune for performance
+#define QCGC_INC_MARK_MIN 1024				// TODO: Tune for performance
 
 /**
  * Fit allocator
@@ -31,8 +32,8 @@
 /**
  * Auto Mark/Collect
  */
-#define QCGC_INCMARK_THRESHOLD (1<<(QCGC_ARENA_SIZE_EXP-4))
-#define QCGC_INCMARK_TO_SWEEP 5
+#define QCGC_INCMARK_THRESHOLD (1<<(QCGC_ARENA_SIZE_EXP-5))
+#define QCGC_INCMARK_TO_SWEEP 10
 
 /**
  * DO NOT MODIFY BELOW HERE
diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -88,6 +88,9 @@
 object_t *_qcgc_allocate_slowpath(size_t size) {
 	bool use_fit_allocator = _qcgc_bump_allocator.ptr == NULL;
 	size_t cells = bytes_to_cells(size);
+#if LOG_ALLOCATOR_SWITCH
+	bool old_use_fit_allocator = use_fit_allocator;
+#endif
 
 	if (UNLIKELY(qcgc_state.cells_since_incmark >
 				qcgc_state.incmark_threshold)) {
@@ -103,7 +106,7 @@
 
 	object_t *result = NULL;
 	if (!use_fit_allocator) {
-		qcgc_bump_allocator_renew_block(false);
+		qcgc_bump_allocator_renew_block(size, false);
 
 		qcgc_state.cells_since_incmark += _qcgc_bump_allocator.end -
 			_qcgc_bump_allocator.ptr;
@@ -124,16 +127,47 @@
 #endif
 
 			result->flags = QCGC_GRAY_FLAG;
+#if LOG_ALLOCATOR_SWITCH
+			if ((_qcgc_bump_allocator.ptr == NULL) != old_use_fit_allocator) {
+				// Allocator switched
+				struct log_info_s {
+					size_t allocations;
+					bool fit_allocator;
+				};
+				struct log_info_s log_info = {
+					qcgc_allocations,
+					_qcgc_bump_allocator.ptr == NULL,
+				};
+				qcgc_event_logger_log(EVENT_ALLOCATOR_SWITCH, sizeof(struct log_info_s),
+						(uint8_t *) &log_info);
+			}
+#endif
 			return result;
 		}
 	}
 
 	// Fit allocate
+	result = qcgc_fit_allocate(size);
 	if (result != NULL) {
 		qcgc_state.cells_since_incmark += bytes_to_cells(size);
+#if LOG_ALLOCATOR_SWITCH
+		if ((_qcgc_bump_allocator.ptr == NULL) != old_use_fit_allocator) {
+			// Allocator switched
+			struct log_info_s {
+				size_t allocations;
+				bool fit_allocator;
+			};
+			struct log_info_s log_info = {
+				qcgc_allocations,
+				_qcgc_bump_allocator.ptr == NULL,
+			};
+			qcgc_event_logger_log(EVENT_ALLOCATOR_SWITCH, sizeof(struct log_info_s),
+					(uint8_t *) &log_info);
+		}
+#endif
 		return result;
 	}
-	qcgc_bump_allocator_renew_block(true);
+	qcgc_bump_allocator_renew_block(size, true);
 	qcgc_state.cells_since_incmark +=
 		_qcgc_bump_allocator.end - _qcgc_bump_allocator.ptr;
 
@@ -150,6 +184,21 @@
 #endif
 
 	result->flags = QCGC_GRAY_FLAG;
+#if LOG_ALLOCATOR_SWITCH
+	if ((_qcgc_bump_allocator.ptr == NULL) != old_use_fit_allocator) {
+		// Allocator switched
+		struct log_info_s {
+			size_t allocations;
+			bool fit_allocator;
+		};
+		struct log_info_s log_info = {
+			qcgc_allocations,
+			_qcgc_bump_allocator.ptr == NULL,
+		};
+		qcgc_event_logger_log(EVENT_ALLOCATOR_SWITCH, sizeof(struct log_info_s),
+				(uint8_t *) &log_info);
+	}
+#endif
 	return result;
 }
 
diff --git a/rpython/translator/c/src/qcgc/qcgc.h b/rpython/translator/c/src/qcgc/qcgc.h
--- a/rpython/translator/c/src/qcgc/qcgc.h
+++ b/rpython/translator/c/src/qcgc/qcgc.h
@@ -59,6 +59,10 @@
 	object_t *items[];
 } object_stack_t;
 
+#if LOG_ALLOCATOR_SWITCH
+size_t qcgc_allocations = 0;
+#endif
+
 /**
  * Arena
  */
@@ -200,6 +204,10 @@
 #endif
 	size_t cells = bytes_to_cells(size);
 
+#if LOG_ALLOCATOR_SWITCH
+	qcgc_allocations++;
+#endif
+
 #if LOG_ALLOCATION
 	qcgc_event_logger_log(EVENT_ALLOCATE, sizeof(size_t),
 			(uint8_t *) &cells);
@@ -281,6 +289,6 @@
  * @param	object	The object to trace
  * @param	visit	The function to be called on the referenced objects
  */
-extern void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object));
+void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object));
 
 #endif
diff --git a/rpython/translator/c/src/qcgc/src/allocator.c b/rpython/translator/c/src/qcgc/src/allocator.c
--- a/rpython/translator/c/src/qcgc/src/allocator.c
+++ b/rpython/translator/c/src/qcgc/src/allocator.c
@@ -33,7 +33,7 @@
 
 	_qcgc_bump_allocator.ptr = NULL;
 	_qcgc_bump_allocator.end = NULL;
-	qcgc_bump_allocator_renew_block(true);
+	qcgc_bump_allocator_renew_block(0,true);
 }
 
 void qcgc_allocator_destroy(void) {
@@ -65,7 +65,7 @@
  * Bump Allocator                                                              *
  ******************************************************************************/
 
-void qcgc_bump_allocator_renew_block(bool force_arena) {
+void qcgc_bump_allocator_renew_block(size_t size, bool force_arena) {
 #if CHECKED
 	if (_qcgc_bump_allocator.end > _qcgc_bump_allocator.ptr) {
 		for (cell_t *it = _qcgc_bump_allocator.ptr + 1;
@@ -79,19 +79,25 @@
 	qcgc_reset_bump_ptr();
 
 	// Always use a huge block if there is one
-	exp_free_list_t *free_list = qcgc_allocator_state.fit_state.
-		large_free_list[QCGC_LARGE_FREE_LISTS - 1];
+	assert(3 < QCGC_LARGE_FREE_LISTS);
+	size_t cells = bytes_to_cells(size);
+	size_t i = is_small(cells) ? 3 : MAX(3, large_index(cells) + 1);
+	for (; i < QCGC_LARGE_FREE_LISTS; i++) {
+		exp_free_list_t *free_list = qcgc_allocator_state.fit_state.
+			large_free_list[i];
+		if (free_list->count > 0) {
+			// Assign block to bump allocator
+			bump_allocator_assign(free_list->items[0].ptr,
+					free_list->items[0].size);
+			free_list = qcgc_exp_free_list_remove_index(free_list, 0);
 
-	if (free_list->count > 0) {
-		// Assign huge block to bump allocator
-		bump_allocator_assign(free_list->items[0].ptr,
-				free_list->items[0].size);
-		free_list = qcgc_exp_free_list_remove_index(free_list, 0);
-		qcgc_allocator_state.fit_state.
-			large_free_list[QCGC_LARGE_FREE_LISTS - 1] = free_list;
-		qcgc_state.free_cells -=
-			_qcgc_bump_allocator.end - _qcgc_bump_allocator.ptr;
-	} else {
+			qcgc_allocator_state.fit_state.large_free_list[i] = free_list;
+			qcgc_state.free_cells -=
+				_qcgc_bump_allocator.end - _qcgc_bump_allocator.ptr;
+		}
+	}
+
+	if (_qcgc_bump_allocator.ptr == NULL) {
 		if (qcgc_allocator_state.free_arenas->count > 0) {
 			// Reuse arena
 			arena_t *arena = qcgc_allocator_state.free_arenas->items[0];
diff --git a/rpython/translator/c/src/qcgc/src/allocator.h b/rpython/translator/c/src/qcgc/src/allocator.h
--- a/rpython/translator/c/src/qcgc/src/allocator.h
+++ b/rpython/translator/c/src/qcgc/src/allocator.h
@@ -100,6 +100,7 @@
 /**
  * Find a new block for the bump allocator
  *
+ * @param	size		Minimal size
  * @param	force_arena	Force generation of new arena if no block is found
  */
-void qcgc_bump_allocator_renew_block(bool force_arena);
+void qcgc_bump_allocator_renew_block(size_t size, bool force_arena);
diff --git a/rpython/translator/c/src/qcgc/src/collector.c b/rpython/translator/c/src/qcgc/src/collector.c
--- a/rpython/translator/c/src/qcgc/src/collector.c
+++ b/rpython/translator/c/src/qcgc/src/collector.c
@@ -155,7 +155,7 @@
 	qcgc_trace_cb(object, &qcgc_push_object);
 }
 
-QCGC_STATIC void qcgc_push_object(object_t *object) {
+QCGC_STATIC QCGC_INLINE void qcgc_push_object(object_t *object) {
 #if CHECKED
 	assert(qcgc_state.phase == GC_MARK);
 #endif
diff --git a/rpython/translator/c/src/qcgc/src/event_logger.c b/rpython/translator/c/src/qcgc/src/event_logger.c
--- a/rpython/translator/c/src/qcgc/src/event_logger.c
+++ b/rpython/translator/c/src/qcgc/src/event_logger.c
@@ -14,7 +14,8 @@
 void qcgc_event_logger_initialize(void) {
 #if EVENT_LOG
 	event_logger_state.logfile = fopen(LOGFILE, "w");
-	qcgc_event_logger_log(EVENT_LOG_START, 0, NULL);
+	size_t arena_cells = 1<<QCGC_ARENA_SIZE_EXP;
+	qcgc_event_logger_log(EVENT_LOG_START, sizeof(size_t), (uint8_t *)&arena_cells);
 
 	if (event_logger_state.logfile == NULL)  {
 		fprintf(stderr, "%s\n", "Failed to create logfile.");
diff --git a/rpython/translator/c/src/qcgc/src/event_logger.h b/rpython/translator/c/src/qcgc/src/event_logger.h
--- a/rpython/translator/c/src/qcgc/src/event_logger.h
+++ b/rpython/translator/c/src/qcgc/src/event_logger.h
@@ -23,6 +23,8 @@
 	EVENT_MARK_DONE,
 
 	EVENT_FREELIST_DUMP,
+
+	EVENT_ALLOCATOR_SWITCH,
 };
 
 /**

From pypy.commits at gmail.com  Wed Sep 21 07:37:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 04:37:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: Tweaks: don't use
 'int(float-value-of-st.st_Xtime)': because precision
Message-ID: <57e27106.c19d1c0a.8cbaa.fddf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87263:adb3947f8bb6
Date: 2016-09-21 13:36 +0200
http://bitbucket.org/pypy/pypy/changeset/adb3947f8bb6/

Log:	Tweaks: don't use 'int(float-value-of-st.st_Xtime)': because
	precision loss can occur, this could in theory give a result that is
	one larger than what the system call returned.

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -226,13 +226,13 @@
     w_keywords = space.newdict()
     stat_float_times = space.fromcache(StatState).stat_float_times
     for i, (name, TYPE) in FIELDS:
-        value = getattr(st, name)
-        if name in ('st_atime', 'st_mtime', 'st_ctime'):
-            value = int(value)   # rounded to an integer for indexed access
-        w_value = space.wrap(value)
         if i < rposix_stat.N_INDEXABLE_FIELDS:
+            # get the first 10 items by indexing; this gives us
+            # 'st_Xtime' as an integer, too
+            w_value = space.wrap(st[i])
             lst[i] = w_value
-        else:
+        elif name.startswith('st_'):    # exclude 'nsec_Xtime'
+            w_value = space.wrap(getattr(st, name))
             space.setitem(w_keywords, space.wrap(name), w_value)
 
     # non-rounded values for name-based access
@@ -243,13 +243,8 @@
                       space.wrap('st_mtime'), space.wrap(st.st_mtime))
         space.setitem(w_keywords,
                       space.wrap('st_ctime'), space.wrap(st.st_ctime))
-    else:
-        space.setitem(w_keywords,
-                      space.wrap('st_atime'), space.wrap(int(st.st_atime)))
-        space.setitem(w_keywords,
-                      space.wrap('st_mtime'), space.wrap(int(st.st_mtime)))
-        space.setitem(w_keywords,
-                      space.wrap('st_ctime'), space.wrap(int(st.st_ctime)))
+    #else:
+    #   filled by the __init__ method
 
     w_tuple = space.newtuple(lst)
     w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -129,9 +129,9 @@
         assert st[4] == st.st_uid
         assert st[5] == st.st_gid
         assert st[6] == st.st_size
-        assert st[7] == int(st.st_atime)
-        assert st[8] == int(st.st_mtime)
-        assert st[9] == int(st.st_ctime)
+        assert st[7] == int(st.st_atime)   # in complete corner cases, rounding
+        assert st[8] == int(st.st_mtime)   # here could maybe get the wrong
+        assert st[9] == int(st.st_ctime)   # integer...
 
         assert stat.S_IMODE(st.st_mode) & stat.S_IRUSR
         assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR
@@ -141,13 +141,12 @@
         assert st.st_size == 14
         assert st.st_nlink == 1
 
-        #if sys.platform.startswith('linux'):
-        #    # expects non-integer timestamps - it's unlikely that they are
-        #    # all three integers
-        #    assert ((st.st_atime, st.st_mtime, st.st_ctime) !=
-        #            (st[7],       st[8],       st[9]))
-        #    assert st.st_blksize * st.st_blocks >= st.st_size
+        assert not hasattr(st, 'nsec_atime')
+
         if sys.platform.startswith('linux'):
+            assert isinstance(st.st_atime, float)
+            assert isinstance(st.st_mtime, float)
+            assert isinstance(st.st_ctime, float)
             assert hasattr(st, 'st_rdev')
 
     def test_stat_float_times(self):

From pypy.commits at gmail.com  Wed Sep 21 07:37:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 04:37:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: os.stat(): support full-precision
 nanosecond times in translated RPython
Message-ID: <57e27104.87adc20a.7f06f.4959@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87262:038a80811172
Date: 2016-09-21 13:35 +0200
http://bitbucket.org/pypy/pypy/changeset/038a80811172/

Log:	os.stat(): support full-precision nanosecond times in translated
	RPython programs, as rbigint numbers.

	Before translation, it relies on Python 2.7's os.stat_result. This
	means that before translation you get lower-precision rbigints.

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.error import TyperError
 
 from rpython.rlib._os_support import _preferred_traits, string_traits
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask
@@ -51,15 +51,18 @@
     ("st_uid",       lltype.Signed),
     ("st_gid",       lltype.Signed),
     ("st_size",      lltype.SignedLongLong),
-    ("st_atime",     lltype.Float),
-    ("st_mtime",     lltype.Float),
-    ("st_ctime",     lltype.Float),
+    ("st_atime",     lltype.SignedLongLong),   # integral number of seconds
+    ("st_mtime",     lltype.SignedLongLong),   #
+    ("st_ctime",     lltype.SignedLongLong),   #
     ("st_blksize",   lltype.Signed),
     ("st_blocks",    lltype.Signed),
     ("st_rdev",      lltype.Signed),
     ("st_flags",     lltype.Signed),
     #("st_gen",       lltype.Signed),     -- new in CPy 2.5, not implemented
     #("st_birthtime", lltype.Float),      -- new in CPy 2.5, not implemented
+    ("nsec_atime",   lltype.Signed),   # number of nanoseconds
+    ("nsec_mtime",   lltype.Signed),   #
+    ("nsec_ctime",   lltype.Signed),   #
 ]
 N_INDEXABLE_FIELDS = 10
 
@@ -79,6 +82,37 @@
     ("f_namemax", lltype.Signed),
 ]
 
+ at specialize.arg(1)
+def get_stat_ns_as_bigint(st, name):
+    """'name' is one of the strings "atime", "mtime" or "ctime".
+    Returns a bigint that represents the number of nanoseconds
+    stored inside the RPython-level os.stat_result 'st'.
+
+    Note that when running untranslated, the os.stat_result type
+    is from Python 2.7, which doesn't store more precision than
+    a float anyway.  You will only get more after translation.
+    """
+    from rpython.rlib.rbigint import rbigint
+
+    if not we_are_translated():
+        as_float = getattr(st, "st_" + name)
+        return rbigint.fromfloat(as_float * 1e9)
+
+    if name == "atime":
+        i, j = 7, -3
+    elif name == "mtime":
+        i, j = 8, -2
+    elif name == "ctime":
+        i, j = 9, -1
+    else:
+        raise AssertionError(name)
+
+    sec = st[i]
+    nsec = st[j]
+    result = rbigint.fromrarith_int(sec).int_mul(1000000000)
+    result = result.int_add(nsec)
+    return result
+
 
 # ____________________________________________________________
 #
@@ -97,7 +131,15 @@
         if not s_attr.is_constant():
             raise annmodel.AnnotatorError("non-constant attr name in getattr()")
         attrname = s_attr.const
-        TYPE = STAT_FIELD_TYPES[attrname]
+        if attrname in ('st_atime', 'st_mtime', 'st_ctime'):
+            # like CPython, in RPython we can read the st_Xtime
+            # attribute and get a floating-point result.  We can also
+            # get a full-precision bigint with get_stat_ns_as_bigint().
+            # The floating-point result is computed like a property
+            # by _ll_get_st_Xtime().
+            TYPE = lltype.Float
+        else:
+            TYPE = STAT_FIELD_TYPES[attrname]
         return lltype_to_annotation(TYPE)
 
     def _get_rmarshall_support_(self):     # for rlib.rmarshal
@@ -105,13 +147,14 @@
         # (we ignore the extra values here for simplicity and portability)
         def stat_result_reduce(st):
             return (st[0], st[1], st[2], st[3], st[4],
-                    st[5], st[6], st[7], st[8], st[9])
+                    st[5], st[6], st[7], st[8], st[9],
+                    st[-3], st[-2], st[-1])
 
         def stat_result_recreate(tup):
-            return make_stat_result(tup + extra_zeroes)
+            return make_stat_result(tup[:10] + extra_zeroes + tup[-3:])
         s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE)
                                        for name, TYPE in PORTABLE_STAT_FIELDS])
-        extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS))
+        extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS) - 3)
         return s_reduced, stat_result_reduce, stat_result_recreate
 
 
@@ -119,7 +162,7 @@
     def getitem((s_sta, s_int)):
         assert s_int.is_constant(), "os.stat()[index]: index must be constant"
         index = s_int.const
-        assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range"
+        assert -3 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range"
         name, TYPE = STAT_FIELDS[index]
         return lltype_to_annotation(TYPE)
 
@@ -152,28 +195,61 @@
     def rtype_getattr(self, hop):
         s_attr = hop.args_s[1]
         attr = s_attr.const
+        if attr in ('st_atime', 'st_mtime', 'st_ctime'):
+            ll_func = globals()['_ll_get_' + attr]
+            v_tuple = hop.inputarg(self, arg=0)
+            return hop.gendirectcall(ll_func, v_tuple)
         try:
             index = self.stat_field_indexes[attr]
         except KeyError:
             raise TyperError("os.stat().%s: field not available" % (attr,))
         return self.redispatch_getfield(hop, index)
 
+ at specialize.memo()
+def _stfld(name):
+    index = STAT_FIELD_NAMES.index(name)
+    return 'item%d' % index
+
+def _ll_get_st_atime(tup):
+    return (float(getattr(tup, _stfld("st_atime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_atime")))
+
+def _ll_get_st_mtime(tup):
+    return (float(getattr(tup, _stfld("st_mtime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_mtime")))
+
+def _ll_get_st_ctime(tup):
+    return (float(getattr(tup, _stfld("st_ctime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_ctime")))
+
 
 class __extend__(pairtype(StatResultRepr, IntegerRepr)):
     def rtype_getitem((r_sta, r_int), hop):
         s_int = hop.args_s[1]
         index = s_int.const
+        if index < 0:
+            index += len(STAT_FIELDS)
         return r_sta.redispatch_getfield(hop, index)
 
 s_StatResult = SomeStatResult()
 
+
 def make_stat_result(tup):
-    """Turn a tuple into an os.stat_result object."""
-    positional = tuple(
-        lltype.cast_primitive(TYPE, value) for value, (name, TYPE) in
-            zip(tup, STAT_FIELDS)[:N_INDEXABLE_FIELDS])
+    """NOT_RPYTHON: Turn a tuple into an os.stat_result object."""
+    assert len(tup) == len(STAT_FIELDS)
+    assert float not in [type(x) for x in tup]
+    positional = []
+    for i in range(N_INDEXABLE_FIELDS):
+        name, TYPE = STAT_FIELDS[i]
+        value = lltype.cast_primitive(TYPE, tup[i])
+        positional.append(value)
     kwds = {}
+    kwds['st_atime'] = tup[7] + 1e-9 * tup[-3]
+    kwds['st_mtime'] = tup[8] + 1e-9 * tup[-2]
+    kwds['st_ctime'] = tup[9] + 1e-9 * tup[-1]
     for value, (name, TYPE) in zip(tup, STAT_FIELDS)[N_INDEXABLE_FIELDS:]:
+        if name.startswith('nsec_'):
+            continue   # ignore the nsec_Xtime here
         kwds[name] = lltype.cast_primitive(TYPE, value)
     return os.stat_result(positional, kwds)
 
@@ -360,6 +436,8 @@
         posix_declaration(ALL_STAT_FIELDS[_i])
     del _i
 
+STAT_FIELDS += ALL_STAT_FIELDS[-3:]   # nsec_Xtime
+
 # these two global vars only list the fields defined in the underlying platform
 STAT_FIELD_TYPES = dict(STAT_FIELDS)      # {'st_xxx': TYPE}
 STAT_FIELD_NAMES = [_name for (_name, _TYPE) in STAT_FIELDS]
@@ -368,16 +446,20 @@
 STATVFS_FIELD_TYPES = dict(STATVFS_FIELDS)
 STATVFS_FIELD_NAMES = [name for name, tp in STATVFS_FIELDS]
 
+
 def build_stat_result(st):
     # only for LL backends
     if TIMESPEC is not None:
-        atim = st.c_st_atim; atime = int(atim.c_tv_sec) + 1E-9 * int(atim.c_tv_nsec)
-        mtim = st.c_st_mtim; mtime = int(mtim.c_tv_sec) + 1E-9 * int(mtim.c_tv_nsec)
-        ctim = st.c_st_ctim; ctime = int(ctim.c_tv_sec) + 1E-9 * int(ctim.c_tv_nsec)
+        atim = st.c_st_atim
+        mtim = st.c_st_mtim
+        ctim = st.c_st_ctim
+        atime, extra_atime = atim.c_tv_sec, int(atim.c_tv_nsec)
+        mtime, extra_mtime = mtim.c_tv_sec, int(mtim.c_tv_nsec)
+        ctime, extra_ctime = ctim.c_tv_sec, int(ctim.c_tv_nsec)
     else:
-        atime = st.c_st_atime
-        mtime = st.c_st_mtime
-        ctime = st.c_st_ctime
+        atime, extra_atime = st.c_st_atime, 0
+        mtime, extra_mtime = st.c_st_mtime, 0
+        ctime, extra_ctime = st.c_st_ctime, 0
 
     result = (st.c_st_mode,
               st.c_st_ino,
@@ -395,6 +477,10 @@
     if "st_rdev"    in STAT_FIELD_TYPES: result += (st.c_st_rdev,)
     if "st_flags"   in STAT_FIELD_TYPES: result += (st.c_st_flags,)
 
+    result += (extra_atime,
+               extra_mtime,
+               extra_ctime)
+
     return make_stat_result(result)
 
 
@@ -455,12 +541,14 @@
             # console or LPT device
             return make_stat_result((win32traits._S_IFCHR,
                                      0, 0, 0, 0, 0,
-                                     0, 0, 0, 0))
+                                     0, 0, 0, 0,
+                                     0, 0, 0))
         elif filetype == win32traits.FILE_TYPE_PIPE:
             # socket or named pipe
             return make_stat_result((win32traits._S_IFIFO,
                                      0, 0, 0, 0, 0,
-                                     0, 0, 0, 0))
+                                     0, 0, 0, 0,
+                                     0, 0, 0))
         elif filetype == win32traits.FILE_TYPE_UNKNOWN:
             error = rwin32.GetLastError_saved()
             if error != 0:
@@ -539,14 +627,11 @@
 #__________________________________________________
 # Helper functions for win32
 if _WIN32:
-    from rpython.rlib.rwin32file import FILE_TIME_to_time_t_float
+    from rpython.rlib.rwin32file import FILE_TIME_to_time_t_nsec
 
     def make_longlong(high, low):
         return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
 
-    # Seconds between 1.1.1601 and 1.1.1970
-    secs_between_epochs = rffi.r_longlong(11644473600)
-
     @specialize.arg(0)
     def win32_xstat(traits, path, traverse=False):
         win32traits = make_win32_traits(traits)
@@ -582,14 +667,15 @@
     def win32_attribute_data_to_stat(win32traits, info):
         st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes)
         st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow)
-        ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime)
-        mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime)
-        atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime)
+        ctime, extra_ctime = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime)
+        mtime, extra_mtime = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime)
+        atime, extra_atime = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime)
 
         result = (st_mode,
                   0, 0, 0, 0, 0,
                   st_size,
-                  atime, mtime, ctime)
+                  atime, mtime, ctime,
+                  extra_atime, extra_mtime, extra_ctime)
 
         return make_stat_result(result)
 
@@ -597,9 +683,9 @@
         # similar to the one above
         st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes)
         st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow)
-        ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime)
-        mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime)
-        atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime)
+        ctime, extra_ctime = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime)
+        mtime, extra_mtime = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime)
+        atime, extra_atime = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime)
 
         # specific to fstat()
         st_ino = make_longlong(info.c_nFileIndexHigh, info.c_nFileIndexLow)
@@ -608,7 +694,8 @@
         result = (st_mode,
                   st_ino, 0, st_nlink, 0, 0,
                   st_size,
-                  atime, mtime, ctime)
+                  atime, mtime, ctime,
+                  extra_atime, extra_mtime, extra_ctime)
 
         return make_stat_result(result)
 
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -6,6 +6,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import intmask
 
 @specialize.memo()
 def make_win32_traits(traits):
@@ -213,13 +214,25 @@
     return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
 
 # Seconds between 1.1.1601 and 1.1.1970
-secs_between_epochs = rffi.r_longlong(11644473600)
+secs_between_epochs = 11644473600.0
+hns_between_epochs = rffi.r_longlong(116444736000000000)  # units of 100 nsec
 
 def FILE_TIME_to_time_t_float(filetime):
     ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
     # FILETIME is in units of 100 nsec
     return float(ft) * (1.0 / 10000000.0) - secs_between_epochs
 
+def FILE_TIME_to_time_t_nsec(filetime):
+    """Like the previous function, but returns a pair: (integer part
+    'time_t' as a r_longlong, fractional part as an int measured in
+    nanoseconds).
+    """
+    ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
+    ft -= hns_between_epochs
+    int_part = ft / 10000000
+    frac_part = ft - (int_part * 10000000)
+    return (int_part, intmask(frac_part) * 100)
+
 def time_t_to_FILE_TIME(time, filetime):
     ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
diff --git a/rpython/rlib/test/test_rposix_stat.py b/rpython/rlib/test/test_rposix_stat.py
--- a/rpython/rlib/test/test_rposix_stat.py
+++ b/rpython/rlib/test/test_rposix_stat.py
@@ -2,12 +2,20 @@
 import py
 from rpython.rlib import rposix_stat
 from rpython.tool.udir import udir
+from rpython.translator.c.test.test_genc import compile
+from rpython.rtyper.lltypesystem import lltype
+
 
 class TestPosixStatFunctions:
     @py.test.mark.skipif("sys.platform == 'win32'",
                          reason="win32 only has the portable fields")
     def test_has_all_fields(self):
-        assert rposix_stat.STAT_FIELDS == rposix_stat.ALL_STAT_FIELDS[:13]
+        # XXX this test is obscure!  it will fail if the exact set of
+        # XXX stat fields found differs from the one we expect on Linux.
+        # XXX Why?
+        assert rposix_stat.STAT_FIELDS == (
+            rposix_stat.ALL_STAT_FIELDS[:13] +
+            rposix_stat.ALL_STAT_FIELDS[-3:])
 
     def test_stat(self):
         def check(f):
@@ -66,3 +74,27 @@
     finally:
         os.close(dirfd)
     assert result.st_atime == tmpdir.join('file').atime()
+
+def test_high_precision_stat_time():
+    def f():
+        st = os.stat('.')
+        # should be supported on all platforms, but give a result whose
+        # precision might be lower than full nanosecond
+        highprec = rposix_stat.get_stat_ns_as_bigint(st, "ctime")
+        return '%s;%s' % (st.st_ctime, highprec.str())
+    fc = compile(f, [])
+    as_string = fc()
+    asfloat, highprec = as_string.split(';')
+    asfloat = float(asfloat)
+    highprec = int(highprec)
+    st = os.stat('.')
+    assert abs(asfloat - st.st_ctime) < 500e-9
+    assert abs(highprec - int(st.st_ctime * 1e9)) < 500
+    assert abs(rposix_stat.get_stat_ns_as_bigint(st, "ctime").tolong()
+               - st.st_ctime * 1e9) < 3
+    if rposix_stat.TIMESPEC is not None:
+        with lltype.scoped_alloc(rposix_stat.STAT_STRUCT.TO) as stresult:
+            rposix_stat.c_stat(".", stresult)
+            assert 0 <= stresult.c_st_ctim.c_tv_nsec <= 999999999
+            assert highprec == (int(stresult.c_st_ctim.c_tv_sec) * 1000000000
+                                + int(stresult.c_st_ctim.c_tv_nsec))

From pypy.commits at gmail.com  Wed Sep 21 08:07:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 05:07:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: Actually,
 we can just move that block of code outside the unrolled loop.
Message-ID: <57e2780a.87941c0a.4e555.f72a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87264:d9aeadc27510
Date: 2016-09-21 14:07 +0200
http://bitbucket.org/pypy/pypy/changeset/d9aeadc27510/

Log:	Actually, we can just move that block of code outside the unrolled
	loop.

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,28 +30,25 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
-    w_initializer_type = None
-    w_initializer = None
-    if len(__args__.arguments_w) > 0:
-        w_initializer = __args__.arguments_w[0]
-        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-            if w_initializer is not None:
-                if w_initializer_type is space.w_str:
-                    a.descr_fromstring(space, w_initializer)
-                elif w_initializer_type is space.w_list:
-                    a.descr_fromlist(space, w_initializer)
-                else:
-                    a.extend(w_initializer, True)
             break
     else:
         raise oefmt(space.w_ValueError,
                     "bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or "
                     "d)")
 
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
+        if w_initializer_type is space.w_str:
+            a.descr_fromstring(space, w_initializer)
+        elif w_initializer_type is space.w_list:
+            a.descr_fromlist(space, w_initializer)
+        else:
+            a.extend(w_initializer, True)
     return a
 
 

From pypy.commits at gmail.com  Wed Sep 21 08:20:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 05:20:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57e27b22.e17cc20a.65bd9.5b15@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87265:5d068bf3a13d
Date: 2016-09-21 14:20 +0200
http://bitbucket.org/pypy/pypy/changeset/5d068bf3a13d/

Log:	hg merge default

diff too long, truncating to 2000 out of 2019 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -13,6 +13,7 @@
 import sys
 import os
 import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -62,8 +63,7 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -75,8 +75,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,7 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break
+                break        
 
     if args:
         vals = []
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.2
+Version: 1.8.4
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.2"
-__version_info__ = (1, 8, 2)
+__version__ = "1.8.4"
+__version_info__ = (1, 8, 4)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.2"
+                               "\ncompiled with cffi version: 1.8.4"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -332,7 +332,7 @@
                         realtype = model.unknown_ptr_type(decl.name)
                     else:
                         realtype, quals = self._get_type_and_quals(
-                            decl.type, name=decl.name)
+                            decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
                 else:
                     raise api.CDefError("unrecognized construct", decl)
@@ -781,11 +781,14 @@
                 exprnode.name in self._int_constants):
             return self._int_constants[exprnode.name]
         #
-        if partial_length_ok:
-            if (isinstance(exprnode, pycparser.c_ast.ID) and
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
                 self._partial_length = True
                 return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
         #
         raise api.FFIError(":%d: unsupported expression: expected a "
                            "simple numeric constant" % exprnode.coord.line)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -587,8 +587,11 @@
     # ----------
     # typedefs
 
+    def _typedef_type(self, tp, name):
+        return self._global_type(tp, "(*(%s *)0)" % (name,))
+
     def _generate_cpy_typedef_collecttype(self, tp, name):
-        self._do_collect_type(tp)
+        self._do_collect_type(self._typedef_type(tp, name))
 
     def _generate_cpy_typedef_decl(self, tp, name):
         pass
@@ -598,6 +601,7 @@
         self._lsts["typename"].append(TypenameExpr(name, type_index))
 
     def _generate_cpy_typedef_ctx(self, tp, name):
+        tp = self._typedef_type(tp, name)
         self._typedef_ctx(tp, name)
         if getattr(tp, "origin", None) == "unknown_type":
             self._struct_ctx(tp, tp.name, approxname=None)
diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt
--- a/pypy/doc/config/translation.profopt.txt
+++ b/pypy/doc/config/translation.profopt.txt
@@ -3,3 +3,14 @@
 RPython program) to gather profile data. Example for pypy-c: "-c 'from
 richards import main;main(); from test import pystone;
 pystone.main()'"
+
+NOTE: be aware of what this does in JIT-enabled executables.  What it
+does is instrument and later optimize the C code that happens to run in
+the example you specify, ignoring any execution of the JIT-generated
+assembler.  That means that you have to choose the example wisely.  If
+it is something that will just generate assembler and stay there, there
+is little value.  If it is something that exercises heavily library
+routines that are anyway written in C, then it will optimize that.  Most
+interesting would be something that causes a lot of JIT-compilation,
+like running a medium-sized test suite several times in a row, in order
+to optimize the warm-up in general.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,8 @@
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -246,6 +246,10 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+            if config.translation.profopt is not None:
+                raise Exception("Cannot use the --profopt option "
+                                "when --shared is on (it is by default). "
+                                "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1952,11 +1952,9 @@
     'NotImplementedError',
     'OSError',
     'OverflowError',
-    'PendingDeprecationWarning',
     'ReferenceError',
     'ResourceWarning',
     'RuntimeError',
-    'RuntimeWarning',
     'StopIteration',
     'SyntaxError',
     'SyntaxWarning',
@@ -1970,10 +1968,12 @@
     'UnicodeError',
     'UnicodeTranslateError',
     'UnicodeWarning',
-    'UserWarning',
     'ValueError',
     'Warning',
     'ZeroDivisionError'
+    'RuntimeWarning',
+    'PendingDeprecationWarning',
+    'UserWarning',
 ]
 
 if sys.platform.startswith("win"):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -63,7 +63,7 @@
         """x.__iter__() <==> iter(x)"""
         return self.space.wrap(self)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -264,25 +264,22 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
-            else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
-            #
             try:
+                if operr is not None:
+                    ec = self.space.getexecutioncontext()
+                    next_instr = self.handle_operation_error(ec, operr)
+                    self.last_instr = intmask(next_instr - 1)
+                else:
+                    # Execution starts just after the last_instr.  Initially,
+                    # last_instr is -1.  After a generator suspends it points to
+                    # the YIELD_VALUE instruction.
+                    next_instr = r_uint(self.last_instr + 1)
+                    if next_instr != 0:
+                        self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
-            executioncontext.return_trace(self, w_exitvalue)
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,12 +57,14 @@
         def f():
             yield 2
         g = f()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
 
     def test_throw3(self):
@@ -241,7 +243,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -545,3 +545,21 @@
         it = yield_raise()
         assert next(it) is KeyError
         assert next(it) is KeyError
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.2"
+VERSION = "1.8.4"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer
 from pypy.module._cffi_backend import ctypeprim
 
 
@@ -22,6 +22,7 @@
     is_nonfunc_pointer_or_array = True
 
     def __init__(self, space, ctptr, length, arraysize, extra):
+        assert isinstance(ctptr, W_CTypePointer)
         W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
                                    ctptr.ctitem)
         self.length = length
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -35,8 +35,7 @@
         assert isinstance(ellipsis, bool)
         extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi)
         size = rffi.sizeof(rffi.VOIDP)
-        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult,
-                                could_cast_anything=False)
+        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult)
         self.fargs = fargs
         self.ellipsis = ellipsis
         self.abi = abi
@@ -59,6 +58,16 @@
                     lltype.free(self.cif_descr, flavor='raw')
                     self.cif_descr = lltype.nullptr(CIF_DESCRIPTION)
 
+    def is_unichar_ptr_or_array(self):
+        return False
+
+    def is_char_or_unichar_ptr_or_array(self):
+        return False
+
+    def string(self, cdataobj, maxlen):
+        # Can't use ffi.string() on a function pointer
+        return W_CType.string(self, cdataobj, maxlen)
+
     def new_ctypefunc_completing_argtypes(self, args_w):
         space = self.space
         nargs_declared = len(self.fargs)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -19,7 +19,6 @@
     # XXX this could be improved with an elidable method get_size()
     # that raises in case it's still -1...
 
-    cast_anything = False
     is_primitive_integer = False
     is_nonfunc_pointer_or_array = False
     is_indirect_arg_for_call_python = False
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -120,7 +120,6 @@
 
 class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar):
     _attrs_ = []
-    cast_anything = True
 
     def cast_to_int(self, cdata):
         return self.space.wrap(ord(cdata[0]))
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -16,12 +16,11 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _attrs_            = ['ctitem', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'accept_str', 'length']
     length = -1
 
-    def __init__(self, space, size, extra, extra_position, ctitem,
-                 could_cast_anything=True):
+    def __init__(self, space, size, extra, extra_position, ctitem):
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
@@ -29,10 +28,11 @@
         #  - for arrays, it is the array item type
         #  - for functions, it is the return type
         self.ctitem = ctitem
-        self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.accept_str = (self.can_cast_anything or
-                            (ctitem.is_primitive_integer and
-                             ctitem.size == rffi.sizeof(lltype.Char)))
+        self.accept_str = (self.is_nonfunc_pointer_or_array and
+                (isinstance(ctitem, ctypevoid.W_CTypeVoid) or
+                 isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or
+                 (ctitem.is_primitive_integer and
+                  ctitem.size == rffi.sizeof(lltype.Char))))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -139,7 +139,10 @@
 
 class W_CTypePtrBase(W_CTypePtrOrArray):
     # base class for both pointers and pointers-to-functions
-    _attrs_ = []
+    _attrs_ = ['is_void_ptr', 'is_voidchar_ptr']
+    _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr']
+    is_void_ptr = False
+    is_voidchar_ptr = False
 
     def convert_to_object(self, cdata):
         ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0]
@@ -156,7 +159,16 @@
             else:
                 raise self._convert_error("compatible pointer", w_ob)
         if self is not other:
-            if not (self.can_cast_anything or other.can_cast_anything):
+            if self.is_void_ptr or other.is_void_ptr:
+                pass     # cast from or to 'void *'
+            elif self.is_voidchar_ptr or other.is_voidchar_ptr:
+                space = self.space
+                msg = ("implicit cast from '%s' to '%s' "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" % (other.name, self.name))
+                space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1)
+            else:
                 raise self._convert_error("compatible pointer", w_ob)
 
         rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
@@ -167,8 +179,8 @@
 
 
 class W_CTypePointer(W_CTypePtrBase):
-    _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types']
-    _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr']
+    _attrs_ = ['is_file', 'cache_array_type', '_array_types']
+    _immutable_fields_ = ['is_file', 'cache_array_type?']
     kind = "pointer"
     cache_array_type = None
     is_nonfunc_pointer_or_array = True
@@ -183,6 +195,8 @@
         self.is_file = (ctitem.name == "struct _IO_FILE" or
                         ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
+        self.is_voidchar_ptr = (self.is_void_ptr or
+                           isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar))
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init, allocator):
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -7,7 +7,6 @@
 
 class W_CTypeVoid(W_CType):
     _attrs_ = []
-    cast_anything = True
     kind = "void"
 
     def __init__(self, space):
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -32,8 +32,8 @@
 @unwrap_spec(w_cdata=cdataobj.W_CData)
 def from_handle(space, w_cdata):
     ctype = w_cdata.ctype
-    if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or
-        not ctype.can_cast_anything):
+    if (not isinstance(ctype, ctypeptr.W_CTypePointer) or
+        not ctype.is_voidchar_ptr):
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3665,3 +3665,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -503,3 +503,15 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_bug_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        q = ffi.new("char[]", "abcd")
+        p = ffi.cast("char(*)(void)", q)
+        raises(TypeError, ffi.string, p)
+
+    def test_negative_array_size(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -8,7 +8,7 @@
 
 @unwrap_spec(cdef=str, module_name=str, source=str)
 def prepare(space, cdef, module_name, source, w_includes=None,
-            w_extra_source=None):
+            w_extra_source=None, w_min_version=None):
     try:
         import cffi
         from cffi import FFI            # <== the system one, which
@@ -16,8 +16,13 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 4, 0):
-        py.test.skip("system cffi module needs to be at least 1.4.0")
+    if w_min_version is None:
+        min_version = (1, 4, 0)
+    else:
+        min_version = tuple(space.unwrap(w_min_version))
+    if cffi.__version_info__ < min_version:
+        py.test.skip("system cffi module needs to be at least %s, got %s" % (
+            min_version, cffi.__version_info__))
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -1790,3 +1795,28 @@
                                 "void f(void) { }")
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
+
+    def test_typedef_array_dotdotdot(self):
+        ffi, lib = self.prepare("""
+            typedef int foo_t[...], bar_t[...];
+            int gv[...];
+            typedef int mat_t[...][...];
+            typedef int vmat_t[][...];
+            """,
+            "test_typedef_array_dotdotdot", """
+            typedef int foo_t[50], bar_t[50];
+            int gv[23];
+            typedef int mat_t[6][7];
+            typedef int vmat_t[][8];
+        """, min_version=(1, 8, 4))
+        assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+        assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+        assert len(ffi.new("foo_t")) == 50
+        assert len(ffi.new("bar_t")) == 50
+        assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+        assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+        assert len(ffi.new("mat_t")) == 6
+        assert len(ffi.new("mat_t")[3]) == 7
+        raises(ffi.error, ffi.sizeof, "vmat_t")
+        p = ffi.new("vmat_t", 4)
+        assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -5,7 +5,6 @@
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long", "w_int")
@@ -27,25 +26,25 @@
     """Return a new PyLongObject object from a C size_t, or NULL on
     failure.
     """
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.LONGLONG], PyObject)
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
     """Return a new PyLongObject object from a C unsigned long, or
     NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONGLONG], PyObject)
 def PyLong_FromUnsignedLongLong(space, val):
     """Return a new PyLongObject object from a C unsigned long long,
     or NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([PyObject], rffi.ULONG, error=-1)
 def PyLong_AsUnsignedLong(space, w_long):
@@ -212,7 +211,10 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    value = rffi.cast(ADDR, p)    # signed integer
+    if value < 0:
+        return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
+    return space.wrap(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -1,5 +1,6 @@
 import sys, py
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import maxint
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
@@ -7,18 +8,20 @@
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
-        value = api.PyLong_FromLong(3)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == 3
+        w_value = api.PyLong_FromLong(3)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == 3
 
-        value = api.PyLong_FromLong(sys.maxint)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == sys.maxint
+        w_value = api.PyLong_FromLong(sys.maxint)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == sys.maxint
 
     def test_aslong(self, space, api):
         w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+        assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsLong(w_value)
         assert value == (sys.maxint - 1)
 
@@ -34,12 +37,16 @@
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsSsize_t(w_value)
         assert value == 2
-        assert space.eq_w(w_value, api.PyLong_FromSsize_t(2))
+        w_val2 = api.PyLong_FromSsize_t(2)
+        assert isinstance(w_val2, W_LongObject)
+        assert space.eq_w(w_value, w_val2)
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
+        assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -101,9 +108,26 @@
         lltype.free(overflow, flavor='raw')
 
     def test_as_voidptr(self, space, api):
+        # CPython returns an int (not a long) depending on the value
+        # passed to PyLong_FromVoidPtr().  In all cases, NULL becomes
+        # the int 0.
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
-        assert space.unwrap(w_l) == 0L
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == 0
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
+        # Positive values also return an int (assuming, like always in
+        # PyPy, that an int is big enough to store any pointer).
+        p = rffi.cast(rffi.VOIDP, maxint)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == maxint
+        assert api.PyLong_AsVoidPtr(w_l) == p
+        # Negative values always return a long.
+        p = rffi.cast(rffi.VOIDP, -maxint-1)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_long)
+        assert space.unwrap(w_l) == maxint+1
+        assert api.PyLong_AsVoidPtr(w_l) == p
 
     def test_sign_and_bits(self, space, api):
         if space.is_true(space.lt(space.sys.get('version_info'),
@@ -133,23 +157,58 @@
         module = self.import_extension('foo', [
             ("from_unsignedlong", "METH_NOARGS",
              """
-                 return PyLong_FromUnsignedLong((unsigned long)-1);
+                 PyObject * obj;
+                 obj = PyLong_FromUnsignedLong((unsigned long)-1);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
         import sys
         assert module.from_unsignedlong() == 2 * sys.maxsize + 1
 
     def test_fromlonglong(self):
         module = self.import_extension('foo', [
-            ("from_longlong", "METH_NOARGS",
+            ("from_longlong", "METH_VARARGS",
              """
-                 return PyLong_FromLongLong((long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """),
-            ("from_unsignedlonglong", "METH_NOARGS",
+            ("from_unsignedlonglong", "METH_VARARGS",
              """
-                 return PyLong_FromUnsignedLongLong((unsigned long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromUnsignedLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
-        assert module.from_longlong() == -1
-        assert module.from_unsignedlonglong() == (1<<64) - 1
+        assert module.from_longlong(-1) == -1
+        assert module.from_longlong(0) == 0
+        assert module.from_unsignedlonglong(0) == 0
+        assert module.from_unsignedlonglong(-1) == (1<<64) - 1
 
     def test_from_size_t(self):
         module = self.import_extension('foo', [
@@ -237,10 +296,15 @@
             ("has_sub", "METH_NOARGS",
              """
                 PyObject *ret, *obj = PyLong_FromLong(42);
-                if (obj->ob_type->tp_as_number->nb_subtract)
-                    ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                if (obj->ob_type != &PyLong_Type)
+                    ret = PyLong_FromLong(-2);
                 else
-                    ret = PyLong_FromLong(-1);
+                {
+                    if (obj->ob_type->tp_as_number->nb_subtract)
+                        ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                    else
+                        ret = PyLong_FromLong(-1);
+                }
                 Py_DECREF(obj);
                 return ret;
              """),
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -64,6 +64,18 @@
         if self.st_ctime is None:
             self.__dict__['st_ctime'] = self[9]
 
+    @property
+    def st_atime_ns(self):
+        return int(self[7]) * 1000000000 + self.nsec_atime
+
+    @property
+    def st_mtime_ns(self):
+        return int(self[8]) * 1000000000 + self.nsec_mtime
+
+    @property
+    def st_ctime_ns(self):
+        return int(self[9]) * 1000000000 + self.nsec_ctime
+
 
 class statvfs_result(metaclass=structseqtype):
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -359,56 +359,43 @@
 
 STAT_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS))
 
-N_INDEXABLE_FIELDS = 10
-
-def _time_ns_from_float(ftime):
-    "Convert a floating-point time (in seconds) into a (s, ns) pair of ints"
-    fracpart, intpart = modf(ftime)
-    if fracpart < 0:
-        fracpart += 1.
-        intpart -= 1.
-    return int(intpart), int(fracpart * 1e9)
-
- at specialize.arg(4)
-def _fill_time(space, lst, index, w_keywords, attrname, ftime):
-    stat_float_times = space.fromcache(StatState).stat_float_times
-    seconds, fractional_ns = _time_ns_from_float(ftime)
-    lst[index] = space.wrap(seconds)
-    if stat_float_times:
-        space.setitem(w_keywords, space.wrap(attrname), space.wrap(ftime))
-    else:
-        space.setitem(w_keywords, space.wrap(attrname), space.wrap(seconds))
-    w_billion = space.wrap(1000000000)
-    w_total_ns = space.add(space.mul(space.wrap(seconds), w_billion),
-                           space.wrap(fractional_ns))
-    space.setitem(w_keywords, space.wrap(attrname + '_ns'), w_total_ns)
-
-STANDARD_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS[:7]))
-EXTRA_FIELDS = unrolling_iterable(rposix_stat.STAT_FIELDS[10:])
+STATVFS_FIELDS = unrolling_iterable(enumerate(rposix_stat.STATVFS_FIELDS))
 
 def build_stat_result(space, st):
-    lst = [None] * N_INDEXABLE_FIELDS
+    FIELDS = STAT_FIELDS    # also when not translating at all
+    lst = [None] * rposix_stat.N_INDEXABLE_FIELDS
     w_keywords = space.newdict()
-    for (i, (name, TYPE)) in STANDARD_FIELDS:
-        value = getattr(st, name)
-        w_value = space.wrap(value)
-        lst[i] = w_value
+    stat_float_times = space.fromcache(StatState).stat_float_times
+    for i, (name, TYPE) in FIELDS:
+        if i < rposix_stat.N_INDEXABLE_FIELDS:
+            # get the first 10 items by indexing; this gives us
+            # 'st_Xtime' as an integer, too
+            w_value = space.wrap(st[i])
+            lst[i] = w_value
+        else:
+            w_value = space.wrap(getattr(st, name))
+            space.setitem(w_keywords, space.wrap(name), w_value)
 
-    _fill_time(space, lst, 7, w_keywords, 'st_atime', st.st_atime)
-    _fill_time(space, lst, 8, w_keywords, 'st_mtime', st.st_mtime)
-    _fill_time(space, lst, 9, w_keywords, 'st_ctime', st.st_ctime)
+    # Note: 'w_keywords' contains the three attributes 'nsec_Xtime'.
+    # We have an app-level property in app_posix.stat_result to
+    # compute the full 'st_Xtime_ns' value.
 
-    for name, TYPE in EXTRA_FIELDS:
-        value = getattr(st, name)
-        w_value = space.wrap(value)
-        space.setitem(w_keywords, space.wrap(name), w_value)
+    # non-rounded values for name-based access
+    if stat_float_times:
+        space.setitem(w_keywords,
+                      space.wrap('st_atime'), space.wrap(st.st_atime))
+        space.setitem(w_keywords,
+                      space.wrap('st_mtime'), space.wrap(st.st_mtime))
+        space.setitem(w_keywords,
+                      space.wrap('st_ctime'), space.wrap(st.st_ctime))
+    #else:
+    #   filled by the __init__ method
 
     w_tuple = space.newtuple(lst)
     w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
                                   space.wrap('stat_result'))
     return space.call_function(w_stat_result, w_tuple, w_keywords)
 
-STATVFS_FIELDS = unrolling_iterable(enumerate(rposix_stat.STATVFS_FIELDS))
 
 def build_statvfs_result(space, st):
     vals_w = [None] * len(rposix_stat.STATVFS_FIELDS)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -125,9 +125,9 @@
         assert st[4] == st.st_uid
         assert st[5] == st.st_gid
         assert st[6] == st.st_size
-        assert st[7] == int(st.st_atime)
-        assert st[8] == int(st.st_mtime)
-        assert st[9] == int(st.st_ctime)
+        assert st[7] == int(st.st_atime)   # in complete corner cases, rounding
+        assert st[8] == int(st.st_mtime)   # here could maybe get the wrong
+        assert st[9] == int(st.st_ctime)   # integer...
 
         assert stat.S_IMODE(st.st_mode) & stat.S_IRUSR
         assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR
@@ -137,13 +137,12 @@
         assert st.st_size == 14
         assert st.st_nlink == 1
 
-        #if sys.platform.startswith('linux'):
-        #    # expects non-integer timestamps - it's unlikely that they are
-        #    # all three integers
-        #    assert ((st.st_atime, st.st_mtime, st.st_ctime) !=
-        #            (st[7],       st[8],       st[9]))
-        #    assert st.st_blksize * st.st_blocks >= st.st_size
+        assert not hasattr(st, 'nsec_atime')
+
         if sys.platform.startswith('linux'):
+            assert isinstance(st.st_atime, float)
+            assert isinstance(st.st_mtime, float)
+            assert isinstance(st.st_ctime, float)
             assert hasattr(st, 'st_rdev')
 
     def test_stat_float_times(self):
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -242,13 +242,33 @@
     from rpython.rtyper.lltypesystem import lltype, rffi
     return space.wrap(rffi.cast(lltype.Signed, handle))
 
+getsizeof_missing = """sys.getsizeof() is not implemented on PyPy.
+
+A memory profiler using this function is most likely to give results
+inconsistent with reality on PyPy.  It would be possible to have
+sys.getsizeof() return a number (with enough work), but that may or
+may not represent how much memory the object uses.  It doesn't even
+make really sense to ask how much *one* object uses, in isolation
+with the rest of the system.  For example, instances have maps,
+which are often shared across many instances; in this case the maps
+would probably be ignored by an implementation of sys.getsizeof(),
+but their overhead is important in some cases if they are many
+instances with unique maps.  Conversely, equal strings may share
+their internal string data even if they are different objects---or
+empty containers may share parts of their internals as long as they
+are empty.  Even stranger, some lists create objects as you read
+them; if you try to estimate the size in memory of range(10**6) as
+the sum of all items' size, that operation will by itself create one
+million integer objects that never existed in the first place.
+"""
+
 def getsizeof(space, w_object, w_default=None):
-    """Not implemented on PyPy."""
     if w_default is None:
-        raise oefmt(space.w_TypeError,
-                    "sys.getsizeof() not implemented on PyPy")
+        raise oefmt(space.w_TypeError, getsizeof_missing)
     return w_default
 
+getsizeof.__doc__ = getsizeof_missing
+
 def intern(space, w_str):
     """``Intern'' the given string.  This enters the string in the (global)
 table of interned strings whose purpose is to speed up dictionary lookups.
@@ -257,4 +277,3 @@
     if space.is_w(space.type(w_str), space.w_unicode):
         return space.new_interned_w_str(w_str)
     raise oefmt(space.w_TypeError, "intern() argument must be string.")
-
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -480,3 +480,7 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_negative_array_size(self):
+        ffi = FFI()
+        py.test.raises(ValueError, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -503,3 +503,7 @@
     assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
     p = ffi.new("int[]", [-123456789])
     assert ffi.unpack(p, 1) == [-123456789]
+
+def test_negative_array_size():
+    ffi = _cffi1_backend.FFI()
+    py.test.raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1981,3 +1981,29 @@
         static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
+
+def test_typedef_array_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef int foo_t[...], bar_t[...];
+        int gv[...];
+        typedef int mat_t[...][...];
+        typedef int vmat_t[][...];
+        """)
+    lib = verify(ffi, "test_typedef_array_dotdotdot", """
+        typedef int foo_t[50], bar_t[50];
+        int gv[23];
+        typedef int mat_t[6][7];
+        typedef int vmat_t[][8];
+    """)
+    assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+    assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+    assert len(ffi.new("foo_t")) == 50
+    assert len(ffi.new("bar_t")) == 50
+    assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+    assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+    assert len(ffi.new("mat_t")) == 6
+    assert len(ffi.new("mat_t")[3]) == 7
+    py.test.raises(ffi.error, ffi.sizeof, "vmat_t")
+    p = ffi.new("vmat_t", 4)
+    assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -288,6 +288,10 @@
             return W_SmallLongObject.fromint(val)
         return W_LongObject.fromint(self, val)
 
+    @specialize.argtype(1)
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+        return W_LongObject.fromrarith_int(val)
+
     def newlong_from_rbigint(self, val):
         return newlong(self, val)
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -26,7 +26,6 @@
         space.raises_w(space.w_OverflowError, space.float_w, w_big)
 
     def test_rint_variants(self):
-        py.test.skip("XXX broken!")
         from rpython.rtyper.tool.rfficache import platform
         space = self.space
         for r in platform.numbertype_to_rclass.values():
@@ -37,8 +36,8 @@
             for x in values:
                 if not r.SIGNED:
                     x &= r.MASK
-                w_obj = space.wrap(r(x))
-                assert space.bigint_w(w_obj).eq(rbigint.fromint(x))
+                w_obj = space.newlong_from_rarith_int(r(x))
+                assert space.bigint_w(w_obj).eq(rbigint.fromlong(x))
 
 
 class AppTestLong:
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -805,9 +805,7 @@
         class AA(object):
             __slots__ = ('a',)
         aa = AA()
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "aa.__class__ = A")
+        aa.__class__ = A
         raises(TypeError, "aa.__class__ = object")
         class Z1(A):
             pass
@@ -869,9 +867,13 @@
             __slots__ = ['a', 'b']
         class Order2(object):
             __slots__ = ['b', 'a']
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "Order1().__class__ = Order2")
+        Order1().__class__ = Order2
+
+        # like CPython, the order of slot names doesn't matter
+        x = Order1()
+        x.a, x.b = 1, 2
+        x.__class__ = Order2
+        assert (x.a, x.b) == (1, 2)
 
         class U1(object):
             __slots__ = ['a', 'b']
@@ -881,10 +883,11 @@
             __slots__ = ['a', 'b']
         class V2(V1):
             __slots__ = ['c', 'd', 'e']
-        # the following line does not work on CPython >= 2.6 either.
-        # that's just obscure.  Really really.  So we just ignore
-        # the whole issue until someone comes complaining.  Then we'll
-        # just kill slots altogether apart from maybe doing a few checks.
+        # the following line does not work on CPython either: we can't
+        # change a class if the old and new class have different layouts
+        # that look compatible but aren't, because they don't have the
+        # same base-layout class (even if these base classes are
+        # themselves compatible)...  obscure.
         raises(TypeError, "U2().__class__ = V2")
 
     def test_name(self):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -103,9 +103,10 @@
     """
     _immutable_ = True
 
-    def __init__(self, typedef, nslots, base_layout=None):
+    def __init__(self, typedef, nslots, newslotnames=[], base_layout=None):
         self.typedef = typedef
         self.nslots = nslots
+        self.newslotnames = newslotnames[:]    # make a fixed-size list
         self.base_layout = base_layout
 
     def issublayout(self, parent):
@@ -115,6 +116,12 @@
                 return False
         return True
 
+    def expand(self, hasdict, weakrefable):
+        """Turn this Layout into a tuple.  If two classes get equal
+        tuples, it means their instances have a fully compatible layout."""
+        return (self.typedef, self.newslotnames, self.base_layout,
+                hasdict, weakrefable)
+
 
 # possible values of compares_by_identity_status
 UNKNOWN = 0
@@ -287,8 +294,7 @@
 
     # compute a tuple that fully describes the instance layout
     def get_full_instance_layout(self):
-        layout = self.layout
-        return (layout, self.hasdict, self.weakrefable)
+        return self.layout.expand(self.hasdict, self.weakrefable)
 
     def compute_default_mro(self):
         return compute_C3_mro(self.space, self)
@@ -1022,11 +1028,15 @@
         w_self.weakrefable = w_self.weakrefable or w_base.weakrefable
     return hasoldstylebase
 
+
 def create_all_slots(w_self, hasoldstylebase, w_bestbase, force_new_layout):
+    from pypy.objspace.std.listobject import StringSort
+
     base_layout = w_bestbase.layout
     index_next_extra_slot = base_layout.nslots
     space = w_self.space
     dict_w = w_self.dict_w
+    newslotnames = []
     if '__slots__' not in dict_w:
         wantdict = True
         wantweakref = True
@@ -1052,9 +1062,22 @@
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
             else:
-                index_next_extra_slot = create_slot(w_self, w_slot_name,
-                                                    slot_name,
-                                                    index_next_extra_slot)
+                newslotnames.append(slot_name)
+        # Sort the list of names collected so far
+        sorter = StringSort(newslotnames, len(newslotnames))
+        sorter.sort()
+        # Try to create all slots in order.  The creation of some of
+        # them might silently fail; then we delete the name from the
+        # list.  At the end, 'index_next_extra_slot' has been advanced
+        # by the final length of 'newslotnames'.
+        i = 0
+        while i < len(newslotnames):
+            if create_slot(w_self, newslotnames[i], index_next_extra_slot):
+                index_next_extra_slot += 1
+                i += 1
+            else:
+                del newslotnames[i]
+    #
     wantdict = wantdict or hasoldstylebase
     if wantdict:
         create_dict_slot(w_self)
@@ -1063,13 +1086,14 @@
     if '__del__' in dict_w:
         w_self.hasuserdel = True
     #
+    assert index_next_extra_slot == base_layout.nslots + len(newslotnames)
     if index_next_extra_slot == base_layout.nslots and not force_new_layout:
         return base_layout
     else:
         return Layout(base_layout.typedef, index_next_extra_slot,
-                      base_layout=base_layout)
+                      newslotnames, base_layout=base_layout)
 
-def create_slot(w_self, w_slot_name, slot_name, index_next_extra_slot):
+def create_slot(w_self, slot_name, index_next_extra_slot):
     space = w_self.space
     if not valid_slot_name(slot_name):
         raise oefmt(space.w_TypeError, "__slots__ must be identifiers")
@@ -1077,16 +1101,17 @@
     slot_name = mangle(slot_name, w_self.name)
     if slot_name in w_self.dict_w:
         raise oefmt(space.w_ValueError,
-                    "%R in __slots__ conflicts with class variable",
-                    w_slot_name)
+                    "'%8' in __slots__ conflicts with class variable",
+                    slot_name)
     else:
         # Force interning of slot names.
         slot_name = space.str_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
         member = Member(index_next_extra_slot, slot_name, w_self)
-        index_next_extra_slot += 1
         w_self.dict_w[slot_name] = space.wrap(member)
-    return index_next_extra_slot
+        return True
+    else:
+        return False
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -200,8 +200,12 @@
             or v.concretetype != lltype.Bool):
             return False
         for op in block.operations[::-1]:
-            if v in op.args:
-                return False   # variable is also used in cur block
+            # check if variable is used in block
+            for arg in op.args:
+                if arg == v:
+                    return False
+                if isinstance(arg, ListOfKind) and v in arg.content:
+                    return False
             if v is op.result:
                 if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne',
                                      'int_gt', 'int_ge',
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -243,6 +243,20 @@
         assert block.exitswitch == (opname, v1, '-live-before')
         assert block.exits == exits
 
+def test_optimize_goto_if_not__argument_to_call():
+    for opname in ['ptr_iszero', 'ptr_nonzero']:
+        v1 = Variable()
+        v3 = Variable(); v3.concretetype = lltype.Bool
+        v4 = Variable()
+        block = Block([v1])
+        callop = SpaceOperation('residual_call_r_i',
+                ["fake", ListOfKind('int', [v3])], v4)
+        block.operations = [SpaceOperation(opname, [v1], v3), callop]
+        block.exitswitch = v3
+        block.exits = exits = [FakeLink(False), FakeLink(True)]
+        res = Transformer().optimize_goto_if_not(block)
+        assert not res
+
 def test_symmetric():
     ops = {'int_add': 'int_add',
            'int_or': 'int_or',
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -90,18 +90,23 @@
         assert vs.make_inputargs(args, optimizer) == []
 
     def test_make_inputargs_2(self):
-        # Ensure that make_inputargs properly errors with VirtualStatesCantMatch
-        # when the type information for a virtual field conflicts. In practice the
-        # expected and given field always share a common subclass.
-        # This check is needed as not all paths to make_inputargs in unroll.py
-        # are guarded by a call to generate_guards.
+        # Ensure that make_inputargs does not error when the lengths of the fields
+        # for the runtime box does not match what the virtual state expected.
+        # This can occur in unroll.py, as not all paths to make_inputargs are
+        # guareded with a generalization_of check. The property is validated
+        # subsequently in all cases, so we just need to ensure that this case does
+        # not cause segfaults.
         optimizer = FakeOptimizer(self.cpu)
         classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
-        innervalue1 = info.InstancePtrInfo(known_class=classbox1, is_virtual=True, descr=self.valuedescr.get_parent_descr())
+        innervalue1 = info.InstancePtrInfo(
+                known_class=classbox1, is_virtual=True,
+                descr=self.valuedescr.get_parent_descr())
         for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
             innervalue1.setfield(field, None, ConstInt(42))
         classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3))
-        innervalue2 = info.InstancePtrInfo(known_class=classbox2, is_virtual=True, descr=self.valuedescr3.get_parent_descr())
+        innervalue2 = info.InstancePtrInfo(
+                known_class=classbox2, is_virtual=True,
+                descr=self.valuedescr3.get_parent_descr())
         for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
             innervalue2.setfield(field, None, ConstInt(42))
 
@@ -111,10 +116,14 @@
         nodebox2.set_forwarded(innervalue2)
 
         constr = VirtualStateConstructor(optimizer)
-        vs = constr.get_virtual_state([nodebox1])
+        vs1 = constr.get_virtual_state([nodebox1])
+        constr = VirtualStateConstructor(optimizer)
+        vs2 = constr.get_virtual_state([nodebox2])
 
-        with py.test.raises(VirtualStatesCantMatch):
-            args = vs.make_inputargs([nodebox2], optimizer, force_boxes=True)
+        # This should succeed with no exceptions
+        vs1.make_inputargs([nodebox2], optimizer, force_boxes=False)
+        assert not vs1.generalization_of(vs2, optimizer)
+        assert not vs2.generalization_of(vs1, optimizer)
 
     def test_position_generalization(self):
         def postest(info1, info2):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -167,7 +167,8 @@
                [self.get_box_replacement(x) for x in end_jump.getarglist()],
                self.optimizer, force_boxes=True)
             for arg in args:
-                self.optimizer.force_box(arg)
+                if arg is not None:
+                    self.optimizer.force_box(arg)
         except VirtualStatesCantMatch:
             raise InvalidLoop("Virtual states did not match "
                               "after picking the virtual state, when forcing"
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -177,14 +177,6 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
-    @staticmethod
-    def descr_issubclass(descr1, descr2, optimizer):
-        if not descr1.is_object() or not descr2.is_object():
-            return True
-        vtable1 = descr1.get_vtable()
-        vtable2 = descr2.get_vtable()
-        return optimizer._check_subclass(vtable1, vtable2)
-
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
@@ -193,13 +185,12 @@
         else:
             assert isinstance(info, AbstractStructPtrInfo)
 
-        for i in range(len(self.fielddescrs)):
+        # The min operation ensures we don't wander off either array, as not all
+        # to make_inputargs have validated their inputs with generate_guards.
+        for i in range(min(len(self.fielddescrs), len(info._fields))):
             state = self.fieldstate[i]
-            descr = self.fielddescrs[i].get_parent_descr()
             if not state:
                 continue
-            if not self.descr_issubclass(info.descr, descr, optimizer.optimizer):
-                raise VirtualStatesCantMatch()
             if state.position > self.position:
                 fieldbox = info._fields[i]
                 state.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4558,3 +4558,20 @@
         self.meta_interp(f, [])
         self.check_resops(guard_nonnull=0)
 
+    def test_loop_before_main_loop(self):
+        fdriver = JitDriver(greens=[], reds='auto')
+        gdriver = JitDriver(greens=[], reds='auto')
+        def f(i, j):
+            while j > 0:   # this loop unrolls because it is in the same
+                j -= 1     # function as a jit_merge_point()
+            while i > 0:
+                fdriver.jit_merge_point()
+                i -= 1
+        def g(i, j, k):
+            while k > 0:
+                gdriver.jit_merge_point()
+                f(i, j)
+                k -= 1
+
+        self.meta_interp(g, [5, 5, 5])
+        self.check_resops(guard_true=10)   # 5 unrolled, plus 5 unrelated
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.error import TyperError
 
 from rpython.rlib._os_support import _preferred_traits, string_traits
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask
@@ -51,15 +51,18 @@
     ("st_uid",       lltype.Signed),
     ("st_gid",       lltype.Signed),
     ("st_size",      lltype.SignedLongLong),
-    ("st_atime",     lltype.Float),
-    ("st_mtime",     lltype.Float),
-    ("st_ctime",     lltype.Float),
+    ("st_atime",     lltype.SignedLongLong),   # integral number of seconds
+    ("st_mtime",     lltype.SignedLongLong),   #
+    ("st_ctime",     lltype.SignedLongLong),   #
     ("st_blksize",   lltype.Signed),
     ("st_blocks",    lltype.Signed),
     ("st_rdev",      lltype.Signed),
     ("st_flags",     lltype.Signed),
     #("st_gen",       lltype.Signed),     -- new in CPy 2.5, not implemented
     #("st_birthtime", lltype.Float),      -- new in CPy 2.5, not implemented
+    ("nsec_atime",   lltype.Signed),   # number of nanoseconds
+    ("nsec_mtime",   lltype.Signed),   #
+    ("nsec_ctime",   lltype.Signed),   #
 ]
 N_INDEXABLE_FIELDS = 10
 
@@ -79,6 +82,37 @@
     ("f_namemax", lltype.Signed),
 ]
 
+ at specialize.arg(1)
+def get_stat_ns_as_bigint(st, name):
+    """'name' is one of the strings "atime", "mtime" or "ctime".
+    Returns a bigint that represents the number of nanoseconds
+    stored inside the RPython-level os.stat_result 'st'.
+
+    Note that when running untranslated, the os.stat_result type
+    is from Python 2.7, which doesn't store more precision than
+    a float anyway.  You will only get more after translation.
+    """
+    from rpython.rlib.rbigint import rbigint
+
+    if not we_are_translated():
+        as_float = getattr(st, "st_" + name)
+        return rbigint.fromfloat(as_float * 1e9)
+
+    if name == "atime":
+        i, j = 7, -3
+    elif name == "mtime":
+        i, j = 8, -2
+    elif name == "ctime":
+        i, j = 9, -1
+    else:
+        raise AssertionError(name)
+
+    sec = st[i]
+    nsec = st[j]
+    result = rbigint.fromrarith_int(sec).int_mul(1000000000)
+    result = result.int_add(nsec)
+    return result
+
 
 # ____________________________________________________________
 #
@@ -97,7 +131,15 @@
         if not s_attr.is_constant():
             raise annmodel.AnnotatorError("non-constant attr name in getattr()")
         attrname = s_attr.const
-        TYPE = STAT_FIELD_TYPES[attrname]
+        if attrname in ('st_atime', 'st_mtime', 'st_ctime'):
+            # like CPython, in RPython we can read the st_Xtime
+            # attribute and get a floating-point result.  We can also
+            # get a full-precision bigint with get_stat_ns_as_bigint().
+            # The floating-point result is computed like a property
+            # by _ll_get_st_Xtime().
+            TYPE = lltype.Float
+        else:
+            TYPE = STAT_FIELD_TYPES[attrname]
         return lltype_to_annotation(TYPE)
 
     def _get_rmarshall_support_(self):     # for rlib.rmarshal
@@ -105,13 +147,14 @@
         # (we ignore the extra values here for simplicity and portability)
         def stat_result_reduce(st):
             return (st[0], st[1], st[2], st[3], st[4],
-                    st[5], st[6], st[7], st[8], st[9])
+                    st[5], st[6], st[7], st[8], st[9],
+                    st[-3], st[-2], st[-1])
 
         def stat_result_recreate(tup):
-            return make_stat_result(tup + extra_zeroes)
+            return make_stat_result(tup[:10] + extra_zeroes + tup[-3:])
         s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE)
                                        for name, TYPE in PORTABLE_STAT_FIELDS])
-        extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS))
+        extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS) - 3)
         return s_reduced, stat_result_reduce, stat_result_recreate
 
 
@@ -119,7 +162,7 @@
     def getitem((s_sta, s_int)):
         assert s_int.is_constant(), "os.stat()[index]: index must be constant"
         index = s_int.const
-        assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range"
+        assert -3 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range"
         name, TYPE = STAT_FIELDS[index]
         return lltype_to_annotation(TYPE)
 
@@ -152,28 +195,61 @@
     def rtype_getattr(self, hop):
         s_attr = hop.args_s[1]
         attr = s_attr.const
+        if attr in ('st_atime', 'st_mtime', 'st_ctime'):
+            ll_func = globals()['_ll_get_' + attr]
+            v_tuple = hop.inputarg(self, arg=0)
+            return hop.gendirectcall(ll_func, v_tuple)
         try:
             index = self.stat_field_indexes[attr]
         except KeyError:
             raise TyperError("os.stat().%s: field not available" % (attr,))
         return self.redispatch_getfield(hop, index)
 
+ at specialize.memo()
+def _stfld(name):
+    index = STAT_FIELD_NAMES.index(name)
+    return 'item%d' % index
+
+def _ll_get_st_atime(tup):
+    return (float(getattr(tup, _stfld("st_atime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_atime")))
+
+def _ll_get_st_mtime(tup):
+    return (float(getattr(tup, _stfld("st_mtime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_mtime")))
+
+def _ll_get_st_ctime(tup):
+    return (float(getattr(tup, _stfld("st_ctime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_ctime")))
+
 
 class __extend__(pairtype(StatResultRepr, IntegerRepr)):
     def rtype_getitem((r_sta, r_int), hop):
         s_int = hop.args_s[1]
         index = s_int.const
+        if index < 0:
+            index += len(STAT_FIELDS)
         return r_sta.redispatch_getfield(hop, index)
 
 s_StatResult = SomeStatResult()
 
+
 def make_stat_result(tup):
-    """Turn a tuple into an os.stat_result object."""
-    positional = tuple(
-        lltype.cast_primitive(TYPE, value) for value, (name, TYPE) in
-            zip(tup, STAT_FIELDS)[:N_INDEXABLE_FIELDS])
+    """NOT_RPYTHON: Turn a tuple into an os.stat_result object."""
+    assert len(tup) == len(STAT_FIELDS)
+    assert float not in [type(x) for x in tup]
+    positional = []
+    for i in range(N_INDEXABLE_FIELDS):
+        name, TYPE = STAT_FIELDS[i]
+        value = lltype.cast_primitive(TYPE, tup[i])
+        positional.append(value)
     kwds = {}
+    kwds['st_atime'] = tup[7] + 1e-9 * tup[-3]
+    kwds['st_mtime'] = tup[8] + 1e-9 * tup[-2]
+    kwds['st_ctime'] = tup[9] + 1e-9 * tup[-1]
     for value, (name, TYPE) in zip(tup, STAT_FIELDS)[N_INDEXABLE_FIELDS:]:
+        if name.startswith('nsec_'):
+            continue   # ignore the nsec_Xtime here
         kwds[name] = lltype.cast_primitive(TYPE, value)
     return os.stat_result(positional, kwds)
 
@@ -360,6 +436,8 @@
         posix_declaration(ALL_STAT_FIELDS[_i])
     del _i
 
+STAT_FIELDS += ALL_STAT_FIELDS[-3:]   # nsec_Xtime
+
 # these two global vars only list the fields defined in the underlying platform
 STAT_FIELD_TYPES = dict(STAT_FIELDS)      # {'st_xxx': TYPE}
 STAT_FIELD_NAMES = [_name for (_name, _TYPE) in STAT_FIELDS]
@@ -368,16 +446,20 @@
 STATVFS_FIELD_TYPES = dict(STATVFS_FIELDS)
 STATVFS_FIELD_NAMES = [name for name, tp in STATVFS_FIELDS]
 
+
 def build_stat_result(st):
     # only for LL backends
     if TIMESPEC is not None:
-        atim = st.c_st_atim; atime = int(atim.c_tv_sec) + 1E-9 * int(atim.c_tv_nsec)
-        mtim = st.c_st_mtim; mtime = int(mtim.c_tv_sec) + 1E-9 * int(mtim.c_tv_nsec)
-        ctim = st.c_st_ctim; ctime = int(ctim.c_tv_sec) + 1E-9 * int(ctim.c_tv_nsec)
+        atim = st.c_st_atim
+        mtim = st.c_st_mtim
+        ctim = st.c_st_ctim
+        atime, extra_atime = atim.c_tv_sec, int(atim.c_tv_nsec)
+        mtime, extra_mtime = mtim.c_tv_sec, int(mtim.c_tv_nsec)
+        ctime, extra_ctime = ctim.c_tv_sec, int(ctim.c_tv_nsec)
     else:
-        atime = st.c_st_atime
-        mtime = st.c_st_mtime
-        ctime = st.c_st_ctime
+        atime, extra_atime = st.c_st_atime, 0
+        mtime, extra_mtime = st.c_st_mtime, 0
+        ctime, extra_ctime = st.c_st_ctime, 0
 
     result = (st.c_st_mode,
               st.c_st_ino,
@@ -395,6 +477,10 @@
     if "st_rdev"    in STAT_FIELD_TYPES: result += (st.c_st_rdev,)
     if "st_flags"   in STAT_FIELD_TYPES: result += (st.c_st_flags,)
 
+    result += (extra_atime,
+               extra_mtime,
+               extra_ctime)
+
     return make_stat_result(result)
 
 
@@ -455,12 +541,14 @@
             # console or LPT device
             return make_stat_result((win32traits._S_IFCHR,
                                      0, 0, 0, 0, 0,
-                                     0, 0, 0, 0))
+                                     0, 0, 0, 0,
+                                     0, 0, 0))
         elif filetype == win32traits.FILE_TYPE_PIPE:
             # socket or named pipe
             return make_stat_result((win32traits._S_IFIFO,
                                      0, 0, 0, 0, 0,
-                                     0, 0, 0, 0))
+                                     0, 0, 0, 0,
+                                     0, 0, 0))
         elif filetype == win32traits.FILE_TYPE_UNKNOWN:
             error = rwin32.GetLastError_saved()
             if error != 0:
@@ -539,14 +627,11 @@
 #__________________________________________________
 # Helper functions for win32
 if _WIN32:
-    from rpython.rlib.rwin32file import FILE_TIME_to_time_t_float
+    from rpython.rlib.rwin32file import FILE_TIME_to_time_t_nsec
 
     def make_longlong(high, low):
         return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
 
-    # Seconds between 1.1.1601 and 1.1.1970
-    secs_between_epochs = rffi.r_longlong(11644473600)
-
     @specialize.arg(0)
     def win32_xstat(traits, path, traverse=False):
         win32traits = make_win32_traits(traits)
@@ -582,14 +667,15 @@
     def win32_attribute_data_to_stat(win32traits, info):
         st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes)
         st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow)
-        ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime)
-        mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime)
-        atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime)
+        ctime, extra_ctime = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime)
+        mtime, extra_mtime = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime)
+        atime, extra_atime = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime)
 
         result = (st_mode,
                   0, 0, 0, 0, 0,
                   st_size,
-                  atime, mtime, ctime)
+                  atime, mtime, ctime,
+                  extra_atime, extra_mtime, extra_ctime)
 
         return make_stat_result(result)
 
@@ -597,9 +683,9 @@
         # similar to the one above
         st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes)
         st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow)
-        ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime)
-        mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime)
-        atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime)
+        ctime, extra_ctime = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime)
+        mtime, extra_mtime = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime)
+        atime, extra_atime = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime)
 
         # specific to fstat()
         st_ino = make_longlong(info.c_nFileIndexHigh, info.c_nFileIndexLow)
@@ -608,7 +694,8 @@
         result = (st_mode,
                   st_ino, 0, st_nlink, 0, 0,
                   st_size,
-                  atime, mtime, ctime)
+                  atime, mtime, ctime,
+                  extra_atime, extra_mtime, extra_ctime)
 
         return make_stat_result(result)
 
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -6,6 +6,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import intmask
 
 @specialize.memo()
 def make_win32_traits(traits):
@@ -213,13 +214,25 @@
     return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
 
 # Seconds between 1.1.1601 and 1.1.1970
-secs_between_epochs = rffi.r_longlong(11644473600)
+secs_between_epochs = 11644473600.0
+hns_between_epochs = rffi.r_longlong(116444736000000000)  # units of 100 nsec
 
 def FILE_TIME_to_time_t_float(filetime):
     ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
     # FILETIME is in units of 100 nsec
     return float(ft) * (1.0 / 10000000.0) - secs_between_epochs
 
+def FILE_TIME_to_time_t_nsec(filetime):
+    """Like the previous function, but returns a pair: (integer part
+    'time_t' as a r_longlong, fractional part as an int measured in
+    nanoseconds).
+    """
+    ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
+    ft -= hns_between_epochs
+    int_part = ft / 10000000
+    frac_part = ft - (int_part * 10000000)
+    return (int_part, intmask(frac_part) * 100)
+
 def time_t_to_FILE_TIME(time, filetime):
     ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
diff --git a/rpython/rlib/test/test_rposix_stat.py b/rpython/rlib/test/test_rposix_stat.py
--- a/rpython/rlib/test/test_rposix_stat.py
+++ b/rpython/rlib/test/test_rposix_stat.py
@@ -2,12 +2,20 @@
 import py
 from rpython.rlib import rposix_stat
 from rpython.tool.udir import udir
+from rpython.translator.c.test.test_genc import compile
+from rpython.rtyper.lltypesystem import lltype
+
 
 class TestPosixStatFunctions:
     @py.test.mark.skipif("sys.platform == 'win32'",
                          reason="win32 only has the portable fields")
     def test_has_all_fields(self):
-        assert rposix_stat.STAT_FIELDS == rposix_stat.ALL_STAT_FIELDS[:13]
+        # XXX this test is obscure!  it will fail if the exact set of
+        # XXX stat fields found differs from the one we expect on Linux.
+        # XXX Why?
+        assert rposix_stat.STAT_FIELDS == (
+            rposix_stat.ALL_STAT_FIELDS[:13] +
+            rposix_stat.ALL_STAT_FIELDS[-3:])
 
     def test_stat(self):
         def check(f):
@@ -66,3 +74,27 @@
     finally:
         os.close(dirfd)
     assert result.st_atime == tmpdir.join('file').atime()
+
+def test_high_precision_stat_time():
+    def f():
+        st = os.stat('.')
+        # should be supported on all platforms, but give a result whose
+        # precision might be lower than full nanosecond
+        highprec = rposix_stat.get_stat_ns_as_bigint(st, "ctime")
+        return '%s;%s' % (st.st_ctime, highprec.str())
+    fc = compile(f, [])
+    as_string = fc()
+    asfloat, highprec = as_string.split(';')
+    asfloat = float(asfloat)
+    highprec = int(highprec)
+    st = os.stat('.')
+    assert abs(asfloat - st.st_ctime) < 500e-9
+    assert abs(highprec - int(st.st_ctime * 1e9)) < 500
+    assert abs(rposix_stat.get_stat_ns_as_bigint(st, "ctime").tolong()
+               - st.st_ctime * 1e9) < 3
+    if rposix_stat.TIMESPEC is not None:
+        with lltype.scoped_alloc(rposix_stat.STAT_STRUCT.TO) as stresult:
+            rposix_stat.c_stat(".", stresult)
+            assert 0 <= stresult.c_st_ctim.c_tv_nsec <= 999999999
+            assert highprec == (int(stresult.c_st_ctim.c_tv_sec) * 1000000000
+                                + int(stresult.c_st_ctim.c_tv_nsec))
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -646,6 +646,9 @@
     def ll_str(self, x):
         return self.getstr()
 
+    def get_ll_eq_function(self):
+        return None
+
 
 class MultipleFrozenPBCReprBase(CanBeNull, Repr):
     def convert_const(self, pbc):
@@ -654,6 +657,9 @@
         frozendesc = self.rtyper.annotator.bookkeeper.getdesc(pbc)
         return self.convert_desc(frozendesc)
 
+    def get_ll_eq_function(self):
+        return None
+
 class MultipleUnrelatedFrozenPBCRepr(MultipleFrozenPBCReprBase):
     """For a SomePBC of frozen PBCs that have no common access set.
     The only possible operation on such a thing is comparison with 'is'."""
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -1729,6 +1729,23 @@
         res = self.interpret(f, [])
         assert res == 42
 
+    def test_equality_of_frozen_pbcs_inside_data_structures(self):
+        class A:
+            def _freeze_(self):
+                return True
+        a1 = A()
+        a2 = A()
+        def f():
+            return [a1] == [a1]
+        def g(i):
+            x1 = [a1, a2][i]
+            x2 = [a1, a2][i]
+            return (x1,) == (x2,)
+        res = self.interpret(f, [])
+        assert res == True
+        res = self.interpret(g, [1])
+        assert res == True
+
 # ____________________________________________________________
 
 def test_hlinvoke_simple():
diff --git a/rpython/translator/backendopt/merge_if_blocks.py b/rpython/translator/backendopt/merge_if_blocks.py
--- a/rpython/translator/backendopt/merge_if_blocks.py
+++ b/rpython/translator/backendopt/merge_if_blocks.py
@@ -20,6 +20,14 @@
         return False
     if isinstance(op.args[0], Constant) and isinstance(op.args[1], Constant):
         return False
+    # check that the constant is hashable (ie not a symbolic)
+    try:
+        if isinstance(op.args[0], Constant):
+            hash(op.args[0].value)
+        else:
+            hash(op.args[1].value)
+    except TypeError:
+        return False
     return True
 
 def merge_chain(chain, checkvar, varmap, graph):
diff --git a/rpython/translator/backendopt/test/test_merge_if_blocks.py b/rpython/translator/backendopt/test/test_merge_if_blocks.py
--- a/rpython/translator/backendopt/test/test_merge_if_blocks.py
+++ b/rpython/translator/backendopt/test/test_merge_if_blocks.py
@@ -2,11 +2,12 @@
 from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks
 from rpython.translator.backendopt.all import backend_optimizations
 from rpython.translator.translator import TranslationContext, graphof as tgraphof
-from rpython.flowspace.model import Block
+from rpython.flowspace.model import Block, checkgraph
 from rpython.translator.backendopt.removenoops import remove_same_as
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, r_int
 from rpython.annotator.model import SomeChar, SomeUnicodeCodePoint
+from rpython.rlib.objectmodel import CDefinedIntSymbolic
 
 def do_test_merge(fn, testvalues):
     t = TranslationContext()
@@ -225,3 +226,29 @@
     malloc.remove_mallocs(t, t.graphs)
     from rpython.translator import simplify
     simplify.join_blocks(graph)
+
+def test_switch_on_symbolic():
+    symb1 = CDefinedIntSymbolic("1", 1)
+    symb2 = CDefinedIntSymbolic("2", 2)
+    symb3 = CDefinedIntSymbolic("3", 3)
+    def fn(x):
+        res = 0
+        if x == symb1:
+            res += x + 1
+        elif x == symb2:
+            res += x + 2
+        elif x == symb3:
+            res += x + 3
+        res += 1
+        return res
+    t = TranslationContext()
+    a = t.buildannotator()
+    a.build_types(fn, [int])

From pypy.commits at gmail.com  Wed Sep 21 08:22:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 05:22:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57e27b6d.c70a1c0a.b2b8c.0af5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87266:c7615cd6e615
Date: 2016-09-21 14:21 +0200
http://bitbucket.org/pypy/pypy/changeset/c7615cd6e615/

Log:	hg merge default

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -33,28 +33,27 @@
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if isinstance(w_initializer, W_ArrayBase):
-                    a.extend(w_initializer, True)
-                elif space.type(w_initializer) is space.w_list:
-                    a.descr_fromlist(space, w_initializer)
-                else:
-                    try:
-                        buf = space.bufferstr_w(w_initializer)
-                    except OperationError as e:
-                        if not e.match(space, space.w_TypeError):
-                            raise
-                        a.extend(w_initializer, True)
-                    else:
-                        a.descr_frombytes(space, buf)
             break
     else:
         raise oefmt(space.w_ValueError,
                     "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f "
                     "or d)")
 
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        if isinstance(w_initializer, W_ArrayBase):
+            a.extend(w_initializer, True)
+        elif space.type(w_initializer) is space.w_list:
+            a.descr_fromlist(space, w_initializer)
+        else:
+            try:
+                buf = space.bufferstr_w(w_initializer)
+            except OperationError as e:
+                if not e.match(space, space.w_TypeError):
+                    raise
+                a.extend(w_initializer, True)
+            else:
+                a.descr_frombytes(space, buf)
     return a
 
 

From pypy.commits at gmail.com  Wed Sep 21 08:29:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 05:29:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: fixes
Message-ID: <57e27d16.411d1c0a.1f8fe.17d7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87267:0ac14db93823
Date: 2016-09-21 14:28 +0200
http://bitbucket.org/pypy/pypy/changeset/0ac14db93823/

Log:	fixes

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -373,7 +373,14 @@
             w_value = space.wrap(st[i])
             lst[i] = w_value
         else:
-            w_value = space.wrap(getattr(st, name))
+            try:
+                value = getattr(st, name)
+            except AttributeError:
+                # untranslated, there is no nsec_Xtime attribute
+                assert name.startswith('nsec_')
+                value = rposix_stat.get_stat_ns_as_bigint(st, name[5:])
+                value = value.tolong() % 1000000000
+            w_value = space.wrap(value)
             space.setitem(w_keywords, space.wrap(name), w_value)
 
     # Note: 'w_keywords' contains the three attributes 'nsec_Xtime'.
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1099,11 +1099,7 @@
         raise oefmt(space.w_TypeError, "__slots__ must be identifiers")
     # create member
     slot_name = mangle(slot_name, w_self.name)
-    if slot_name in w_self.dict_w:
-        raise oefmt(space.w_ValueError,
-                    "'%8' in __slots__ conflicts with class variable",
-                    slot_name)
-    else:
+    if slot_name not in w_self.dict_w:
         # Force interning of slot names.
         slot_name = space.str_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
@@ -1111,7 +1107,10 @@
         w_self.dict_w[slot_name] = space.wrap(member)
         return True
     else:
-        return False
+        # never returns False, but that's to minimize the diff with pypy2
+        raise oefmt(space.w_ValueError,
+                    "'%8' in __slots__ conflicts with class variable",
+                    slot_name)
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:

From pypy.commits at gmail.com  Wed Sep 21 08:34:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 05:34:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Add tests for st.st_Xtime_ns
Message-ID: <57e27e63.516d1c0a.454d1.7baf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87268:703718ff9b54
Date: 2016-09-21 14:33 +0200
http://bitbucket.org/pypy/pypy/changeset/703718ff9b54/

Log:	Add tests for st.st_Xtime_ns

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -137,14 +137,17 @@
         assert st.st_size == 14
         assert st.st_nlink == 1
 
-        assert not hasattr(st, 'nsec_atime')
-
         if sys.platform.startswith('linux'):
             assert isinstance(st.st_atime, float)
             assert isinstance(st.st_mtime, float)
             assert isinstance(st.st_ctime, float)
             assert hasattr(st, 'st_rdev')
 
+        assert isinstance(st.st_atime_ns, int)
+        assert abs(st.st_atime_ns - 1e9*st.st_atime) < 500
+        assert abs(st.st_mtime_ns - 1e9*st.st_mtime) < 500
+        assert abs(st.st_ctime_ns - 1e9*st.st_ctime) < 500
+
     def test_stat_float_times(self):
         path = self.path
         posix = self.posix

From pypy.commits at gmail.com  Wed Sep 21 08:39:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 05:39:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Manual merge of a translation fix I've
 done only in the py3.5 branch (sorry)
Message-ID: <57e27f83.4a301c0a.592a3.1057@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87269:80e7245f0eb0
Date: 2016-09-21 14:38 +0200
http://bitbucket.org/pypy/pypy/changeset/80e7245f0eb0/

Log:	Manual merge of a translation fix I've done only in the py3.5 branch
	(sorry)

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -77,19 +77,21 @@
     def unicode_w(self, space):
         return self._value
 
-    @jit.elidable
     def identifier_w(self, space):
-        identifier = self._utf8
-        if identifier is not None:
-            return identifier
-        u = self._value
-        eh = unicodehelper.rpy_encode_error_handler()
         try:
-            identifier = unicode_encode_utf_8(u, len(u), None,
-                                              errorhandler=eh)
-        except unicodehelper.RUnicodeEncodeError as ue:
-            raise wrap_encode_error(space, ue)
-        self._utf8 = identifier
+            # call the elidable function, with a jit.call_shortcut in case
+            # self._utf8 is already computed
+            identifier = g_identifier_w(self, space)
+        except UnicodeEncodeError:
+            # bah, this is just to get an official app-level
+            # UnicodeEncodeError
+            u = self._value
+            eh = unicodehelper.rpy_encode_error_handler()
+            try:
+                identifier = unicode_encode_utf_8(u, len(u), None,
+                                                  errorhandler=eh)
+            except unicodehelper.RUnicodeEncodeError as ue:
+                raise wrap_encode_error(space, ue)
         return identifier
 
     def listview_unicode(self):
@@ -1306,6 +1308,16 @@
     return unicodehelper.encode_utf8(space, u''.join(result),
                                      allow_surrogates=allow_surrogates)
 
+ at jit.elidable
+ at jit.call_shortcut
+def g_identifier_w(self, space):
+    """This is a global function because of @jit.call_shortcut"""
+    identifier = self._utf8
+    if identifier is not None:
+        return identifier
+    identifier = self._value.encode('utf-8')
+    self._utf8 = identifier
+    return identifier
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=True, unicode_output=True, quotes=True, prefix='')

From pypy.commits at gmail.com  Wed Sep 21 09:05:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 06:05:51 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57e285af.87941c0a.4e555.0ee0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r800:c50cd1b3de71
Date: 2016-09-21 15:05 +0200
http://bitbucket.org/pypy/pypy.org/changeset/c50cd1b3de71/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $65133 of $105000 (62.0%)
+   $65142 of $105000 (62.0%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5583 left
+    we have $5591 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Sep 21 09:55:46 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 21 Sep 2016 06:55:46 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: remove some unused options,
 change some tests in micronumpy
Message-ID: <57e29162.0558c20a.88d0a.7768@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87270:c712f368bb91
Date: 2016-09-21 15:55 +0200
http://bitbucket.org/pypy/pypy/changeset/c712f368bb91/

Log:	remove some unused options, change some tests in micronumpy

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -7,19 +7,19 @@
 class TestMicroNumPy(BaseTestPyPyC):
 
     arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),
-             ('+','float','int',   9*7843,   7843, 4.0,5.0),
-             ('+','int','float',   8*2571,   2571, 9.0,-1.0),
-             ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
-             ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
-             ('-','float','float', -2*5523,  5523, 1.0,3.0),
-             ('*','float','float', 3*2999,   2999, 1.0,3.0),
-             ('/','float','float', 3*7632,   7632, 3.0,1.0),
-             ('/','float','float', 1.5*7632, 7632, 3.0,2.0),
-             ('&','int','int',     0,        1500, 1,0),
-             ('&','int','int',     1500,     1500, 1,1),
-             ('|','int','int',     1500,     1500, 0,1),
-             ('|','int','int',     0,        1500, 0,0),
-            ]
+                  ('+','float','int',   9*7843,   7843, 4.0,5.0),
+                  ('+','int','float',   8*2571,   2571, 9.0,-1.0),
+                  ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
+                  ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
+                  ('-','float','float', -2*5523,  5523, 1.0,3.0),
+                  ('*','float','float', 3*2999,   2999, 1.0,3.0),
+                  ('/','float','float', 3*7632,   7632, 3.0,1.0),
+                  ('/','float','float', 1.5*7632, 7632, 3.0,2.0),
+                  ('&','int','int',     0,        1500, 1,0),
+                  ('&','int','int',     1500,     1500, 1,1),
+                  ('|','int','int',     1500,     1500, 0,1),
+                  ('|','int','int',     0,        1500, 0,0),
+                 ]
     type_permuated = []
     types = { 'int': ['int32','int64','int8','int16'],
               'float': ['float32', 'float64']
@@ -54,7 +54,10 @@
         assert log.jit_summary.vecopt_tried == 0
         assert log.jit_summary.vecopt_success == 0
         assert vlog.jit_summary.vecopt_tried > 0
-        assert vlog.jit_summary.vecopt_success > 0
+        if adtype in ('int64','float64') and bdtype in ('int64','float64'):
+            assert vlog.jit_summary.vecopt_success > 0
+        else:
+            assert vlog.jit_summary.vecopt_success >= 0
 
 
     arith_comb = [
@@ -88,14 +91,17 @@
             return a.{method}()
         """.format(method=op, dtype=dtype, count=count, a=a)
         exec py.code.Source(source).compile()
+        log = self.run(main, [], vec=0)
         vlog = self.run(main, [], vec=1)
-        log = self.run(main, [], vec=0)
         assert log.result == vlog.result
         assert log.result == result
         assert log.jit_summary.vecopt_tried == 0
         assert log.jit_summary.vecopt_success == 0
         assert vlog.jit_summary.vecopt_tried > 0
-        assert vlog.jit_summary.vecopt_success > 0
+        if dtype in ('int64','float64') and (dtype != 'int64' and op != 'prod'):
+            assert vlog.jit_summary.vecopt_success > 0
+        else:
+            assert vlog.jit_summary.vecopt_success >= 0
 
     def test_reduce_logical_xor(self):
         def main():
@@ -166,17 +172,30 @@
         assert log.result is True
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
+        # loop.match("""
+        #     f31 = raw_load_f(i9, i29, 1, 0, descr=<ArrayF 8>)
+        #     guard_not_invalidated(descr=...)
+        #     v32 = float_ne(f31, 0.000000)
+        #     guard_true(i32, descr=...)
+        #     i36 = int_add(i24, 1)
+        #     i37 = int_add(i29, 8)
+        #     i38 = int_ge(i36, i30)
+        #     guard_false(i38, descr=...)
+        #     jump(..., descr=...)
+        #     """)
+        # vector version
         assert loop.match("""
-            f31 = raw_load_f(i9, i29, descr=<ArrayF 8>)
             guard_not_invalidated(descr=...)
-            i32 = float_ne(f31, 0.000000)
-            guard_true(i32, descr=...)
-            i36 = int_add(i24, 1)
-            i37 = int_add(i29, 8)
-            i38 = int_ge(i36, i30)
-            guard_false(i38, descr=...)
-            jump(..., descr=...)
-        """)
+            i38 = int_add(i25, 2)
+            i39 = int_ge(i38, i33)
+            guard_false(i39, descr=...)
+            v42 = vec_load_f(i9, i32, 1, 0, descr=<ArrayF 8>)
+            v43 = vec_float_ne(v42, v36)
+            f46 = vec_unpack_f(v42, 0, 1)
+            vec_guard_true(v43, descr=...)
+            i48 = int_add(i32, 16)
+            i50 = int_add(i25, 2)
+            jump(..., descr=...)""")
 
     def test_array_getitem_basic(self):
         def main():
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -34,7 +34,7 @@
 # Bootstrapping
 
 def apply_jit(translator, backend_name="auto", inline=False,
-              vec=True, enable_opts=ALL_OPTS_NAMES, **kwds):
+              vec=False, enable_opts=ALL_OPTS_NAMES, **kwds):
     if 'CPUClass' not in kwds:
         from rpython.jit.backend.detect_cpu import getcpuclass
         kwds['CPUClass'] = getcpuclass(backend_name)
@@ -84,8 +84,8 @@
                     loop_longevity=0, retrace_limit=5, function_threshold=4,
                     disable_unrolling=sys.maxint,
                     enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15,
-                    max_unroll_recursion=7, vec=1, vec_all=0, vec_cost=0,
-                    vec_length=60, vec_ratio=2, vec_guard_ratio=3, **kwds):
+                    max_unroll_recursion=7, vec=0, vec_all=0, vec_cost=0,
+                    **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -112,9 +112,6 @@
         jd.warmstate.set_param_vec(vec)
         jd.warmstate.set_param_vec_all(vec_all)
         jd.warmstate.set_param_vec_cost(vec_cost)
-        jd.warmstate.set_param_vec_length(vec_length)
-        jd.warmstate.set_param_vec_ratio(vec_ratio)
-        jd.warmstate.set_param_vec_guard_ratio(vec_guard_ratio)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -308,23 +308,14 @@
             if self.warmrunnerdesc.memory_manager:
                 self.warmrunnerdesc.memory_manager.max_unroll_recursion = value
 
-    def set_param_vec(self, value):
-        self.vec = bool(value)
+    def set_param_vec(self, ivalue):
+        self.vec = bool(ivalue)
 
-    def set_param_vec_all(self, value):
-        self.vec_all = bool(value)
+    def set_param_vec_all(self, ivalue):
+        self.vec_all = bool(ivalue)
 
-    def set_param_vec_cost(self, value):
-        self.vec_cost = bool(value)
-
-    def set_param_vec_length(self, value):
-        self.vec_length = int(value)
-
-    def set_param_vec_ratio(self, value):
-        self.vec_ratio = value / 10.0
-
-    def set_param_vec_guard_ratio(self, value):
-        self.vec_guard_ratio = value / 10.0
+    def set_param_vec_cost(self, ivalue):
+        self.vec_cost = value
 
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -575,14 +575,10 @@
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function',
     'vec': 'turn on the vectorization optimization (vecopt). ' \
-           'Supports x86 (SSE 4.1), powerpc (SVX), s390x',
-    'vec_all': 'try to vectorize trace loops that occur outside of the numpy library.',
-    'vec_cost': 'threshold for which traces to bail. 0 means the costs.',
-    'vec_length': 'the amount of instructions allowed in "all" traces.',
-    'vec_ratio': 'an integer (0-10 transfored into a float by X / 10.0) statements that have vector equivalents '
-                 'divided by the total number of trace instructions.',
-    'vec_guard_ratio': 'an integer (0-10 transfored into a float by X / 10.0) divided by the'
-                       ' total number of trace instructions.',
+           'Supports x86 (SSE 4.1), powerpc (SVX), s390x SIMD',
+    'vec_cost': 'threshold for which traces to bail. Unpacking increases the counter,'\
+                ' vector operation decrease the cost',
+    'vec_all': 'try to vectorize trace loops that occur outside of the numpypy library',
 }
 
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
@@ -598,12 +594,9 @@
               'disable_unrolling': 200,
               'enable_opts': 'all',
               'max_unroll_recursion': 7,
-              'vec': 1,
+              'vec': 0,
               'vec_all': 0,
               'vec_cost': 0,
-              'vec_length': 60,
-              'vec_ratio': 2,
-              'vec_guard_ratio': 5,
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 

From pypy.commits at gmail.com  Wed Sep 21 10:01:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 07:01:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Move the docstrings to the proper place
Message-ID: <57e292a0.11331c0a.70110.2afc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87271:0c7e8fb3f755
Date: 2016-09-21 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/0c7e8fb3f755/

Log:	Move the docstrings to the proper place

diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -40,9 +40,6 @@
     st_atime = structseqfield(11, "time of last access")
     st_mtime = structseqfield(12, "time of last modification")
     st_ctime = structseqfield(13, "time of last change")
-    st_atime_ns = structseqfield(14, "time of last access in nanoseconds")
-    st_mtime_ns = structseqfield(15, "time of last modification in nanoseconds")
-    st_ctime_ns = structseqfield(16, "time of last change in nanoseconds")
 
     if "st_blksize" in posix._statfields:
         st_blksize = structseqfield(20, "blocksize for filesystem I/O")
@@ -66,14 +63,17 @@
 
     @property
     def st_atime_ns(self):
+        "time of last access in nanoseconds"
         return int(self[7]) * 1000000000 + self.nsec_atime
 
     @property
     def st_mtime_ns(self):
+        "time of last modification in nanoseconds"
         return int(self[8]) * 1000000000 + self.nsec_mtime
 
     @property
     def st_ctime_ns(self):
+        "time of last change in nanoseconds"
         return int(self[9]) * 1000000000 + self.nsec_ctime
 
 

From pypy.commits at gmail.com  Wed Sep 21 10:46:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 21 Sep 2016 07:46:50 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57e29d5a.a699c20a.a3f49.933e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87272:b8579705f7e6
Date: 2016-09-21 16:46 +0200
http://bitbucket.org/pypy/pypy/changeset/b8579705f7e6/

Log:	merge default

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.3
+Version: 1.8.4
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.3"
-__version_info__ = (1, 8, 3)
+__version__ = "1.8.4"
+__version_info__ = (1, 8, 4)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.3"
+                               "\ncompiled with cffi version: 1.8.4"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -332,7 +332,7 @@
                         realtype = model.unknown_ptr_type(decl.name)
                     else:
                         realtype, quals = self._get_type_and_quals(
-                            decl.type, name=decl.name)
+                            decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
                 else:
                     raise api.CDefError("unrecognized construct", decl)
@@ -781,11 +781,14 @@
                 exprnode.name in self._int_constants):
             return self._int_constants[exprnode.name]
         #
-        if partial_length_ok:
-            if (isinstance(exprnode, pycparser.c_ast.ID) and
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
                 self._partial_length = True
                 return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
         #
         raise api.FFIError(":%d: unsupported expression: expected a "
                            "simple numeric constant" % exprnode.coord.line)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -587,8 +587,11 @@
     # ----------
     # typedefs
 
+    def _typedef_type(self, tp, name):
+        return self._global_type(tp, "(*(%s *)0)" % (name,))
+
     def _generate_cpy_typedef_collecttype(self, tp, name):
-        self._do_collect_type(tp)
+        self._do_collect_type(self._typedef_type(tp, name))
 
     def _generate_cpy_typedef_decl(self, tp, name):
         pass
@@ -598,6 +601,7 @@
         self._lsts["typename"].append(TypenameExpr(name, type_index))
 
     def _generate_cpy_typedef_ctx(self, tp, name):
+        tp = self._typedef_type(tp, name)
         self._typedef_ctx(tp, name)
         if getattr(tp, "origin", None) == "unknown_type":
             self._struct_ctx(tp, tp.name, approxname=None)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.3"
+VERSION = "1.8.4"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -510,3 +510,8 @@
         q = ffi.new("char[]", "abcd")
         p = ffi.cast("char(*)(void)", q)
         raises(TypeError, ffi.string, p)
+
+    def test_negative_array_size(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -8,7 +8,7 @@
 
 @unwrap_spec(cdef=str, module_name=str, source=str)
 def prepare(space, cdef, module_name, source, w_includes=None,
-            w_extra_source=None):
+            w_extra_source=None, w_min_version=None):
     try:
         import cffi
         from cffi import FFI            # <== the system one, which
@@ -16,8 +16,13 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 4, 0):
-        py.test.skip("system cffi module needs to be at least 1.4.0")
+    if w_min_version is None:
+        min_version = (1, 4, 0)
+    else:
+        min_version = tuple(space.unwrap(w_min_version))
+    if cffi.__version_info__ < min_version:
+        py.test.skip("system cffi module needs to be at least %s, got %s" % (
+            min_version, cffi.__version_info__))
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -1790,3 +1795,28 @@
                                 "void f(void) { }")
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
+
+    def test_typedef_array_dotdotdot(self):
+        ffi, lib = self.prepare("""
+            typedef int foo_t[...], bar_t[...];
+            int gv[...];
+            typedef int mat_t[...][...];
+            typedef int vmat_t[][...];
+            """,
+            "test_typedef_array_dotdotdot", """
+            typedef int foo_t[50], bar_t[50];
+            int gv[23];
+            typedef int mat_t[6][7];
+            typedef int vmat_t[][8];
+        """, min_version=(1, 8, 4))
+        assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+        assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+        assert len(ffi.new("foo_t")) == 50
+        assert len(ffi.new("bar_t")) == 50
+        assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+        assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+        assert len(ffi.new("mat_t")) == 6
+        assert len(ffi.new("mat_t")[3]) == 7
+        raises(ffi.error, ffi.sizeof, "vmat_t")
+        p = ffi.new("vmat_t", 4)
+        assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,28 +30,25 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
-    w_initializer_type = None
-    w_initializer = None
-    if len(__args__.arguments_w) > 0:
-        w_initializer = __args__.arguments_w[0]
-        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-            if w_initializer is not None:
-                if w_initializer_type is space.w_str:
-                    a.descr_fromstring(space, w_initializer)
-                elif w_initializer_type is space.w_list:
-                    a.descr_fromlist(space, w_initializer)
-                else:
-                    a.extend(w_initializer, True)
             break
     else:
         raise oefmt(space.w_ValueError,
                     "bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or "
                     "d)")
 
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
+        if w_initializer_type is space.w_str:
+            a.descr_fromstring(space, w_initializer)
+        elif w_initializer_type is space.w_list:
+            a.descr_fromlist(space, w_initializer)
+        else:
+            a.extend(w_initializer, True)
     return a
 
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -226,13 +226,13 @@
     w_keywords = space.newdict()
     stat_float_times = space.fromcache(StatState).stat_float_times
     for i, (name, TYPE) in FIELDS:
-        value = getattr(st, name)
-        if name in ('st_atime', 'st_mtime', 'st_ctime'):
-            value = int(value)   # rounded to an integer for indexed access
-        w_value = space.wrap(value)
         if i < rposix_stat.N_INDEXABLE_FIELDS:
+            # get the first 10 items by indexing; this gives us
+            # 'st_Xtime' as an integer, too
+            w_value = space.wrap(st[i])
             lst[i] = w_value
-        else:
+        elif name.startswith('st_'):    # exclude 'nsec_Xtime'
+            w_value = space.wrap(getattr(st, name))
             space.setitem(w_keywords, space.wrap(name), w_value)
 
     # non-rounded values for name-based access
@@ -243,13 +243,8 @@
                       space.wrap('st_mtime'), space.wrap(st.st_mtime))
         space.setitem(w_keywords,
                       space.wrap('st_ctime'), space.wrap(st.st_ctime))
-    else:
-        space.setitem(w_keywords,
-                      space.wrap('st_atime'), space.wrap(int(st.st_atime)))
-        space.setitem(w_keywords,
-                      space.wrap('st_mtime'), space.wrap(int(st.st_mtime)))
-        space.setitem(w_keywords,
-                      space.wrap('st_ctime'), space.wrap(int(st.st_ctime)))
+    #else:
+    #   filled by the __init__ method
 
     w_tuple = space.newtuple(lst)
     w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -129,9 +129,9 @@
         assert st[4] == st.st_uid
         assert st[5] == st.st_gid
         assert st[6] == st.st_size
-        assert st[7] == int(st.st_atime)
-        assert st[8] == int(st.st_mtime)
-        assert st[9] == int(st.st_ctime)
+        assert st[7] == int(st.st_atime)   # in complete corner cases, rounding
+        assert st[8] == int(st.st_mtime)   # here could maybe get the wrong
+        assert st[9] == int(st.st_ctime)   # integer...
 
         assert stat.S_IMODE(st.st_mode) & stat.S_IRUSR
         assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR
@@ -141,13 +141,12 @@
         assert st.st_size == 14
         assert st.st_nlink == 1
 
-        #if sys.platform.startswith('linux'):
-        #    # expects non-integer timestamps - it's unlikely that they are
-        #    # all three integers
-        #    assert ((st.st_atime, st.st_mtime, st.st_ctime) !=
-        #            (st[7],       st[8],       st[9]))
-        #    assert st.st_blksize * st.st_blocks >= st.st_size
+        assert not hasattr(st, 'nsec_atime')
+
         if sys.platform.startswith('linux'):
+            assert isinstance(st.st_atime, float)
+            assert isinstance(st.st_mtime, float)
+            assert isinstance(st.st_ctime, float)
             assert hasattr(st, 'st_rdev')
 
     def test_stat_float_times(self):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -480,3 +480,7 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_negative_array_size(self):
+        ffi = FFI()
+        py.test.raises(ValueError, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -503,3 +503,7 @@
     assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
     p = ffi.new("int[]", [-123456789])
     assert ffi.unpack(p, 1) == [-123456789]
+
+def test_negative_array_size():
+    ffi = _cffi1_backend.FFI()
+    py.test.raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1981,3 +1981,29 @@
         static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
+
+def test_typedef_array_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef int foo_t[...], bar_t[...];
+        int gv[...];
+        typedef int mat_t[...][...];
+        typedef int vmat_t[][...];
+        """)
+    lib = verify(ffi, "test_typedef_array_dotdotdot", """
+        typedef int foo_t[50], bar_t[50];
+        int gv[23];
+        typedef int mat_t[6][7];
+        typedef int vmat_t[][8];
+    """)
+    assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+    assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+    assert len(ffi.new("foo_t")) == 50
+    assert len(ffi.new("bar_t")) == 50
+    assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+    assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+    assert len(ffi.new("mat_t")) == 6
+    assert len(ffi.new("mat_t")[3]) == 7
+    py.test.raises(ffi.error, ffi.sizeof, "vmat_t")
+    p = ffi.new("vmat_t", 4)
+    assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.error import TyperError
 
 from rpython.rlib._os_support import _preferred_traits, string_traits
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask
@@ -51,15 +51,18 @@
     ("st_uid",       lltype.Signed),
     ("st_gid",       lltype.Signed),
     ("st_size",      lltype.SignedLongLong),
-    ("st_atime",     lltype.Float),
-    ("st_mtime",     lltype.Float),
-    ("st_ctime",     lltype.Float),
+    ("st_atime",     lltype.SignedLongLong),   # integral number of seconds
+    ("st_mtime",     lltype.SignedLongLong),   #
+    ("st_ctime",     lltype.SignedLongLong),   #
     ("st_blksize",   lltype.Signed),
     ("st_blocks",    lltype.Signed),
     ("st_rdev",      lltype.Signed),
     ("st_flags",     lltype.Signed),
     #("st_gen",       lltype.Signed),     -- new in CPy 2.5, not implemented
     #("st_birthtime", lltype.Float),      -- new in CPy 2.5, not implemented
+    ("nsec_atime",   lltype.Signed),   # number of nanoseconds
+    ("nsec_mtime",   lltype.Signed),   #
+    ("nsec_ctime",   lltype.Signed),   #
 ]
 N_INDEXABLE_FIELDS = 10
 
@@ -79,6 +82,37 @@
     ("f_namemax", lltype.Signed),
 ]
 
+ at specialize.arg(1)
+def get_stat_ns_as_bigint(st, name):
+    """'name' is one of the strings "atime", "mtime" or "ctime".
+    Returns a bigint that represents the number of nanoseconds
+    stored inside the RPython-level os.stat_result 'st'.
+
+    Note that when running untranslated, the os.stat_result type
+    is from Python 2.7, which doesn't store more precision than
+    a float anyway.  You will only get more after translation.
+    """
+    from rpython.rlib.rbigint import rbigint
+
+    if not we_are_translated():
+        as_float = getattr(st, "st_" + name)
+        return rbigint.fromfloat(as_float * 1e9)
+
+    if name == "atime":
+        i, j = 7, -3
+    elif name == "mtime":
+        i, j = 8, -2
+    elif name == "ctime":
+        i, j = 9, -1
+    else:
+        raise AssertionError(name)
+
+    sec = st[i]
+    nsec = st[j]
+    result = rbigint.fromrarith_int(sec).int_mul(1000000000)
+    result = result.int_add(nsec)
+    return result
+
 
 # ____________________________________________________________
 #
@@ -97,7 +131,15 @@
         if not s_attr.is_constant():
             raise annmodel.AnnotatorError("non-constant attr name in getattr()")
         attrname = s_attr.const
-        TYPE = STAT_FIELD_TYPES[attrname]
+        if attrname in ('st_atime', 'st_mtime', 'st_ctime'):
+            # like CPython, in RPython we can read the st_Xtime
+            # attribute and get a floating-point result.  We can also
+            # get a full-precision bigint with get_stat_ns_as_bigint().
+            # The floating-point result is computed like a property
+            # by _ll_get_st_Xtime().
+            TYPE = lltype.Float
+        else:
+            TYPE = STAT_FIELD_TYPES[attrname]
         return lltype_to_annotation(TYPE)
 
     def _get_rmarshall_support_(self):     # for rlib.rmarshal
@@ -105,13 +147,14 @@
         # (we ignore the extra values here for simplicity and portability)
         def stat_result_reduce(st):
             return (st[0], st[1], st[2], st[3], st[4],
-                    st[5], st[6], st[7], st[8], st[9])
+                    st[5], st[6], st[7], st[8], st[9],
+                    st[-3], st[-2], st[-1])
 
         def stat_result_recreate(tup):
-            return make_stat_result(tup + extra_zeroes)
+            return make_stat_result(tup[:10] + extra_zeroes + tup[-3:])
         s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE)
                                        for name, TYPE in PORTABLE_STAT_FIELDS])
-        extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS))
+        extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS) - 3)
         return s_reduced, stat_result_reduce, stat_result_recreate
 
 
@@ -119,7 +162,7 @@
     def getitem((s_sta, s_int)):
         assert s_int.is_constant(), "os.stat()[index]: index must be constant"
         index = s_int.const
-        assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range"
+        assert -3 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range"
         name, TYPE = STAT_FIELDS[index]
         return lltype_to_annotation(TYPE)
 
@@ -152,28 +195,61 @@
     def rtype_getattr(self, hop):
         s_attr = hop.args_s[1]
         attr = s_attr.const
+        if attr in ('st_atime', 'st_mtime', 'st_ctime'):
+            ll_func = globals()['_ll_get_' + attr]
+            v_tuple = hop.inputarg(self, arg=0)
+            return hop.gendirectcall(ll_func, v_tuple)
         try:
             index = self.stat_field_indexes[attr]
         except KeyError:
             raise TyperError("os.stat().%s: field not available" % (attr,))
         return self.redispatch_getfield(hop, index)
 
+ at specialize.memo()
+def _stfld(name):
+    index = STAT_FIELD_NAMES.index(name)
+    return 'item%d' % index
+
+def _ll_get_st_atime(tup):
+    return (float(getattr(tup, _stfld("st_atime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_atime")))
+
+def _ll_get_st_mtime(tup):
+    return (float(getattr(tup, _stfld("st_mtime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_mtime")))
+
+def _ll_get_st_ctime(tup):
+    return (float(getattr(tup, _stfld("st_ctime"))) +
+            1E-9 * getattr(tup, _stfld("nsec_ctime")))
+
 
 class __extend__(pairtype(StatResultRepr, IntegerRepr)):
     def rtype_getitem((r_sta, r_int), hop):
         s_int = hop.args_s[1]
         index = s_int.const
+        if index < 0:
+            index += len(STAT_FIELDS)
         return r_sta.redispatch_getfield(hop, index)
 
 s_StatResult = SomeStatResult()
 
+
 def make_stat_result(tup):
-    """Turn a tuple into an os.stat_result object."""
-    positional = tuple(
-        lltype.cast_primitive(TYPE, value) for value, (name, TYPE) in
-            zip(tup, STAT_FIELDS)[:N_INDEXABLE_FIELDS])
+    """NOT_RPYTHON: Turn a tuple into an os.stat_result object."""
+    assert len(tup) == len(STAT_FIELDS)
+    assert float not in [type(x) for x in tup]
+    positional = []
+    for i in range(N_INDEXABLE_FIELDS):
+        name, TYPE = STAT_FIELDS[i]
+        value = lltype.cast_primitive(TYPE, tup[i])
+        positional.append(value)
     kwds = {}
+    kwds['st_atime'] = tup[7] + 1e-9 * tup[-3]
+    kwds['st_mtime'] = tup[8] + 1e-9 * tup[-2]
+    kwds['st_ctime'] = tup[9] + 1e-9 * tup[-1]
     for value, (name, TYPE) in zip(tup, STAT_FIELDS)[N_INDEXABLE_FIELDS:]:
+        if name.startswith('nsec_'):
+            continue   # ignore the nsec_Xtime here
         kwds[name] = lltype.cast_primitive(TYPE, value)
     return os.stat_result(positional, kwds)
 
@@ -360,6 +436,8 @@
         posix_declaration(ALL_STAT_FIELDS[_i])
     del _i
 
+STAT_FIELDS += ALL_STAT_FIELDS[-3:]   # nsec_Xtime
+
 # these two global vars only list the fields defined in the underlying platform
 STAT_FIELD_TYPES = dict(STAT_FIELDS)      # {'st_xxx': TYPE}
 STAT_FIELD_NAMES = [_name for (_name, _TYPE) in STAT_FIELDS]
@@ -368,16 +446,20 @@
 STATVFS_FIELD_TYPES = dict(STATVFS_FIELDS)
 STATVFS_FIELD_NAMES = [name for name, tp in STATVFS_FIELDS]
 
+
 def build_stat_result(st):
     # only for LL backends
     if TIMESPEC is not None:
-        atim = st.c_st_atim; atime = int(atim.c_tv_sec) + 1E-9 * int(atim.c_tv_nsec)
-        mtim = st.c_st_mtim; mtime = int(mtim.c_tv_sec) + 1E-9 * int(mtim.c_tv_nsec)
-        ctim = st.c_st_ctim; ctime = int(ctim.c_tv_sec) + 1E-9 * int(ctim.c_tv_nsec)
+        atim = st.c_st_atim
+        mtim = st.c_st_mtim
+        ctim = st.c_st_ctim
+        atime, extra_atime = atim.c_tv_sec, int(atim.c_tv_nsec)
+        mtime, extra_mtime = mtim.c_tv_sec, int(mtim.c_tv_nsec)
+        ctime, extra_ctime = ctim.c_tv_sec, int(ctim.c_tv_nsec)
     else:
-        atime = st.c_st_atime
-        mtime = st.c_st_mtime
-        ctime = st.c_st_ctime
+        atime, extra_atime = st.c_st_atime, 0
+        mtime, extra_mtime = st.c_st_mtime, 0
+        ctime, extra_ctime = st.c_st_ctime, 0
 
     result = (st.c_st_mode,
               st.c_st_ino,
@@ -395,6 +477,10 @@
     if "st_rdev"    in STAT_FIELD_TYPES: result += (st.c_st_rdev,)
     if "st_flags"   in STAT_FIELD_TYPES: result += (st.c_st_flags,)
 
+    result += (extra_atime,
+               extra_mtime,
+               extra_ctime)
+
     return make_stat_result(result)
 
 
@@ -455,12 +541,14 @@
             # console or LPT device
             return make_stat_result((win32traits._S_IFCHR,
                                      0, 0, 0, 0, 0,
-                                     0, 0, 0, 0))
+                                     0, 0, 0, 0,
+                                     0, 0, 0))
         elif filetype == win32traits.FILE_TYPE_PIPE:
             # socket or named pipe
             return make_stat_result((win32traits._S_IFIFO,
                                      0, 0, 0, 0, 0,
-                                     0, 0, 0, 0))
+                                     0, 0, 0, 0,
+                                     0, 0, 0))
         elif filetype == win32traits.FILE_TYPE_UNKNOWN:
             error = rwin32.GetLastError_saved()
             if error != 0:
@@ -539,14 +627,11 @@
 #__________________________________________________
 # Helper functions for win32
 if _WIN32:
-    from rpython.rlib.rwin32file import FILE_TIME_to_time_t_float
+    from rpython.rlib.rwin32file import FILE_TIME_to_time_t_nsec
 
     def make_longlong(high, low):
         return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
 
-    # Seconds between 1.1.1601 and 1.1.1970
-    secs_between_epochs = rffi.r_longlong(11644473600)
-
     @specialize.arg(0)
     def win32_xstat(traits, path, traverse=False):
         win32traits = make_win32_traits(traits)
@@ -582,14 +667,15 @@
     def win32_attribute_data_to_stat(win32traits, info):
         st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes)
         st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow)
-        ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime)
-        mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime)
-        atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime)
+        ctime, extra_ctime = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime)
+        mtime, extra_mtime = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime)
+        atime, extra_atime = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime)
 
         result = (st_mode,
                   0, 0, 0, 0, 0,
                   st_size,
-                  atime, mtime, ctime)
+                  atime, mtime, ctime,
+                  extra_atime, extra_mtime, extra_ctime)
 
         return make_stat_result(result)
 
@@ -597,9 +683,9 @@
         # similar to the one above
         st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes)
         st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow)
-        ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime)
-        mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime)
-        atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime)
+        ctime, extra_ctime = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime)
+        mtime, extra_mtime = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime)
+        atime, extra_atime = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime)
 
         # specific to fstat()
         st_ino = make_longlong(info.c_nFileIndexHigh, info.c_nFileIndexLow)
@@ -608,7 +694,8 @@
         result = (st_mode,
                   st_ino, 0, st_nlink, 0, 0,
                   st_size,
-                  atime, mtime, ctime)
+                  atime, mtime, ctime,
+                  extra_atime, extra_mtime, extra_ctime)
 
         return make_stat_result(result)
 
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -6,6 +6,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import intmask
 
 @specialize.memo()
 def make_win32_traits(traits):
@@ -213,13 +214,25 @@
     return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
 
 # Seconds between 1.1.1601 and 1.1.1970
-secs_between_epochs = rffi.r_longlong(11644473600)
+secs_between_epochs = 11644473600.0
+hns_between_epochs = rffi.r_longlong(116444736000000000)  # units of 100 nsec
 
 def FILE_TIME_to_time_t_float(filetime):
     ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
     # FILETIME is in units of 100 nsec
     return float(ft) * (1.0 / 10000000.0) - secs_between_epochs
 
+def FILE_TIME_to_time_t_nsec(filetime):
+    """Like the previous function, but returns a pair: (integer part
+    'time_t' as a r_longlong, fractional part as an int measured in
+    nanoseconds).
+    """
+    ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
+    ft -= hns_between_epochs
+    int_part = ft / 10000000
+    frac_part = ft - (int_part * 10000000)
+    return (int_part, intmask(frac_part) * 100)
+
 def time_t_to_FILE_TIME(time, filetime):
     ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
diff --git a/rpython/rlib/test/test_rposix_stat.py b/rpython/rlib/test/test_rposix_stat.py
--- a/rpython/rlib/test/test_rposix_stat.py
+++ b/rpython/rlib/test/test_rposix_stat.py
@@ -2,12 +2,20 @@
 import py
 from rpython.rlib import rposix_stat
 from rpython.tool.udir import udir
+from rpython.translator.c.test.test_genc import compile
+from rpython.rtyper.lltypesystem import lltype
+
 
 class TestPosixStatFunctions:
     @py.test.mark.skipif("sys.platform == 'win32'",
                          reason="win32 only has the portable fields")
     def test_has_all_fields(self):
-        assert rposix_stat.STAT_FIELDS == rposix_stat.ALL_STAT_FIELDS[:13]
+        # XXX this test is obscure!  it will fail if the exact set of
+        # XXX stat fields found differs from the one we expect on Linux.
+        # XXX Why?
+        assert rposix_stat.STAT_FIELDS == (
+            rposix_stat.ALL_STAT_FIELDS[:13] +
+            rposix_stat.ALL_STAT_FIELDS[-3:])
 
     def test_stat(self):
         def check(f):
@@ -66,3 +74,27 @@
     finally:
         os.close(dirfd)
     assert result.st_atime == tmpdir.join('file').atime()
+
+def test_high_precision_stat_time():
+    def f():
+        st = os.stat('.')
+        # should be supported on all platforms, but give a result whose
+        # precision might be lower than full nanosecond
+        highprec = rposix_stat.get_stat_ns_as_bigint(st, "ctime")
+        return '%s;%s' % (st.st_ctime, highprec.str())
+    fc = compile(f, [])
+    as_string = fc()
+    asfloat, highprec = as_string.split(';')
+    asfloat = float(asfloat)
+    highprec = int(highprec)
+    st = os.stat('.')
+    assert abs(asfloat - st.st_ctime) < 500e-9
+    assert abs(highprec - int(st.st_ctime * 1e9)) < 500
+    assert abs(rposix_stat.get_stat_ns_as_bigint(st, "ctime").tolong()
+               - st.st_ctime * 1e9) < 3
+    if rposix_stat.TIMESPEC is not None:
+        with lltype.scoped_alloc(rposix_stat.STAT_STRUCT.TO) as stresult:
+            rposix_stat.c_stat(".", stresult)
+            assert 0 <= stresult.c_st_ctim.c_tv_nsec <= 999999999
+            assert highprec == (int(stresult.c_st_ctim.c_tv_sec) * 1000000000
+                                + int(stresult.c_st_ctim.c_tv_nsec))

From pypy.commits at gmail.com  Wed Sep 21 10:58:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 21 Sep 2016 07:58:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Renaming a built-in module does not
 change the name of the corresponding usemodules.* option
Message-ID: <57e2a025.0d8d1c0a.4f0be.4bfe@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r87273:ffe795342bf8
Date: 2016-09-21 15:57 +0100
http://bitbucket.org/pypy/pypy/changeset/ffe795342bf8/

Log:	Renaming a built-in module does not change the name of the
	corresponding usemodules.* option

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -24,7 +24,7 @@
 default_modules = essential_modules.copy()
 default_modules.update([
     "_codecs", "atexit", "gc", "_weakref", "marshal", "errno", "imp",
-    "itertools", "math", "cmath", "_sre", "_pickle_support", "_operator",
+    "itertools", "math", "cmath", "_sre", "_pickle_support", "operator",
     "parser", "symbol", "token", "_ast", "_random", "__pypy__",
     "_string", "_testing", "time"
 ])
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -26,6 +26,7 @@
     exceptions='builtins',
     struct='_struct',
     thread='_thread',
+    operator='_operator',
     )
 
 class AppError(Exception):

From pypy.commits at gmail.com  Wed Sep 21 11:05:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 08:05:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: Future py3k compat
Message-ID: <57e2a1d0.a468c20a.69aee.9d35@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87274:94f2a130efb0
Date: 2016-09-21 17:05 +0200
http://bitbucket.org/pypy/pypy/changeset/94f2a130efb0/

Log:	Future py3k compat

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -507,7 +507,7 @@
     def test_bug_1(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
-        q = ffi.new("char[]", "abcd")
+        q = ffi.new("char[]", b"abcd")
         p = ffi.cast("char(*)(void)", q)
         raises(TypeError, ffi.string, p)
 

From pypy.commits at gmail.com  Wed Sep 21 11:10:00 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 21 Sep 2016 08:10:00 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: remove unsupported operation in
 the stress tests (call_cond_value)
Message-ID: <57e2a2c8.a970c20a.8e88b.964a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87275:f180564486f9
Date: 2016-09-21 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/f180564486f9/

Log:	remove unsupported operation in the stress tests (call_cond_value)

diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.test.test_random import check_random_function, Random
 from rpython.jit.backend.test.test_ll_random import LLtypeOperationBuilder
 from rpython.jit.backend.detect_cpu import getcpuclass
+from rpython.jit.metainterp.resoperation import rop
 import platform
 
 CPU = getcpuclass()
@@ -18,6 +19,14 @@
     cpu.setup_once()
     r = Random()
     r.jumpahead(piece*99999999)
+    OPERATIONS = LLtypeOperationBuilder.OPERATIONS[:]
+    if not cpu.supports_cond_call_value:
+        # remove COND_CALL_VALUE_I if the cpu does not support it
+        ops = LLtypeOperationBuilder.OPERATIONS
+        LLtypeOperationBuilder.OPERATIONS = [op for op in ops \
+                if op.opnum != rop.COND_CALL_VALUE_I]
     for i in range(piece*per_piece, (piece+1)*per_piece):
         print "        i = %d; r.setstate(%s)" % (i, r.getstate())
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)
+    # restore the old list
+    LLtypeOperationBuilder.OPERATIONS = OPERATIONS

From pypy.commits at gmail.com  Wed Sep 21 11:10:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 08:10:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57e2a2f6.c4371c0a.11dfd.c53e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87276:ec5c171f3870
Date: 2016-09-21 17:10 +0200
http://bitbucket.org/pypy/pypy/changeset/ec5c171f3870/

Log:	hg merge default

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -507,7 +507,7 @@
     def test_bug_1(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
-        q = ffi.new("char[]", "abcd")
+        q = ffi.new("char[]", b"abcd")
         p = ffi.cast("char(*)(void)", q)
         raises(TypeError, ffi.string, p)
 

From pypy.commits at gmail.com  Wed Sep 21 11:14:33 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 21 Sep 2016 08:14:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix translation
Message-ID: <57e2a3d9.e50fc20a.7e186.a20e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r87277:54ced20c6ee9
Date: 2016-09-21 16:13 +0100
http://bitbucket.org/pypy/pypy/changeset/54ced20c6ee9/

Log:	Fix translation

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -102,7 +102,7 @@
             else:
                 w_stop = w_length
     return w_start, w_stop, w_step
-    
+
 min_jitdriver = jit.JitDriver(name='min',
         greens=['has_key', 'has_item', 'w_type'], reds='auto')
 max_jitdriver = jit.JitDriver(name='max',
@@ -136,7 +136,7 @@
                         "%s() expects at least one argument",
                         implementation_of)
         w_key = None
-        has_default = False
+        w_default = None
         if any_kwds:
             kwds = args.keywords
             for n in range(len(kwds)):
@@ -144,13 +144,12 @@
                     w_key = args.keywords_w[n]
                 elif kwds[n] == "default":
                     w_default = args.keywords_w[n]
-                    has_default = True
                 else:
                     raise oefmt(space.w_TypeError,
                                 "%s() got unexpected keyword argument",
                                 implementation_of)
 
-        if has_default and len(args_w) > 1:
+        if w_default is not None and len(args_w) > 1:
             raise oefmt(space.w_TypeError,
                 "Cannot specify a default for %s() with multiple positional arguments",
                 implementation_of)
@@ -180,7 +179,7 @@
                 w_max_item = w_item
                 w_max_val = w_compare_with
         if w_max_item is None:
-            if has_default:
+            if w_default is not None:
                 w_max_item = w_default
             else:
                 raise oefmt(space.w_ValueError, "arg is an empty sequence")
@@ -420,7 +419,7 @@
     def descr_repr(self, space):
         if not space.is_true(space.eq(self.w_step, space.newint(1))):
             return space.mod(space.wrap("range(%d, %d, %d)"),
-                             space.newtuple([self.w_start, self.w_stop, 
+                             space.newtuple([self.w_start, self.w_stop,
                                              self.w_step]))
         else:
             return space.mod(space.wrap("range(%d, %d)"),
@@ -433,7 +432,7 @@
         "Get a range item, when known to be inside bounds"
         # return self.start + (i * self.step)
         return space.add(self.w_start, space.mul(w_index, self.w_step))
-        
+
     def _compute_item(self, space, w_index):
         w_zero = space.newint(0)
         w_index = space.index(w_index)
@@ -767,7 +766,7 @@
         __iter__ = interp2app(W_Map.iter_w),
         __next__ = interp2app(W_Map.next_w),
         __reduce__ = interp2app(W_Map.descr_reduce),
-        __doc__ = """\ 
+        __doc__ = """\
 Make an iterator that computes the function using arguments from
 each of the iterables.  Stops when the shortest iterable is exhausted.""")
 

From pypy.commits at gmail.com  Wed Sep 21 11:15:43 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 21 Sep 2016 08:15:43 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: translation fix
Message-ID: <57e2a41f.94071c0a.3a61.51c9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87278:beeb5219ac63
Date: 2016-09-21 17:15 +0200
http://bitbucket.org/pypy/pypy/changeset/beeb5219ac63/

Log:	translation fix

diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -315,7 +315,7 @@
         self.vec_all = bool(ivalue)
 
     def set_param_vec_cost(self, ivalue):
-        self.vec_cost = value
+        self.vec_cost = ivalue
 
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)

From pypy.commits at gmail.com  Wed Sep 21 11:23:04 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 21 Sep 2016 08:23:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove unsupported operation in the
 stress tests (call_cond_value)
Message-ID: <57e2a5d8.02c11c0a.22cc0.f0ec@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r87279:fb75857b6dd4
Date: 2016-09-21 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/fb75857b6dd4/

Log:	remove unsupported operation in the stress tests (call_cond_value)

diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.test.test_random import check_random_function, Random
 from rpython.jit.backend.test.test_ll_random import LLtypeOperationBuilder
 from rpython.jit.backend.detect_cpu import getcpuclass
+from rpython.jit.metainterp.resoperation import rop
 import platform
 
 CPU = getcpuclass()
@@ -18,6 +19,14 @@
     cpu.setup_once()
     r = Random()
     r.jumpahead(piece*99999999)
+    OPERATIONS = LLtypeOperationBuilder.OPERATIONS[:]
+    if not cpu.supports_cond_call_value:
+        # remove COND_CALL_VALUE_I if the cpu does not support it
+        ops = LLtypeOperationBuilder.OPERATIONS
+        LLtypeOperationBuilder.OPERATIONS = [op for op in ops \
+                if op.opnum != rop.COND_CALL_VALUE_I]
     for i in range(piece*per_piece, (piece+1)*per_piece):
         print "        i = %d; r.setstate(%s)" % (i, r.getstate())
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)
+    # restore the old list
+    LLtypeOperationBuilder.OPERATIONS = OPERATIONS

From pypy.commits at gmail.com  Wed Sep 21 11:35:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 21 Sep 2016 08:35:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove dunder aliases from _operator,
 as on CPython
Message-ID: <57e2a8d8.4756c20a.b4de8.a6d6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r87280:7ed96cd63af0
Date: 2016-09-21 16:34 +0100
http://bitbucket.org/pypy/pypy/changeset/7ed96cd63af0/

Log:	Remove dunder aliases from _operator, as on CPython

diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -4,21 +4,8 @@
     """Operator Builtin Module. """
     applevel_name = '_operator'
 
-    # HACK! override loaders to be able to access different operations
-    # under same name. I.e., operator.eq == operator.__eq__
-
-    def __init__(self, space, w_name):
-        def create_lambda(name, alsoname):
-            return lambda space : self.getdictvalue(space, alsoname)
-
-        MixedModule.__init__(self, space, w_name)
-        for name, alsoname in self.mapping.iteritems():
-            self.loaders[name] = create_lambda(name, alsoname)
-
     appleveldefs = {}
-
     app_names = ['countOf', 'attrgetter', 'itemgetter', 'methodcaller']
-
     for name in app_names:
         appleveldefs[name] = 'app_operator.%s' % name
 
@@ -41,53 +28,3 @@
 
     for name in interp_names:
         interpleveldefs[name] = 'interp_operator.%s' % name
-
-    mapping = {
-        '__abs__' : 'abs',
-        '__add__' : 'add',
-        '__and__' : 'and_',
-        '__concat__' : 'concat',
-        '__contains__' : 'contains',
-        '__index__' : 'index',
-        '__delitem__' : 'delitem',
-        '__eq__' : 'eq',
-        '__floordiv__' : 'floordiv',
-        '__ge__' : 'ge',
-        '__getitem__' : 'getitem',
-        '__gt__' : 'gt',
-        '__inv__' : 'inv',
-        '__invert__' : 'invert',
-        '__le__' : 'le',
-        '__lshift__' : 'lshift',
-        '__lt__' : 'lt',
-        '__mod__' : 'mod',
-        '__mul__' : 'mul',
-        '__ne__' : 'ne',
-        '__neg__' : 'neg',
-        '__not__' : 'not_',
-        '__or__' : 'or_',
-        '__pos__' : 'pos',
-        '__pow__' : 'pow',
-        '__rshift__' : 'rshift',
-        '__setitem__' : 'setitem',
-        '__sub__' : 'sub',
-        '__truediv__' : 'truediv',
-        '__xor__' : 'xor',
-        '__matmul__' : 'matmul',
-        # in-place
-        '__iadd__' : 'iadd',
-        '__iand__' : 'iand',
-        '__iconcat__' : 'iconcat',
-        '__ifloordiv__' : 'ifloordiv',
-        '__ilshift__' : 'ilshift',
-        '__imod__' : 'imod',
-        '__imul__' : 'imul',
-        '__ior__' : 'ior',
-        '__ipow__' : 'ipow',
-        '__irshift__' : 'irshift',
-        '__isub__' : 'isub',
-        '__itruediv__' : 'itruediv',
-        '__ixor__' : 'ixor',
-        '__imatmul__' : 'imatmul',
-    }
-
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -3,10 +3,6 @@
 class AppTestOperator:
     spaceconfig = dict(usemodules=['operator'])
 
-    def test_equality(self):
-        import _operator as operator
-        assert operator.eq == operator.__eq__
-
     def test_getters_are_not_regular_functions(self):
         import _operator as operator
         class A(object):
@@ -191,7 +187,6 @@
     def test_index(self):
         import _operator as operator
         assert operator.index(42) == 42
-        assert operator.__index__(42) == 42
         raises(TypeError, operator.index, "abc")
         exc = raises(TypeError, operator.index, "abc")
         assert str(exc.value) == "'str' object cannot be interpreted as an integer"

From pypy.commits at gmail.com  Wed Sep 21 11:55:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 08:55:01 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Remove DONE entries.  Add #2310.
Message-ID: <57e2ad55.8759c20a.1522a.ac0d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5708:1c567dfcb89a
Date: 2016-09-21 17:54 +0200
http://bitbucket.org/pypy/extradoc/changeset/1c567dfcb89a/

Log:	Remove DONE entries. Add #2310.

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -14,11 +14,6 @@
 * richard: extended slicing for memory view
 * richard: bytes % args, bytearray % args: PEP 461
 
-* arigo: look at test failures relaced to os.scandir() or the pathlib
-  module, or the enum module
-
-* arigo: (py3k issue #2278) add full precision for st_mtime_ns
-
 
 
 Misc stuff not formally in any milestone
@@ -31,19 +26,8 @@
   ``dict``.  The main pain point is ``move_to_end(last=False)``.  See
   https://mail.python.org/pipermail/python-dev/2016-August/145837.html
 
-* interpreter/generator.py: move the common functionality from
-  GeneratorIterator and Coroutine to the base class.  Review all
-  calls to _PyGen_yf() in genobject.c.  This is needed before
-  adding gi_yieldfrom/cr_await to generator/coroutines.  (Waiting
-  because some work might be going on with raffael_t.)
-
 * compare ``dir(posix)`` on py3.5 and cpython 3.5.
 
-* review all unwrap_spec() that are meant to pass booleans (now
-  with '=int').  Argument clinic turns these to PyObject_IsTrue(), i.e.
-  accepting any object whatsoever(?!), which is supposedly a feature
-  (see http://bugs.python.org/issue14705).  DONE
-
 * ``KeyError('pip.exceptions',) in weakref callback <function
   _get_module_lock.<locals>.cb at 0x00007f118e2c0020> ignored``
   we're getting them now on start-up, investigate
@@ -51,8 +35,7 @@
 * ``print 5`` should give
   ``SyntaxError: Missing parentheses in call to 'print'``
 
-* ``_utf8`` in W_UnicodeObject used to be quasi-immutable,
-  document why it doesn't work and do a proper fix.  DONE
+* Windows: issue 2310: kill WindowsError
 
 
 Milestone 1 (Aug-Sep-Oct 2016)

From pypy.commits at gmail.com  Wed Sep 21 12:00:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 09:00:20 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: updates
Message-ID: <57e2ae94.ca0e1c0a.a952.5aad@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5709:5fb52c896f0b
Date: 2016-09-21 18:00 +0200
http://bitbucket.org/pypy/extradoc/changeset/5fb52c896f0b/

Log:	updates

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -37,6 +37,8 @@
 
 * Windows: issue 2310: kill WindowsError
 
+* bytearray: 'del x[:10]' is now amortized constant-time
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------
@@ -47,7 +49,7 @@
 planned:
 
 * PEP 492, coroutines with async and await syntax.  (The complete PEP
-  is included.)
+  is included.)  DONE
 
 * PEP 465, a new matrix multiplication operator: a @ b.
 

From pypy.commits at gmail.com  Wed Sep 21 12:17:51 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 21 Sep 2016 09:17:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Use built-in module _operator to define
 built-in functions bin(), oct(), hex().
Message-ID: <57e2b2af.a970c20a.8e88b.b07b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r87281:a014e942245e
Date: 2016-09-21 17:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a014e942245e/

Log:	Use built-in module _operator to define built-in functions bin(),
	oct(), hex().

	This should prevent potential bootstrapping issues caused by trying
	to import pure-Python stdlib modules too early. Also, mark _operator
	as an essential module.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -16,7 +16,7 @@
 
 essential_modules = set([
     "exceptions", "_io", "sys", "builtins", "posix", "_warnings",
-    "itertools", "_frozen_importlib",
+    "itertools", "_frozen_importlib", "operator",
 ])
 if sys.platform == "win32":
     essential_modules.add("_winreg")
@@ -24,7 +24,7 @@
 default_modules = essential_modules.copy()
 default_modules.update([
     "_codecs", "atexit", "gc", "_weakref", "marshal", "errno", "imp",
-    "itertools", "math", "cmath", "_sre", "_pickle_support", "operator",
+    "itertools", "math", "cmath", "_sre", "_pickle_support",
     "parser", "symbol", "token", "_ast", "_random", "__pypy__",
     "_string", "_testing", "time"
 ])
diff --git a/pypy/module/__builtin__/app_operation.py b/pypy/module/__builtin__/app_operation.py
--- a/pypy/module/__builtin__/app_operation.py
+++ b/pypy/module/__builtin__/app_operation.py
@@ -1,16 +1,16 @@
-import operator
+import _operator
 
 def bin(x):
     """Return the binary representation of an integer."""
-    value = operator.index(x)
+    value = _operator.index(x)
     return value.__format__("#b")
 
 def oct(x):
     """Return the octal representation of an integer."""
-    x = operator.index(x)
+    x = _operator.index(x)
     return x.__format__("#o")
 
 def hex(x):
     """Return the hexadecimal representation of an integer."""
-    x = operator.index(x)
+    x = _operator.index(x)
     return x.__format__("#x")
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 
 class AppTestOperator:
-    spaceconfig = dict(usemodules=['operator'])
-
     def test_getters_are_not_regular_functions(self):
         import _operator as operator
         class A(object):

From pypy.commits at gmail.com  Wed Sep 21 13:21:20 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 21 Sep 2016 10:21:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: removed my hack,
 use filter instead (thx armin)
Message-ID: <57e2c190.669fc20a.5756b.d230@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r87282:9310b8a0e6fa
Date: 2016-09-21 19:20 +0200
http://bitbucket.org/pypy/pypy/changeset/9310b8a0e6fa/

Log:	removed my hack, use filter instead (thx armin)

diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -710,6 +710,12 @@
 
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
+
+    def filter(self, builder):
+        if not builder.cpu.supports_cond_call_value and \
+           self.opnum == rop.COND_CALL_VALUE_I:
+            raise CannotProduceOperation
+
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -19,14 +19,6 @@
     cpu.setup_once()
     r = Random()
     r.jumpahead(piece*99999999)
-    OPERATIONS = LLtypeOperationBuilder.OPERATIONS[:]
-    if not cpu.supports_cond_call_value:
-        # remove COND_CALL_VALUE_I if the cpu does not support it
-        ops = LLtypeOperationBuilder.OPERATIONS
-        LLtypeOperationBuilder.OPERATIONS = [op for op in ops \
-                if op.opnum != rop.COND_CALL_VALUE_I]
     for i in range(piece*per_piece, (piece+1)*per_piece):
         print "        i = %d; r.setstate(%s)" % (i, r.getstate())
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)
-    # restore the old list
-    LLtypeOperationBuilder.OPERATIONS = OPERATIONS

From pypy.commits at gmail.com  Wed Sep 21 13:24:02 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 21 Sep 2016 10:24:02 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Convert _immutable_ to
 _value_class_
Message-ID: <57e2c232.04141c0a.25833.747b@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87284:2cb63cfbee57
Date: 2016-09-21 13:01 -0400
http://bitbucket.org/pypy/pypy/changeset/2cb63cfbee57/

Log:	Convert _immutable_ to _value_class_

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -62,10 +62,6 @@
     """Raised when the _immutable_ or _immutable_fields_ hints are
     not consistent across a class hierarchy."""
 
-class ValueClassConflictError(Exception):
-    """Raise when the _value_class_ hints are not consistent across
-    the class heirarchy"""
-
 def getclassrepr(rtyper, classdef):
     if classdef is None:
         return rtyper.rootclass_repr
@@ -163,7 +159,7 @@
 OBJECT_VTABLE = lltype.ForwardReference()
 CLASSTYPE = Ptr(OBJECT_VTABLE)
 OBJECT = GcStruct('object', ('typeptr', CLASSTYPE),
-                  hints={'immutable': True, 'shouldntbenull': True,
+                  hints={'value_class': True, 'shouldntbenull': True,
                          'typeptr': True},
                   rtti=True)
 OBJECTPTR = Ptr(OBJECT)
@@ -175,7 +171,7 @@
                             ('name', Ptr(rstr.STR)),
                             ('hash', Signed),
                             ('instantiate', Ptr(FuncType([], OBJECTPTR))),
-                            hints={'immutable': True}))
+                            hints={'value_class': True}))
 # non-gc case
 NONGCOBJECT = Struct('nongcobject', ('typeptr', CLASSTYPE))
 NONGCOBJECTPTR = Ptr(NONGCOBJECT)
@@ -276,7 +272,7 @@
         #
         self.rbase = getclassrepr(self.rtyper, self.classdef.basedef)
         self.rbase.setup()
-        kwds = {'hints': {'immutable': True}}
+        kwds = {'hints': {'value_class': True}}
         vtable_type = Struct('%s_vtable' % self.classdef.name,
                                 ('super', self.rbase.vtable_type),
                                 *llfields, **kwds)
@@ -520,7 +516,6 @@
             if hints is None:
                 hints = {}
             hints = self._check_for_immutable_hints(hints)
-            hints = self._check_for_value_class_hints(hints)
             kwds = {}
             if self.gcflavor == 'gc':
                 kwds['rtti'] = True
@@ -544,27 +539,15 @@
     def _check_for_immutable_hints(self, hints):
         hints = hints.copy()
         classdesc = self.classdef.classdesc
-        immut = classdesc.get_param('_immutable_', inherit=False)
         value_class = classdesc.get_param('_value_class_', inherit=False)
 
-        if immut is None:
-            immut = value_class
-        elif value_class is not None and value_class and not immut:
-            raise ImmutableConflictError(
-                "class %r: _immutable_ != True and _value_class_ = True")
-
-        if immut is None:
-            if classdesc.get_param('_immutable_', inherit=True):
-                raise ImmutableConflictError(
-                    "class %r inherits from its parent _immutable_=True, "
-                    "so it should also declare _immutable_=True" % (
+        if value_class is not None:
+            if value_class is not True:
+                raise TyperError(
+                    "class %r: _value_class_ = something else than True" % (
                         self.classdef,))
-        elif immut is not True:
-            raise TyperError(
-                "class %r: _immutable_ = something else than True" % (
-                    self.classdef,))
-        else:
-            hints['immutable'] = True
+            else:
+                hints['value_class'] = True
         self.immutable_field_set = classdesc.immutable_fields
         if (classdesc.immutable_fields or
                 'immutable_fields' in self.rbase.object_type._hints):
@@ -572,30 +555,6 @@
             hints['immutable_fields'] = accessor
         return hints
 
-    def _check_for_value_class_hints(self, hints):
-        """Look for value class hints in the class heirarchy to extract the proper
-        hints and ensure consistency of the _value_class_ annotation. This is
-        mostly equivalent to _check_for_immutable_hints except that
-        _value_class_=True imples _immutable_=True."""
-        hints = hints.copy()
-        classdesc = self.classdef.classdesc
-        value_class = classdesc.get_param('_value_class_', inherit=False)
-        if value_class is None:
-            if classdesc.get_param('_value_class_', inherit=True):
-                raise ValueClassConflictError(
-                    "class %r inherits from its parent _value_class_=True, "
-                    "so it should also declare _value_class_=True" % (
-                        self.classdef,))
-        elif value_class is not True:
-            raise TyperError(
-                "class %r: _value_class_ = something else than True" % (
-                    self.classdef,))
-        else:
-            # _value_class_ = True implies _immutable_ = True
-            hints['value_class'] = True
-            hints['immutable'] = True
-        return hints
-
     def __repr__(self):
         if self.classdef is None:
             clsname = 'object'
@@ -678,7 +637,7 @@
                 mangled_name, r = self._get_field(name)
             except KeyError:
                 continue
-            if quasi and hints.get("immutable"):
+            if quasi and hints.get("value_class"):
                 raise TyperError(
                     "can't have _immutable_ = True and a quasi-immutable field "
                     "%s in class %s" % (name, self.classdef))
@@ -689,7 +648,7 @@
     def _check_for_immutable_conflicts(self):
         # check for conflicts, i.e. a field that is defined normally as
         # mutable in some parent class but that is now declared immutable
-        is_self_immutable = "immutable" in self.object_type._hints
+        is_value_class = "value_class" in self.object_type._hints
         base = self
         while base.classdef is not None:
             base = base.rbase
@@ -704,9 +663,9 @@
                 if base.object_type._immutable_field(mangled):
                     continue
                 # 'fieldname' is a mutable, non-Void field in the parent
-                if is_self_immutable:
+                if is_value_class:
                     raise ImmutableConflictError(
-                        "class %r has _immutable_=True, but parent class %r "
+                        "class %r has _value_class_=True, but parent class %r "
                         "defines (at least) the mutable field %r" %
                         (self, base, fieldname))
                 if (fieldname in self.immutable_field_set or
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -868,12 +868,13 @@
             return B()
         t, typer, graph = self.gengraph(f, [])
         B_TYPE = deref(graph.getreturnvar().concretetype)
-        assert B_TYPE._hints["immutable"]
+        assert B_TYPE._hints["value_class"]
         A_TYPE = B_TYPE.super
         accessor = A_TYPE._hints["immutable_fields"]
         assert accessor.fields == {"inst_v": IR_IMMUTABLE}
 
     def test_immutable_subclass_1(self):
+        py.test.skip("should not fail anymore")
         from rpython.rtyper.rclass import ImmutableConflictError
         from rpython.jit.metainterp.typesystem import deref
         class A(object):
@@ -898,7 +899,7 @@
             return B()
         t, typer, graph = self.gengraph(f, [])
         B_TYPE = deref(graph.getreturnvar().concretetype)
-        assert B_TYPE._hints["immutable"]
+        assert B_TYPE._hints["value_class"]
 
     def test_immutable_subclass_void(self):
         from rpython.jit.metainterp.typesystem import deref
@@ -914,7 +915,7 @@
             return B()
         t, typer, graph = self.gengraph(f, [])
         B_TYPE = deref(graph.getreturnvar().concretetype)
-        assert B_TYPE._hints["immutable"]
+        assert B_TYPE._hints["value_class"]
 
     def test_quasi_immutable(self):
         from rpython.jit.metainterp.typesystem import deref
@@ -1306,70 +1307,3 @@
             return a.next.next.next.next is not None
         assert self.interpret(f, []) == True
 
-#    def test_value_class(self):
-#
-#        class I(object):
-#            _value_class_ = True
-#
-#            def __init__(self, v):
-#                self.v = v
-#
-#        i = I(3)
-#        def f():
-#            return i.v
-#
-#        t, typer, graph = self.gengraph(f, [], backendopt=True)
-#        assert summary(graph) == {}
-#
-#    def test_value_class_conflicts_with_immut(self):
-#        from rpython.rtyper.rclass import ImmutableConflictError
-#
-#        class I(object):
-#            _immutable_   = False
-#            _value_class_ = True
-#
-#            def __init__(self, v):
-#                self.v = v
-#
-#        i = I(3)
-#        def f():
-#            return i.v
-#
-#        py.test.raises(ImmutableConflictError, self.gengraph, f, [])
-#
-#    def test_value_class_implies_immutable(self):
-#        from rpython.jit.metainterp.typesystem import deref
-#
-#        class I(object):
-#            _value_class_ = True
-#
-#            def __init__(self, v):
-#                self.v = v
-#
-#        i = I(3)
-#        def f():
-#            return i
-#
-#        t, typer, graph = self.gengraph(f, [])
-#        I_TYPE = deref(graph.getreturnvar().concretetype)
-#        assert I_TYPE._hints['immutable']
-#        assert I_TYPE._hints['value_class']
-#
-#    def test_value_class_subclass_not_value_class(self):
-#        from rpython.rtyper.rclass import ValueClassConflictError
-#
-#        class Base(object):
-#            _value_class_ = True
-#
-#        class I(Base):
-#            _immutable_   = True
-#
-#            def __init__(self, v):
-#                self.v = v
-#
-#        i = I(3)
-#        def f():
-#            return i.v
-#
-#        py.test.raises(ValueClassConflictError, self.gengraph, f, [])
-

From pypy.commits at gmail.com  Wed Sep 21 13:24:00 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 21 Sep 2016 10:24:00 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Replace all _immutable_=True
 annotations with _immutable_fields_=[...]
Message-ID: <57e2c230.04e21c0a.a57f0.79d4@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87283:b71c2014dd9d
Date: 2016-09-21 12:11 -0400
http://bitbucket.org/pypy/pypy/changeset/b71c2014dd9d/

Log:	Replace all _immutable_=True annotations with
	_immutable_fields_=[...] Tests converted to use _value_class_=True

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -332,7 +332,6 @@
 
             activation_cls = type("BuiltinActivation_UwS_%s" % label,
                              (BuiltinActivation,), d)
-            activation_cls._immutable_ = True
 
             cache[key] = activation_cls, self.run_args
             return activation_cls
@@ -346,7 +345,7 @@
 
 
 class BuiltinActivation(object):
-    _immutable_ = True
+    _immutable_fields_ = ['behavior']
 
     def __init__(self, behavior):
         """NOT_RPYTHON"""
@@ -564,7 +563,10 @@
 
 class BuiltinCode(Code):
     "The code object implementing a built-in (interpreter-level) hook."
-    _immutable_ = True
+    _immutable_fields_ = ['docstring', 'identifier', '_argnames',
+                          'descrmismatch_op', 'descr_reqcls',
+                          'sig', 'minargs', 'maxargs', 'activation',
+                          '_bltin', '_unwrap_spec', 'func__args__']
     hidden_applevel = True
     descrmismatch_op = None
     descr_reqcls = None
@@ -727,7 +729,6 @@
 
 
 class BuiltinCodePassThroughArguments0(BuiltinCode):
-    _immutable_ = True
 
     def funcrun(self, func, args):
         space = func.space
@@ -747,7 +748,6 @@
 
 
 class BuiltinCodePassThroughArguments1(BuiltinCode):
-    _immutable_ = True
     fast_natural_arity = Code.PASSTHROUGHARGS1
 
     def funcrun_obj(self, func, w_obj, args):
@@ -768,7 +768,6 @@
 
 
 class BuiltinCode0(BuiltinCode):
-    _immutable_ = True
     fast_natural_arity = 0
 
     def fastcall_0(self, space, w_func):
@@ -785,7 +784,6 @@
 
 
 class BuiltinCode1(BuiltinCode):
-    _immutable_ = True
     fast_natural_arity = 1
 
     def fastcall_1(self, space, w_func, w1):
@@ -805,7 +803,6 @@
 
 
 class BuiltinCode2(BuiltinCode):
-    _immutable_ = True
     fast_natural_arity = 2
 
     def fastcall_2(self, space, w_func, w1, w2):
@@ -825,7 +822,6 @@
 
 
 class BuiltinCode3(BuiltinCode):
-    _immutable_ = True
     fast_natural_arity = 3
 
     def fastcall_3(self, space, func, w1, w2, w3):
@@ -845,7 +841,6 @@
 
 
 class BuiltinCode4(BuiltinCode):
-    _immutable_ = True
     fast_natural_arity = 4
 
     def fastcall_4(self, space, func, w1, w2, w3, w4):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1344,14 +1344,13 @@
                 WHY_CONTINUE,   SContinueLoop
                 WHY_YIELD       not needed
     """
-    _immutable_ = True
     def nomoreblocks(self):
         raise BytecodeCorruption("misplaced bytecode - should not return")
 
 class SReturnValue(SuspendedUnroller):
     """Signals a 'return' statement.
     Argument is the wrapped object to return."""
-    _immutable_ = True
+    _immutable_fields_ = ['w_returnvalue']
     kind = 0x01
     def __init__(self, w_returnvalue):
         self.w_returnvalue = w_returnvalue
@@ -1361,7 +1360,7 @@
 class SApplicationException(SuspendedUnroller):
     """Signals an application-level exception
     (i.e. an OperationException)."""
-    _immutable_ = True
+    _immutable_fields_ = ['operr']
     kind = 0x02
     def __init__(self, operr):
         self.operr = operr
@@ -1370,14 +1369,13 @@
 
 class SBreakLoop(SuspendedUnroller):
     """Signals a 'break' statement."""
-    _immutable_ = True
     kind = 0x04
 SBreakLoop.singleton = SBreakLoop()
 
 class SContinueLoop(SuspendedUnroller):
     """Signals a 'continue' statement.
     Argument is the bytecode position of the beginning of the loop."""
-    _immutable_ = True
+    _immutable_fields_ = ['jump_to']
     kind = 0x08
     def __init__(self, jump_to):
         self.jump_to = jump_to
@@ -1387,7 +1385,7 @@
     """Abstract base class for frame blocks from the blockstack,
     used by the SETUP_XXX and POP_BLOCK opcodes."""
 
-    _immutable_ = True
+    _immutable_fields_ = ['handlerposition', 'valuestackdepth', 'previous']
 
     def __init__(self, frame, handlerposition, previous):
         self.handlerposition = handlerposition
@@ -1426,7 +1424,6 @@
 class LoopBlock(FrameBlock):
     """A loop block.  Stores the end-of-loop pointer in case of 'break'."""
 
-    _immutable_ = True
     _opname = 'SETUP_LOOP'
     handling_mask = SBreakLoop.kind | SContinueLoop.kind
 
@@ -1448,7 +1445,6 @@
 class ExceptBlock(FrameBlock):
     """An try:except: block.  Stores the position of the exception handler."""
 
-    _immutable_ = True
     _opname = 'SETUP_EXCEPT'
     handling_mask = SApplicationException.kind
 
@@ -1472,7 +1468,6 @@
 class FinallyBlock(FrameBlock):
     """A try:finally: block.  Stores the position of the exception handler."""
 
-    _immutable_ = True
     _opname = 'SETUP_FINALLY'
     handling_mask = -1     # handles every kind of SuspendedUnroller
 
@@ -1487,8 +1482,6 @@
 
 class WithBlock(FinallyBlock):
 
-    _immutable_ = True
-
     def handle(self, frame, unroller):
         if isinstance(unroller, SApplicationException):
             unroller.operr.normalize_exception(frame.space)
diff --git a/pypy/interpreter/signature.py b/pypy/interpreter/signature.py
--- a/pypy/interpreter/signature.py
+++ b/pypy/interpreter/signature.py
@@ -1,8 +1,7 @@
 from rpython.rlib import jit
 
 class Signature(object):
-    _immutable_ = True
-    _immutable_fields_ = ["argnames[*]"]
+    _immutable_fields_ = ["argnames[*]", "varargname", "kwargname"]
     __slots__ = ("argnames", "varargname", "kwargname")
 
     def __init__(self, argnames, varargname=None, kwargname=None):
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -353,7 +353,7 @@
 
 class Member(W_Root):
     """For slots."""
-    _immutable_ = True
+    _immutable_fields_ = ["index", "name", "w_cls"]
 
     def __init__(self, index, name, w_cls):
         self.index = index
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -7,7 +7,7 @@
 
 
 class ByteBuffer(Buffer):
-    _immutable_ = True
+    _immutable_fields_ = ['data', 'readonly']
 
     def __init__(self, len):
         self.data = ['\x00'] * len
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -7,7 +7,8 @@
 
 
 class W_Allocator(W_Root):
-    _immutable_ = True
+    _immutable_fields_ = ['ffi', 'w_alloc', 'w_free',
+                          'should_clear_after_alloc']
 
     def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
         self.ffi = ffi    # may be None
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -11,7 +11,7 @@
 
 
 class LLBuffer(Buffer):
-    _immutable_ = True
+    _immutable_fields_ = ['raw_cdata', 'size', 'readonly']
 
     def __init__(self, raw_cdata, size):
         self.raw_cdata = raw_cdata
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -378,7 +378,7 @@
 rffi_fclose = rffi.llexternal("fclose", [FILEP], rffi.INT)
 
 class CffiFileObj(object):
-    _immutable_ = True
+    _immutable_fields_ = ['llf']
 
     def __init__(self, fd, mode):
         self.llf = rffi_fdopen(fd, mode)
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -189,7 +189,7 @@
 
 
 class W_CField(W_Root):
-    _immutable_ = True
+    _immutable_fields_ = ['ctype', 'offset', 'bitshift', 'bitsize', 'flags']
 
     BS_REGULAR     = -1
     BS_EMPTY_ARRAY = -2
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -14,7 +14,7 @@
 
 
 class W_Library(W_Root):
-    _immutable_ = True
+    _immutable_fields_ = ['handle', 'name']
 
     def __init__(self, space, filename, flags):
         self.space = space
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -24,7 +24,8 @@
 
     This class cannot be used for variadic functions.
     """
-    _immutable_ = True
+    _immutable_fields_ = ['ffi', 'fnptr', 'directfnptr', 'rawfunctype',
+                          'fnname', 'modulename']
 
     def __init__(self, space, ffi, fnptr, directfnptr,
                  rawfunctype, fnname, modulename):
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -106,7 +106,7 @@
 )
 
 class RawBuffer(Buffer):
-    _immutable_ = True
+    _immutable_fields_ = ['buf', 'start', 'length', 'readonly']
 
     def __init__(self, buf, start, length):
         self.buf = buf
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -5,7 +5,7 @@
 
 
 class RawFFIBuffer(Buffer):
-    _immutable_ = True
+    _immutable_fields_ = ['datainstance', 'readonly']
 
     def __init__(self, datainstance):
         self.datainstance = datainstance
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -591,7 +591,7 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(Buffer):
-    _immutable_ = True
+    _immutable_fields_ = ["array", "readonly"]
 
     def __init__(self, array, readonly):
         self.array = array
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -20,7 +20,7 @@
 std_string_name = 'std::basic_string<char>'
 
 class _Arg:         # poor man's union
-    _immutable_ = True
+    _immutable_fields_ = ['_handle', '_long', '_string', '_voidp']
     def __init__(self, h = 0, l = -1, s = '', vp = rffi.cast(rffi.VOIDP, 0)):
         self._handle = h
         self._long   = l
diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -723,11 +723,10 @@
 
     for c_type, names in type_info:
         class BasicConverter(ffitypes.typeid(c_type), IntTypeConverterMixin, TypeConverter):
-            _immutable_ = True
+            _immutable_fields_ = ['default']
             def __init__(self, space, default):
                 self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
-            _immutable_ = True
             libffitype = jit_libffi.types.pointer
         for name in names:
             _converters[name] = BasicConverter
@@ -740,11 +739,10 @@
 
     for c_type, names in type_info:
         class BasicConverter(ffitypes.typeid(c_type), IntTypeConverterMixin, TypeConverter):
-            _immutable_ = True
+            _immutable_fields_ = ['default']
             def __init__(self, space, default):
                 self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
-            _immutable_ = True
             libffitype = jit_libffi.types.pointer
             typecode = 'r'
             def convert_argument(self, space, w_obj, address, call_local):
@@ -766,11 +764,10 @@
 
     for c_type, names in type_info:
         class BasicConverter(ffitypes.typeid(c_type), IntTypeConverterMixin, TypeConverter):
-            _immutable_ = True
+            _immutable_fields_ = ['default']
             def __init__(self, space, default):
                 self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
-            _immutable_ = True
             libffitype = jit_libffi.types.pointer
         for name in names:
             _converters[name] = BasicConverter
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -326,7 +326,6 @@
 
     for c_type, stub, names in type_info:
         class BasicExecutor(ffitypes.typeid(c_type), NumericExecutorMixin, FunctionExecutor):
-            _immutable_ = True
             c_stubcall  = staticmethod(stub)
         class BasicRefExecutor(ffitypes.typeid(c_type), NumericRefExecutorMixin, FunctionExecutor):
             _immutable_fields_ = ['libffitype']
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -136,7 +136,7 @@
 
 
 class W_CPPLibrary(W_Root):
-    _immutable_ = True
+    _immutable_fields_ = ["cdll"]
 
     def __init__(self, space, cdll):
         self.cdll = cdll
@@ -156,7 +156,9 @@
 
     _attrs_ = ['space', 'scope', 'index', 'cppmethod', 'arg_defs', 'args_required',
                'converters', 'executor', '_funcaddr', 'cif_descr', 'uses_local']
-    _immutable_ = True
+    _immutable_fields_ = ['scope', 'index', 'cppmethod', 'arg_defs', 'args_required',
+               'converters', 'executor', '_funcaddr', 'cif_descr', 'uses_local']
+
 
     def __init__(self, space, declaring_scope, method_index, arg_defs, args_required):
         self.space = space
@@ -403,8 +405,6 @@
 class CPPFunction(CPPMethod):
     """Global (namespaced) function dispatcher."""
 
-    _immutable_ = True
-
     @staticmethod
     def unpack_cppthis(space, w_cppinstance, declaring_scope):
         return capi.C_NULL_OBJECT
@@ -417,7 +417,7 @@
     """Method dispatcher that first resolves the template instance."""
 
     _attrs_ = ['space', 'templ_args']
-    _immutable_ = True
+    _immutable_fields_ = ['templ_args']
 
     def __init__(self, space, templ_args, declaring_scope, method_index, arg_defs, args_required):
         self.space = space
@@ -452,8 +452,6 @@
     reflection layer only, since the C++ class may have an overloaded operator
     new, disallowing malloc here."""
 
-    _immutable_ = True
-
     @staticmethod
     def unpack_cppthis(space, w_cppinstance, declaring_scope):
         return rffi.cast(capi.C_OBJECT, declaring_scope.handle)
@@ -473,8 +471,6 @@
     operator[](int). The former function takes an extra argument to assign to
     the return type of the latter."""
 
-    _immutable_ = True
-
     def call(self, cppthis, args_w):
         end = len(args_w)-1
         if 0 <= end:
@@ -624,7 +620,7 @@
 
 class W_CPPDataMember(W_Root):
     _attrs_ = ['space', 'scope', 'converter', 'offset']
-    _immutable_fields = ['scope', 'converter', 'offset']
+    _immutable_fields_ = ['scope', 'converter', 'offset']
 
     def __init__(self, space, declaring_scope, type_name, offset):
         self.space = space
@@ -998,7 +994,7 @@
 
 class W_CPPTemplateType(W_Root):
     _attrs_ = ['space', 'name', 'handle']
-    _immutable_fields = ['name', 'handle']
+    _immutable_fields_ = ['name', 'handle']
 
     def __init__(self, space, name, opaque_handle):
         self.space = space
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -300,7 +300,8 @@
 
 class CPyBuffer(Buffer):
     # Similar to Py_buffer
-    _immutable_ = True
+    _immutable_fields_ = ['ptr', 'size', 'w_obj', 'format', 'shape', 'strides',
+                          'ndim', 'itemsize', 'readonly']
 
     def __init__(self, ptr, size, w_obj, format='B', shape=None,
                 strides=None, ndim=1, itemsize=1, readonly=True):
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -702,7 +702,7 @@
 
 
 class ArrayBuffer(Buffer):
-    _immutable_ = True
+    _immutable_fields_ = ['impl', 'readonly']
 
     def __init__(self, impl, readonly):
         self.impl = impl
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -331,7 +331,7 @@
 
 
 class MMapBuffer(Buffer):
-    _immutable_ = True
+    _immutable_fields_ = ['mmap', 'readonly']
 
     def __init__(self, space, mmap, readonly):
         self.space = space
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1217,7 +1217,7 @@
 
 
 class BytearrayBuffer(Buffer):
-    _immutable_ = True
+    _immutable_fields_ = ['data', 'readonly']
 
     def __init__(self, data, readonly):
         self.data = data
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -100,7 +100,7 @@
     the first is either the same or a parent layout of the second.
     The Layouts have single inheritance, unlike W_TypeObjects.
     """
-    _immutable_ = True
+    _immutable_fields_ = ['typedef', 'nslots', 'base_layout']
 
     def __init__(self, typedef, nslots, newslotnames=[], base_layout=None):
         self.typedef = typedef
diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py
--- a/rpython/flowspace/argument.py
+++ b/rpython/flowspace/argument.py
@@ -4,8 +4,7 @@
 from rpython.flowspace.model import const
 
 class Signature(object):
-    _immutable_ = True
-    _immutable_fields_ = ["argnames[*]"]
+    _immutable_fields_ = ["argnames[*]", "varargnames", "kwargname"]
     __slots__ = ("argnames", "varargname", "kwargname")
 
     def __init__(self, argnames, varargname=None, kwargname=None):
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -12,7 +12,6 @@
 
 
 class AbstractPosition(object):
-    _immutable_ = True
     _attrs_ = ()
 
 def make_generator_entry_graph(func):
@@ -42,7 +41,6 @@
 def make_generatoriterator_class(var_names):
     class GeneratorIterator(object):
         class Entry(AbstractPosition):
-            _immutable_ = True
             varnames = var_names
 
         def __init__(self, entry):
@@ -135,8 +133,7 @@
                 varnames = get_variable_names(newlink.args)
                 #
                 class Resume(AbstractPosition):
-                    _immutable_ = True
-                    _attrs_ = varnames
+                    _immutable_fields_ = _attrs_ = varnames
                     block = newblock
                 Resume.__name__ = 'Resume%d' % len(mappings)
                 mappings.append(Resume)
diff --git a/rpython/flowspace/test/test_generator.py b/rpython/flowspace/test/test_generator.py
--- a/rpython/flowspace/test/test_generator.py
+++ b/rpython/flowspace/test/test_generator.py
@@ -39,11 +39,11 @@
         return self
 
 class AbstractPosition(object):
-    _immutable_ = True
+    _immutable_fields_ = []
 class Entry1(AbstractPosition):
-    _immutable_ = True
+    _immutable_fields_ = []
 class Yield1(AbstractPosition):
-    _immutable_ = True
+    _immutable_fields_ = []
 
 def f_explicit(n):
     e = Entry1()
diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py
--- a/rpython/jit/backend/arm/locations.py
+++ b/rpython/jit/backend/arm/locations.py
@@ -2,7 +2,7 @@
 from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, JITFRAME_FIXED_SIZE
 
 class AssemblerLocation(object):
-    _immutable_ = True
+    _immutable_fields_ = ['type']
     type = INT
 
     def is_imm(self):
@@ -33,7 +33,7 @@
         raise NotImplementedError # only for stack
 
 class RegisterLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['value']
     width = WORD
 
     def __init__(self, value):
@@ -50,7 +50,6 @@
 
 
 class VFPRegisterLocation(RegisterLocation):
-    _immutable_ = True
     type = FLOAT
     width = 2 * WORD
 
@@ -71,7 +70,6 @@
 
 class SVFPRegisterLocation(VFPRegisterLocation):
     """Single Precission VFP Register"""
-    _immutable_ = True
     width = WORD
     type = 'S'
 
@@ -79,7 +77,7 @@
         return 'vfp(s%d)' % self.value
 
 class ImmLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['value']
     width = WORD
 
     def __init__(self, value):
@@ -98,7 +96,7 @@
 class ConstFloatLoc(AssemblerLocation):
     """This class represents an imm float value which is stored in memory at
     the address stored in the field value"""
-    _immutable_ = True
+    _immutable_fields_ = ['value']
     width = 2 * WORD
     type = FLOAT
 
@@ -121,7 +119,7 @@
         return True
 
 class StackLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['width', 'position', 'value']
 
     def __init__(self, position, fp_offset, type=INT):
         if type == FLOAT:
@@ -154,7 +152,7 @@
         return self.type == FLOAT
 
 class RawSPStackLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['width', 'value']
 
     def __init__(self, sp_offset, type=INT):
         if type == FLOAT:
diff --git a/rpython/jit/backend/ppc/locations.py b/rpython/jit/backend/ppc/locations.py
--- a/rpython/jit/backend/ppc/locations.py
+++ b/rpython/jit/backend/ppc/locations.py
@@ -12,7 +12,6 @@
 
 
 class AssemblerLocation(object):
-    _immutable_ = True
     type = INT
 
     def is_imm(self):
@@ -40,7 +39,7 @@
         raise NotImplementedError
 
 class RegisterLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ["value"]
     width = WORD
 
     def __init__(self, value):
@@ -56,7 +55,6 @@
         return self.value
 
 class FPRegisterLocation(RegisterLocation):
-    _immutable_ = True
     type = FLOAT
     width = FWORD
 
@@ -76,7 +74,7 @@
         return self.value + 100
 
 class ImmLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ["value"]
     width = WORD
 
 
@@ -95,7 +93,7 @@
 class ConstFloatLoc(AssemblerLocation):
     """This class represents an imm float value which is stored in memory at
     the address stored in the field value"""
-    _immutable_ = True
+    _immutable_fields_ = ["value"]
     width = FWORD
     type = FLOAT
 
@@ -115,7 +113,7 @@
         return self.value
 
 class StackLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ["width", "position", "value", "type"]
 
     def __init__(self, position, fp_offset, type=INT):
         if type == FLOAT:
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -18,7 +18,7 @@
 
 class AssemblerLocation(object):
     _attrs_ = ('value', '_location_code')
-    _immutable_ = True
+    _immutable_fields_ = ['value', '_location_code']
     def _getregkey(self):
         return self.value
 
@@ -55,7 +55,7 @@
     """ The same as stack location, but does not know it's position.
     Mostly usable for raw frame access
     """
-    _immutable_ = True
+    _immutable_fields_ = ['type']
     _location_code = 'b'
 
     def __init__(self, value, type=INT):
@@ -85,7 +85,7 @@
 class RawEspLoc(AssemblerLocation):
     """ Esp-based location
     """
-    _immutable_ = True
+    _immutable_fields_ = ['type']
     _location_code = 's'
 
     def __init__(self, value, type):
@@ -111,7 +111,7 @@
         return self.type == FLOAT
 
 class FrameLoc(RawEbpLoc):
-    _immutable_ = True
+    _immutable_fields_ = ['position', 'type']
     
     def __init__(self, position, ebp_offset, type):
         # _getregkey() returns self.value; the value returned must not
@@ -129,7 +129,7 @@
         return self.position
 
 class RegLoc(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['is_xmm']
     def __init__(self, regnum, is_xmm):
         assert regnum >= 0
         self.value = regnum
@@ -175,10 +175,10 @@
         return True
 
 class ImmediateAssemblerLocation(AssemblerLocation):
-    _immutable_ = True
+    pass
 
 class ImmedLoc(ImmediateAssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['_is_float']
     _location_code = 'i'
 
     def __init__(self, value, is_float=False):
@@ -205,7 +205,7 @@
         return self._is_float
 
 class AddressLoc(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['loc_a', 'loc_m']
 
     # The address is base_loc + (scaled_loc << scale) + static_offset
     def __init__(self, base_loc, scaled_loc, scale=0, static_offset=0):
@@ -277,7 +277,6 @@
         return result
 
 class ConstFloatLoc(ImmediateAssemblerLocation):
-    _immutable_ = True
     _location_code = 'j'
 
     def __init__(self, address):
@@ -298,7 +297,7 @@
         # any assembler instruction.  Instead, it is meant to be decomposed
         # in two 32-bit halves.  On 64-bit, FloatImmedLoc() is a function
         # instead; see below.
-        _immutable_ = True
+        _immutable_fields_ = ['aslonglong']
         _location_code = '#'     # don't use me
 
         def __init__(self, floatstorage):
diff --git a/rpython/jit/backend/zarch/conditions.py b/rpython/jit/backend/zarch/conditions.py
--- a/rpython/jit/backend/zarch/conditions.py
+++ b/rpython/jit/backend/zarch/conditions.py
@@ -2,7 +2,6 @@
 from rpython.rlib.objectmodel import specialize
 
 class ConditionLocation(loc.ImmLocation):
-    _immutable_ = True
     def __repr__(self):
         s = ""
         if self.value & 0x1 != 0:
diff --git a/rpython/jit/backend/zarch/locations.py b/rpython/jit/backend/zarch/locations.py
--- a/rpython/jit/backend/zarch/locations.py
+++ b/rpython/jit/backend/zarch/locations.py
@@ -4,8 +4,8 @@
 FWORD = 8
 
 class AssemblerLocation(object):
-    _immutable_ = True
     type = INT
+    _immutable_fields_ = ['type']
 
     def is_imm(self):
         return False
@@ -41,7 +41,7 @@
         raise NotImplementedError # only for stack
 
 class RegisterLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['value']
     width = WORD
 
     def __init__(self, value):
@@ -65,7 +65,7 @@
 class ConstFloatLoc(AssemblerLocation):
     """This class represents an imm float value which is stored in memory at
     the address stored in the field value"""
-    _immutable_ = True
+    _immutable_fields_ = ['value']
     width = FWORD
     type = FLOAT
 
@@ -88,7 +88,6 @@
         return self.value
 
 class FloatRegisterLocation(RegisterLocation):
-    _immutable_ = True
     type = FLOAT
     width = DOUBLE_WORD
 
@@ -108,7 +107,7 @@
         return True
 
 class ImmLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['value']
     width = WORD
 
     def __init__(self, value):
@@ -124,7 +123,7 @@
         return True
 
 class StackLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['width', 'position', 'value']
 
     def __init__(self, position, fp_offset, type=INT):
         if type == FLOAT:
@@ -157,7 +156,8 @@
         return self.type == FLOAT
 
 class AddressLocation(AssemblerLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['displace', 'base', 'index', 'length', 'basereg',
+                          'base', 'index', 'length']
 
     def __init__(self, basereg, indexreg, displace, length):
         self.displace = displace
@@ -177,7 +177,7 @@
             self.length = length.value
 
 class PoolLoc(AddressLocation):
-    _immutable_ = True
+    _immutable_fields_ = ['isfloat']
     width = WORD
 
     def __init__(self, offset, isfloat=False):
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -304,7 +304,7 @@
         myjitdriver = JitDriver(greens = [], reds = ['y', 'res', 'x'])
         class I:
             __slots__ = 'intval'
-            _immutable_ = True
+            _immutable_fields_ = ['intval']
             def __init__(self, intval):
                 self.intval = intval
         def f(i, y):
@@ -845,7 +845,7 @@
 
     def test_getfield_immutable(self):
         class A:
-            _immutable_ = True
+            _immutable_fields_ = ['foo']
         a1 = A()
         a1.foo = 5
         a2 = A()
diff --git a/rpython/jit/metainterp/test/test_immutable.py b/rpython/jit/metainterp/test/test_immutable.py
--- a/rpython/jit/metainterp/test/test_immutable.py
+++ b/rpython/jit/metainterp/test/test_immutable.py
@@ -90,7 +90,7 @@
 
     def test_array_in_immutable(self):
         class X(object):
-            _immutable_ = True
+            _value_class_ = True
             _immutable_fields_ = ["lst[*]"]
 
             def __init__(self, lst, y):
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -288,12 +288,12 @@
         myjitdriver = JitDriver(greens=['stufflist'], reds=['n', 'i'])
 
         class Stuff(object):
-            _immutable_ = True
+            _immutable_fields_ = ['x']
             def __init__(self, x):
                 self.x = x
 
         class StuffList(object):
-            _immutable_ = True
+            _immutable_fields_ = ['lst']
 
         def f(n, a, i):
             stufflist = StuffList()
diff --git a/rpython/jit/tl/tiny2_hotpath.py b/rpython/jit/tl/tiny2_hotpath.py
--- a/rpython/jit/tl/tiny2_hotpath.py
+++ b/rpython/jit/tl/tiny2_hotpath.py
@@ -43,7 +43,7 @@
     pass
 
 class IntBox(Box):
-    _immutable_ = True
+    _immutable_fields_ = ['intval']
     def __init__(self, intval):
         self.intval = intval
     def as_int(self):
@@ -52,7 +52,7 @@
         return str(self.intval)
 
 class StrBox(Box):
-    _immutable_ = True
+    _immutable_fields_ = ['strval']
     def __init__(self, strval):
         self.strval = strval
     def as_int(self):
diff --git a/rpython/jit/tl/tiny3_hotpath.py b/rpython/jit/tl/tiny3_hotpath.py
--- a/rpython/jit/tl/tiny3_hotpath.py
+++ b/rpython/jit/tl/tiny3_hotpath.py
@@ -45,7 +45,7 @@
     pass
 
 class IntBox(Box):
-    _immutable_ = True
+    _immutable_fields_ = ['intval']
     def __init__(self, intval):
         self.intval = intval
     def as_int(self):
@@ -56,7 +56,7 @@
         return str(self.intval)
 
 class FloatBox(Box):
-    _immutable_ = True
+    _immutable_fields_ = ['floatval']
     def __init__(self, floatval):
         self.floatval = floatval
     def as_int(self):
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -7,7 +7,7 @@
 class Buffer(object):
     """Abstract base class for buffers."""
     __slots__ = ['readonly']
-    _immutable_ = True
+    _immutable_fields_ = __slots__
 
     def getlength(self):
         """Returns the size in bytes (even if getitemsize() > 1)."""
@@ -77,7 +77,7 @@
 
 class StringBuffer(Buffer):
     __slots__ = ['value']
-    _immutable_ = True
+    _immutable_fields_ = ['value']
 
     def __init__(self, value):
         self.value = value
@@ -108,7 +108,7 @@
 
 class SubBuffer(Buffer):
     __slots__ = ['buffer', 'offset', 'size']
-    _immutable_ = True
+    _immutable_fields_ = __slots__
 
     def __init__(self, buffer, offset, size):
         self.readonly = buffer.readonly
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -136,8 +136,7 @@
 
 class rbigint(object):
     """This is a reimplementation of longs using a list of digits."""
-    _immutable_ = True
-    _immutable_fields_ = ["_digits"]
+    _immutable_fields_ = ["_digits", "size", "sign"]
 
     def __init__(self, digits=[NULLDIGIT], sign=0, size=0):
         if not we_are_translated():
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -243,7 +243,7 @@
 # ____________________________________________________________
 
 class Mark(object):
-    _immutable_ = True
+    _immutable_fields_ = ['gid', 'position', 'prev']
 
     def __init__(self, gid, position, prev):
         self.gid = gid
diff --git a/rpython/rtyper/test/test_generator.py b/rpython/rtyper/test/test_generator.py
--- a/rpython/rtyper/test/test_generator.py
+++ b/rpython/rtyper/test/test_generator.py
@@ -23,11 +23,11 @@
         # merging two different generators is not supported
         # right now, but we can use workarounds like here
         class MyGen:
-            _immutable_ = True
+            _immutable_fields_ = []
             def next(self):
                 raise NotImplementedError
         class MyG1(MyGen):
-            _immutable_ = True
+            _immutable_fields_ = ["_gen"]
             def __init__(self, a):
                 self._gen = self.g1(a)
             def next(self):
@@ -37,7 +37,7 @@
                 yield a + 1
                 yield a + 2
         class MyG2(MyGen):
-            _immutable_ = True
+            _immutable_fields_ = ["_gen"]
             def __init__(self):
                 self._gen = self.g2()
             def next(self):
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -745,9 +745,9 @@
         assert res == 0
 
 
-    def test_immutable(self):
+    def test_value_class(self):
         class I(object):
-            _immutable_ = True
+            _value_class_ = True
 
             def __init__(self, v):
                 self.v = v
@@ -849,10 +849,10 @@
         class A(object):
             pass
         class B(A):
-            _immutable_ = True
+            _value_class_ = True
         def f():
             A().v = 123
-            B()             # crash: class B has _immutable_ = True
+            B()             # crash: class B has _value_class_ = True
                             # but class A defines 'v' to be mutable
         py.test.raises(ImmutableConflictError, self.gengraph, f, [])
 
@@ -861,7 +861,7 @@
         class A(object):
             _immutable_fields_ = ['v']
         class B(A):
-            _immutable_ = True
+            _value_class_ = True
         def f():
             A().v = 123
             B().w = 456
@@ -877,7 +877,7 @@
         from rpython.rtyper.rclass import ImmutableConflictError
         from rpython.jit.metainterp.typesystem import deref
         class A(object):
-            _immutable_ = True
+            _value_class_ = True
         class B(A):
             pass
         def f():
@@ -891,7 +891,7 @@
         class A(object):
             pass
         class B(A):
-            _immutable_ = True
+            _value_class_ = True
         def f():
             A()
             B().v = 123
@@ -905,12 +905,12 @@
         class A(object):
             pass
         class B(A):
-            _immutable_ = True
+            _value_class_ = True
         def myfunc():
             pass
         def f():
             A().f = myfunc    # it's ok to add Void attributes to A
-            B().v = 123       # even though only B is declared _immutable_
+            B().v = 123       # even though only B is declared _value_class_
             return B()
         t, typer, graph = self.gengraph(f, [])
         B_TYPE = deref(graph.getreturnvar().concretetype)
@@ -946,7 +946,7 @@
     def test_quasi_immutable_clashes_with_immutable(self):
         from rpython.jit.metainterp.typesystem import deref
         class A(object):
-            _immutable_ = True
+            _value_class_ = True
             _immutable_fields_ = ['a?']
         def f():
             a = A()
@@ -1306,70 +1306,70 @@
             return a.next.next.next.next is not None
         assert self.interpret(f, []) == True
 
-    def test_value_class(self):
+#    def test_value_class(self):
+#
+#        class I(object):
+#            _value_class_ = True
+#
+#            def __init__(self, v):
+#                self.v = v
+#
+#        i = I(3)
+#        def f():
+#            return i.v
+#
+#        t, typer, graph = self.gengraph(f, [], backendopt=True)
+#        assert summary(graph) == {}
+#
+#    def test_value_class_conflicts_with_immut(self):
+#        from rpython.rtyper.rclass import ImmutableConflictError
+#
+#        class I(object):
+#            _immutable_   = False
+#            _value_class_ = True
+#
+#            def __init__(self, v):
+#                self.v = v
+#
+#        i = I(3)
+#        def f():
+#            return i.v
+#
+#        py.test.raises(ImmutableConflictError, self.gengraph, f, [])
+#
+#    def test_value_class_implies_immutable(self):
+#        from rpython.jit.metainterp.typesystem import deref
+#
+#        class I(object):
+#            _value_class_ = True
+#
+#            def __init__(self, v):
+#                self.v = v
+#
+#        i = I(3)
+#        def f():
+#            return i
+#
+#        t, typer, graph = self.gengraph(f, [])
+#        I_TYPE = deref(graph.getreturnvar().concretetype)
+#        assert I_TYPE._hints['immutable']
+#        assert I_TYPE._hints['value_class']
+#
+#    def test_value_class_subclass_not_value_class(self):
+#        from rpython.rtyper.rclass import ValueClassConflictError
+#
+#        class Base(object):
+#            _value_class_ = True
+#
+#        class I(Base):
+#            _immutable_   = True
+#
+#            def __init__(self, v):
+#                self.v = v
+#
+#        i = I(3)
+#        def f():
+#            return i.v
+#
+#        py.test.raises(ValueClassConflictError, self.gengraph, f, [])
 
-        class I(object):
-            _value_class_ = True
-
-            def __init__(self, v):
-                self.v = v
-
-        i = I(3)
-        def f():
-            return i.v
-
-        t, typer, graph = self.gengraph(f, [], backendopt=True)
-        assert summary(graph) == {}
-
-    def test_value_class_conflicts_with_immut(self):
-        from rpython.rtyper.rclass import ImmutableConflictError
-
-        class I(object):
-            _immutable_   = False
-            _value_class_ = True
-
-            def __init__(self, v):
-                self.v = v
-
-        i = I(3)
-        def f():
-            return i.v
-
-        py.test.raises(ImmutableConflictError, self.gengraph, f, [])
-
-    def test_value_class_implies_immutable(self):
-        from rpython.jit.metainterp.typesystem import deref
-
-        class I(object):
-            _value_class_ = True
-
-            def __init__(self, v):
-                self.v = v
-
-        i = I(3)
-        def f():
-            return i
-
-        t, typer, graph = self.gengraph(f, [])
-        I_TYPE = deref(graph.getreturnvar().concretetype)
-        assert I_TYPE._hints['immutable']
-        assert I_TYPE._hints['value_class']
-
-    def test_value_class_subclass_not_value_class(self):
-        from rpython.rtyper.rclass import ValueClassConflictError
-
-        class Base(object):
-            _value_class_ = True
-
-        class I(Base):
-            _immutable_   = True
-
-            def __init__(self, v):
-                self.v = v
-
-        i = I(3)
-        def f():
-            return i.v
-
-        py.test.raises(ValueClassConflictError, self.gengraph, f, [])
-

From pypy.commits at gmail.com  Wed Sep 21 13:24:03 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 21 Sep 2016 10:24:03 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Change is_immutable to
 is_value_class
Message-ID: <57e2c233.29a9c20a.149f9.c4a4@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87285:28ad6be3c6c6
Date: 2016-09-21 13:13 -0400
http://bitbucket.org/pypy/pypy/changeset/28ad6be3c6c6/

Log:	Change is_immutable to is_value_class

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -136,9 +136,6 @@
                                                                self.S)
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self._vtable))
 
-    def is_immutable(self):
-        return heaptracker.is_immutable_struct(self.S)
-
     def is_value_class(self):
         return heaptracker.is_value_class(self.S)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -60,19 +60,16 @@
     size = 0      # help translation
     tid = llop.combine_ushort(lltype.Signed, 0, 0)
     vtable = lltype.nullptr(rclass.OBJECT_VTABLE)
-    immutable_flag = False
     value_class_flag = False
 
     def __init__(self, size, gc_fielddescrs=None, all_fielddescrs=None,
                  vtable=lltype.nullptr(rclass.OBJECT_VTABLE),
-                 immutable_flag=False,
                  value_class_flag=False):
         assert lltype.typeOf(vtable) == lltype.Ptr(rclass.OBJECT_VTABLE)
         self.size = size
         self.gc_fielddescrs = gc_fielddescrs
         self.all_fielddescrs = all_fielddescrs
         self.vtable = vtable
-        self.immutable_flag = immutable_flag
         self.value_class_flag = value_class_flag
 
     def get_all_fielddescrs(self):
@@ -94,9 +91,6 @@
         # fields
         return objptr.typeptr == cls or rclass.ll_isinstance(objptr, cls)
 
-    def is_immutable(self):
-        return self.immutable_flag
-
     def is_value_class(self):
         return self.value_class_flag
 
@@ -114,14 +108,12 @@
         return cache[STRUCT]
     except KeyError:
         size = symbolic.get_size(STRUCT, gccache.translate_support_code)
-        immutable_flag = heaptracker.is_immutable_struct(STRUCT)
         value_class_flag = heaptracker.is_value_class(STRUCT)
         if vtable:
             assert heaptracker.has_gcstruct_a_vtable(STRUCT)
         else:
             assert not heaptracker.has_gcstruct_a_vtable(STRUCT)
         sizedescr = SizeDescr(size, vtable=vtable,
-                              immutable_flag=immutable_flag,
                               value_class_flag=value_class_flag)
         gccache.init_size_descr(STRUCT, sizedescr)
         cache[STRUCT] = sizedescr
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -17,8 +17,8 @@
     descr_t = get_size_descr(c0, T)
     assert descr_s.size == symbolic.get_size(S, False)
     assert descr_t.size == symbolic.get_size(T, False)
-    assert descr_s.is_immutable() == False
-    assert descr_t.is_immutable() == False
+    assert descr_s.is_value_class() == False
+    assert descr_t.is_value_class() == False
     assert descr_t.gc_fielddescrs == []
     assert len(descr_s.gc_fielddescrs) == 1
     assert descr_s == get_size_descr(c0, S)
@@ -26,7 +26,7 @@
     #
     descr_s = get_size_descr(c1, S)
     assert isinstance(descr_s.size, Symbolic)
-    assert descr_s.is_immutable() == False
+    assert descr_s.is_value_class() == False
 
     PARENT = lltype.Struct('P', ('x', lltype.Ptr(T)))
     STRUCT = lltype.GcStruct('S', ('parent', PARENT), ('y', lltype.Ptr(T)))
@@ -34,38 +34,18 @@
     assert len(descr_struct.gc_fielddescrs) == 2
 
 def test_get_size_descr_immut():
-    S = lltype.GcStruct('S', hints={'immutable': True})
+    S = lltype.GcStruct('S', hints={'value_class': True})
     T = lltype.GcStruct('T', ('parent', S),
                         ('x', lltype.Char),
-                        hints={'immutable': True})
+                        hints={'value_class': True})
     U = lltype.GcStruct('U', ('parent', T),
                         ('u', lltype.Ptr(T)),
                         ('v', lltype.Signed),
-                        hints={'immutable': True})
+                        hints={'value_class': True})
     V = lltype.GcStruct('V', ('parent', U),
                         ('miss1', lltype.Void),
                         ('miss2', lltype.Void),
-                        hints={'immutable': True})
-    for STRUCT in [S, T, U, V]:
-        for translated in [False, True]:
-            c0 = GcCache(translated)
-            descr_s = get_size_descr(c0, STRUCT)
-            assert descr_s.is_immutable() == True
-
-def test_get_size_descr_value_class():
-    hints = {'immutable': True, 'value_class': True}
-    S = lltype.GcStruct('S', hints=hints)
-    T = lltype.GcStruct('T', ('parent', S),
-                        ('x', lltype.Char),
-                        hints=hints)
-    U = lltype.GcStruct('U', ('parent', T),
-                        ('u', lltype.Ptr(T)),
-                        ('v', lltype.Signed),
-                        hints=hints)
-    V = lltype.GcStruct('V', ('parent', U),
-                        ('miss1', lltype.Void),
-                        ('miss2', lltype.Void),
-                        hints=hints)
+                        hints={'value_class': True})
     for STRUCT in [S, T, U, V]:
         for translated in [False, True]:
             c0 = GcCache(translated)
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -20,9 +20,6 @@
     a -= r_uint(1 << (b8 - 1))     # a -= 128
     return intmask(a)
 
-def is_immutable_struct(S):
-    return isinstance(S, lltype.GcStruct) and S._hints.get('immutable', False)
-
 def is_value_class(S):
     return isinstance(S, lltype.GcStruct) and S._hints.get('value_class', False)
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -30,7 +30,7 @@
     def getconst(self):
         raise Exception("not a constant")
 
-    def _is_immutable_and_filled_with_constants(self, optimizer, memo=None):
+    def _is_value_class_and_filled_with_constants(self, optimizer, memo=None):
         return False
 
 
@@ -134,7 +134,7 @@
         if self.is_virtual():
             optforce.forget_numberings()
             #
-            if self._is_immutable_and_filled_with_constants(optforce.optimizer):
+            if self._is_value_class_and_filled_with_constants(optforce.optimizer):
                 constptr = optforce.optimizer.constant_fold(op)
                 op.set_forwarded(constptr)
                 self._is_virtual = False
@@ -262,7 +262,7 @@
         getfield_op = ResOperation(opnum, [structbox], descr=fielddescr)
         shortboxes.add_heap_op(op, getfield_op)
 
-    def _is_immutable_and_filled_with_constants(self, optimizer, memo=None):
+    def _is_value_class_and_filled_with_constants(self, optimizer, memo=None):
         # check if it is possible to force the given structure into a
         # compile-time constant: this is allowed only if it is declared
         # immutable, if all fields are already filled, and if each field
@@ -270,7 +270,7 @@
         # which also answers True to the same question.
         #
         assert self.is_virtual()
-        if not self.descr.is_immutable():
+        if not self.descr.is_value_class():
             return False
         if memo is not None and self in memo:
             return True       # recursive case: assume yes
@@ -287,7 +287,7 @@
                     # recursive check
                     if memo is None:
                         memo = {self: None}
-                    if not fieldinfo._is_immutable_and_filled_with_constants(
+                    if not fieldinfo._is_value_class_and_filled_with_constants(
                             optimizer, memo):
                         return False
                 else:

From pypy.commits at gmail.com  Wed Sep 21 13:27:15 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 21 Sep 2016 10:27:15 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: failargs can have None entries,
 consider this while iterating
Message-ID: <57e2c2f3.c398c20a.f5f21.e316@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87286:46098ed56db2
Date: 2016-09-21 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/46098ed56db2/

Log:	failargs can have None entries, consider this while iterating

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -46,7 +46,7 @@
         if op.is_guard():
             args = args[:] + op.getfailargs()
         for arg in args:
-            if arg.is_constant() or arg.is_inputarg():
+            if arg is None or arg.is_constant() or arg.is_inputarg():
                 continue
             if arg not in self.seen:
                 box = self.renamer.rename_box(arg)

From pypy.commits at gmail.com  Wed Sep 21 13:42:12 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 21 Sep 2016 10:42:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: 2to3fy test
Message-ID: <57e2c674.90941c0a.cb1d5.9505@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87287:29e03dc11794
Date: 2016-09-21 18:41 +0100
http://bitbucket.org/pypy/pypy/changeset/29e03dc11794/

Log:	2to3fy test

diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,7 +3,7 @@
 import py, pytest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
-only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 def test_pragma_version():
     from pypy.module.sys.version import CPYTHON_VERSION
@@ -46,10 +46,18 @@
     def test_pypy_versions(self):
         import sys
         init = """
+        static struct PyModuleDef moduledef = {
+                PyModuleDef_HEAD_INIT,
+                "foo",          /* m_name */
+                NULL,           /* m_doc */
+                -1,             /* m_size */
+                NULL            /* m_methods */
+            };
         if (Py_IsInitialized()) {
-            PyObject *m = Py_InitModule("foo", NULL);
+            PyObject *m = PyModule_Create(&moduledef);
             PyModule_AddStringConstant(m, "pypy_version", PYPY_VERSION);
             PyModule_AddIntConstant(m, "pypy_version_num", PYPY_VERSION_NUM);
+            return m;
         }
         Py_RETURN_NONE;
         """

From pypy.commits at gmail.com  Wed Sep 21 14:21:52 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 21 Sep 2016 11:21:52 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Fix some JIT tests
Message-ID: <57e2cfc0.9102c20a.7fd22.dd9b@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87288:9811cbee3e13
Date: 2016-09-21 14:09 -0400
http://bitbucket.org/pypy/pypy/changeset/9811cbee3e13/

Log:	Fix some JIT tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -124,11 +124,11 @@
     NODE3.become(lltype.GcStruct('NODE3', ('parent', OBJECT),
                             ('value', lltype.Signed),
                             ('next', lltype.Ptr(NODE3)),
-                            hints={'immutable': True}))
+                            hints={'value_class': True}))
 
     big_fields = [('big' + i, lltype.Signed) for i in string.ascii_lowercase]
     BIG = lltype.GcForwardReference()
-    BIG.become(lltype.GcStruct('BIG', *big_fields, hints={'immutable': True}))
+    BIG.become(lltype.GcStruct('BIG', *big_fields, hints={'value_class': True}))
 
     for field, _ in big_fields:
         locals()[field + 'descr'] = cpu.fielddescrof(BIG, field)
@@ -202,7 +202,7 @@
                                                 ('intval', lltype.Signed))
     INTOBJ_IMMUT = lltype.GcStruct('INTOBJ_IMMUT', ('parent', OBJECT),
                                             ('intval', lltype.Signed),
-                                            hints={'immutable': True})
+                                            hints={'value_class': True})
     intobj_noimmut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     intobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     noimmut_intval = cpu.fielddescrof(INTOBJ_NOIMMUT, 'intval')
@@ -214,7 +214,7 @@
 
     PTROBJ_IMMUT = lltype.GcStruct('PTROBJ_IMMUT', ('parent', OBJECT),
                                             ('ptrval', lltype.Ptr(OBJECT)),
-                                            hints={'immutable': True})
+                                            hints={'value_class': True})
     ptrobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     ptrobj_immut_descr = cpu.sizeof(PTROBJ_IMMUT, ptrobj_immut_vtable)
     immut_ptrval = cpu.fielddescrof(PTROBJ_IMMUT, 'ptrval')

From pypy.commits at gmail.com  Wed Sep 21 14:21:53 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 21 Sep 2016 11:21:53 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Make a note and undo some changes
 to tests
Message-ID: <57e2cfc1.43e61c0a.64418.c12b@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87289:41ba0c22b58a
Date: 2016-09-21 14:19 -0400
http://bitbucket.org/pypy/pypy/changeset/41ba0c22b58a/

Log:	Make a note and undo some changes to tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -124,11 +124,11 @@
     NODE3.become(lltype.GcStruct('NODE3', ('parent', OBJECT),
                             ('value', lltype.Signed),
                             ('next', lltype.Ptr(NODE3)),
-                            hints={'value_class': True}))
+                            hints={'immutable': True}))
 
     big_fields = [('big' + i, lltype.Signed) for i in string.ascii_lowercase]
     BIG = lltype.GcForwardReference()
-    BIG.become(lltype.GcStruct('BIG', *big_fields, hints={'value_class': True}))
+    BIG.become(lltype.GcStruct('BIG', *big_fields, hints={'immutable': True}))
 
     for field, _ in big_fields:
         locals()[field + 'descr'] = cpu.fielddescrof(BIG, field)
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -370,6 +370,9 @@
         return _struct(self, n, initialization='example')
 
     def _immutable_field(self, field):
+        # The translator no longer accepts the _immutable_=True annotation
+        # but it is useful to have an to have a shorthand for immutable objects
+        # which are value classes for testing purposes.
         if (self._hints.get('immutable') or
             self._hints.get('value_class')):
             return True

From pypy.commits at gmail.com  Wed Sep 21 15:08:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 21 Sep 2016 12:08:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <57e2dabd.670dc20a.e0377.03e4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87290:4b1d7e36364a
Date: 2016-09-21 20:08 +0100
http://bitbucket.org/pypy/pypy/changeset/4b1d7e36364a/

Log:	2to3

diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -46,7 +46,7 @@
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
-        assert isinstance(w_value, W_LongObject)
+        assert space.isinstance_w(w_value, space.w_int)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -108,24 +108,20 @@
         lltype.free(overflow, flavor='raw')
 
     def test_as_voidptr(self, space, api):
-        # CPython returns an int (not a long) depending on the value
-        # passed to PyLong_FromVoidPtr().  In all cases, NULL becomes
-        # the int 0.
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
         assert space.is_w(space.type(w_l), space.w_int)
         assert space.unwrap(w_l) == 0
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
-        # Positive values also return an int (assuming, like always in
-        # PyPy, that an int is big enough to store any pointer).
+
         p = rffi.cast(rffi.VOIDP, maxint)
         w_l = api.PyLong_FromVoidPtr(p)
         assert space.is_w(space.type(w_l), space.w_int)
         assert space.unwrap(w_l) == maxint
         assert api.PyLong_AsVoidPtr(w_l) == p
-        # Negative values always return a long.
+
         p = rffi.cast(rffi.VOIDP, -maxint-1)
         w_l = api.PyLong_FromVoidPtr(p)
-        assert space.is_w(space.type(w_l), space.w_long)
+        assert space.is_w(space.type(w_l), space.w_int)
         assert space.unwrap(w_l) == maxint+1
         assert api.PyLong_AsVoidPtr(w_l) == p
 
@@ -287,7 +283,7 @@
                     return PyLong_FromLong(3);
                  if (str + strlen(str) != end)
                     return PyLong_FromLong(4);
-                 return PyLong_FromLong(0); 
+                 return PyLong_FromLong(0);
              """)])
         assert module.from_str() == 0
 
@@ -343,4 +339,4 @@
         assert module.has_pow() == 0
         assert module.has_hex() == '0x2aL'
         assert module.has_oct() == '052L'
-                
+

From pypy.commits at gmail.com  Wed Sep 21 16:14:47 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 21 Sep 2016 13:14:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: replace a number of _annspecialcase_ by
 @specialize in rlib/
Message-ID: <57e2ea37.ca0e1c0a.a952.b9f5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r87291:f4db251591bc
Date: 2016-09-21 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/f4db251591bc/

Log:	replace a number of _annspecialcase_ by @specialize in rlib/

diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -359,12 +359,13 @@
     tpe.members[n] = lltype.nullptr(FFI_TYPE_P.TO)
     return tpe
 
+ at specialize.memo()
 def cast_type_to_ffitype(tp):
     """ This function returns ffi representation of rpython type tp
     """
     return TYPE_MAP[tp]
-cast_type_to_ffitype._annspecialcase_ = 'specialize:memo'
 
+ at specialize.argtype(1)
 def push_arg_as_ffiptr(ffitp, arg, ll_buf):
     # This is for primitive types.  Note that the exact type of 'arg' may be
     # different from the expected 'c_size'.  To cope with that, we fall back
@@ -396,7 +397,6 @@
                 arg >>= 8
         else:
             raise AssertionError
-push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)'
 
 
 # type defs for callback and closure userdata
@@ -470,12 +470,12 @@
 FUNCFLAG_USE_ERRNO = 8
 FUNCFLAG_USE_LASTERROR = 16
 
+ at specialize.arg(1)     # hack :-/
 def get_call_conv(flags, from_jit):
     if _WIN32 and not _WIN64 and (flags & FUNCFLAG_CDECL == 0):
         return FFI_STDCALL
     else:
         return FFI_DEFAULT_ABI
-get_call_conv._annspecialcase_ = 'specialize:arg(1)'     # hack :-/
 
 
 class AbstractFuncPtr(object):
@@ -599,6 +599,7 @@
         else:
             self.restype_size = -1
 
+    @specialize.argtype(1)
     def push_arg(self, value):
         #if self.pushed_args == self.argnum:
         #    raise TypeError("Too many arguments, eats %d, pushed %d" %
@@ -618,7 +619,6 @@
         push_arg_as_ffiptr(self.argtypes[self.pushed_args], value,
                            self.ll_args[self.pushed_args])
         self.pushed_args += 1
-    push_arg._annspecialcase_ = 'specialize:argtype(1)'
 
     def _check_args(self):
         if self.pushed_args < self.argnum:
@@ -627,6 +627,7 @@
     def _clean_args(self):
         self.pushed_args = 0
 
+    @specialize.arg(1)
     def call(self, RES_TP):
         self._check_args()
         ffires = c_ffi_call(self.ll_cif, self.funcsym,
@@ -645,7 +646,6 @@
         self._clean_args()
         check_fficall_result(ffires, self.flags)
         return res
-    call._annspecialcase_ = 'specialize:arg(1)'
 
     def __del__(self):
         if self.ll_args:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -280,6 +280,7 @@
 
 
 @oopspec("jit.isconstant(value)")
+ at specialize.call_location()
 def isconstant(value):
     """
     While tracing, returns whether or not the value is currently known to be
@@ -289,9 +290,9 @@
     This is for advanced usage only.
     """
     return NonConstant(False)
-isconstant._annspecialcase_ = "specialize:call_location"
 
 @oopspec("jit.isvirtual(value)")
+ at specialize.call_location()
 def isvirtual(value):
     """
     Returns if this value is virtual, while tracing, it's relatively
@@ -300,7 +301,6 @@
     This is for advanced usage only.
     """
     return NonConstant(False)
-isvirtual._annspecialcase_ = "specialize:call_location"
 
 @specialize.call_location()
 def loop_unrolling_heuristic(lst, size, cutoff=2):
@@ -401,28 +401,27 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Signed, _we_are_jitted)
 
-
+ at oopspec('jit.current_trace_length()')
 def current_trace_length():
     """During JIT tracing, returns the current trace length (as a constant).
     If not tracing, returns -1."""
     if NonConstant(False):
         return 73
     return -1
-current_trace_length.oopspec = 'jit.current_trace_length()'
 
+ at oopspec('jit.debug(string, arg1, arg2, arg3, arg4)')
 def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1,
                       arg3=-sys.maxint-1, arg4=-sys.maxint-1):
     """When JITted, cause an extra operation JIT_DEBUG to appear in
     the graphs.  Should not be left after debugging."""
     keepalive_until_here(string) # otherwise the whole function call is removed
-jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)'
 
+ at oopspec('jit.assert_green(value)')
+ at specialize.argtype(0)
 def assert_green(value):
     """Very strong assert: checks that 'value' is a green
     (a JIT compile-time constant)."""
     keepalive_until_here(value)
-assert_green._annspecialcase_ = 'specialize:argtype(0)'
-assert_green.oopspec = 'jit.assert_green(value)'
 
 class AssertGreenFailed(Exception):
     pass
@@ -457,6 +456,7 @@
 # ____________________________________________________________
 # VRefs
 
+ at oopspec('virtual_ref(x)')
 @specialize.argtype(0)
 def virtual_ref(x):
     """Creates a 'vref' object that contains a reference to 'x'.  Calls
@@ -467,14 +467,13 @@
     dereferenced (by the call syntax 'vref()'), it returns 'x', which is
     then forced."""
     return DirectJitVRef(x)
-virtual_ref.oopspec = 'virtual_ref(x)'
 
+ at oopspec('virtual_ref_finish(x)')
 @specialize.argtype(1)
 def virtual_ref_finish(vref, x):
     """See docstring in virtual_ref(x)"""
     keepalive_until_here(x)   # otherwise the whole function call is removed
     _virtual_ref_finish(vref, x)
-virtual_ref_finish.oopspec = 'virtual_ref_finish(x)'
 
 def non_virtual_ref(x):
     """Creates a 'vref' that just returns x when called; nothing more special.
@@ -831,6 +830,7 @@
     jit_opencoder_model
     """
 
+ at specialize.arg(0)
 def set_user_param(driver, text):
     """Set the tunable JIT parameters from a user-supplied string
     following the format 'param=value,param=value', or 'off' to
@@ -866,7 +866,6 @@
                     break
             else:
                 raise ValueError
-set_user_param._annspecialcase_ = 'specialize:arg(0)'
 
 # ____________________________________________________________
 #
diff --git a/rpython/rlib/listsort.py b/rpython/rlib/listsort.py
--- a/rpython/rlib/listsort.py
+++ b/rpython/rlib/listsort.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rlib.objectmodel import specialize
 
 
 ## ------------------------------------------------------------------------
@@ -141,6 +142,12 @@
         # or, IOW, the first k elements of a should precede key, and the last
         # n-k should follow key.
 
+        # hint for the annotator: the argument 'rightmost' is always passed in as
+        # a constant (either True or False), so we can specialize the function for
+        # the two cases.  (This is actually needed for technical reasons: the
+        # variable 'lower' must contain a known method, which is the case in each
+        # specialized version but not in the unspecialized one.)
+        @specialize.arg(4)
         def gallop(self, key, a, hint, rightmost):
             assert 0 <= hint < a.len
             if rightmost:
@@ -212,12 +219,6 @@
             assert lastofs == ofs         # so a[ofs-1] < key <= a[ofs]
             return ofs
 
-        # hint for the annotator: the argument 'rightmost' is always passed in as
-        # a constant (either True or False), so we can specialize the function for
-        # the two cases.  (This is actually needed for technical reasons: the
-        # variable 'lower' must contain a known method, which is the case in each
-        # specialized version but not in the unspecialized one.)
-        gallop._annspecialcase_ = "specialize:arg(4)"
 
         # ____________________________________________________________
 
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -37,6 +37,7 @@
 from rpython.rlib import objectmodel
 from rpython.flowspace.model import Constant, const
 from rpython.flowspace.specialcase import register_flow_sc
+from rpython.rlib.objectmodel import specialize
 
 """
 Long-term target:
@@ -135,14 +136,15 @@
     # We deal directly with overflow there anyway.
     return r_longlonglong(n)
 
+ at specialize.argtype(0)
 def widen(n):
     from rpython.rtyper.lltypesystem import lltype
     if _should_widen_type(lltype.typeOf(n)):
         return intmask(n)
     else:
         return n
-widen._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def _should_widen_type(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Bool:
@@ -153,19 +155,18 @@
     assert issubclass(r_class, base_int)
     return r_class.BITS < LONG_BIT or (
         r_class.BITS == LONG_BIT and r_class.SIGNED)
-_should_widen_type._annspecialcase_ = 'specialize:memo'
 
 # the replacement for sys.maxint
 maxint = int(LONG_TEST - 1)
 # for now, it should be equal to sys.maxint on all supported platforms
 assert maxint == sys.maxint
 
+ at specialize.argtype(0)
 def is_valid_int(r):
     if objectmodel.we_are_translated():
         return isinstance(r, int)
     return isinstance(r, (base_int, int, long, bool)) and (
         -maxint - 1 <= r <= maxint)
-is_valid_int._annspecialcase_ = 'specialize:argtype(0)'
 
 def ovfcheck(r):
     "NOT_RPYTHON"
@@ -225,12 +226,12 @@
         return build_int(None, self_type.SIGNED, max(self_type.BITS, other_type.BITS))
     raise AssertionError("Merging these types (%s, %s) is not supported" % (self_type, other_type))
 
+ at specialize.memo()
 def signedtype(t):
     if t in (bool, int, long):
         return True
     else:
         return t.SIGNED
-signedtype._annspecialcase_ = 'specialize:memo'
 
 def normalizedinttype(t):
     if t is int:
@@ -241,11 +242,12 @@
         assert t.BITS <= r_longlong.BITS
         return build_int(None, t.SIGNED, r_longlong.BITS)
 
+ at specialize.argtype(0)
 def most_neg_value_of_same_type(x):
     from rpython.rtyper.lltypesystem import lltype
     return most_neg_value_of(lltype.typeOf(x))
-most_neg_value_of_same_type._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def most_neg_value_of(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -256,13 +258,13 @@
         return r_class(-(r_class.MASK >> 1) - 1)
     else:
         return r_class(0)
-most_neg_value_of._annspecialcase_ = 'specialize:memo'
 
+ at specialize.argtype(0)
 def most_pos_value_of_same_type(x):
     from rpython.rtyper.lltypesystem import lltype
     return most_pos_value_of(lltype.typeOf(x))
-most_pos_value_of_same_type._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def most_pos_value_of(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -273,8 +275,8 @@
         return r_class(r_class.MASK >> 1)
     else:
         return r_class(r_class.MASK)
-most_pos_value_of._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def is_signed_integer_type(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -284,7 +286,6 @@
         return r_class.SIGNED
     except KeyError:
         return False   # not an integer type
-is_signed_integer_type._annspecialcase_ = 'specialize:memo'
 
 def highest_bit(n):
     """
@@ -676,7 +677,7 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_mod(lltype.Signed, x, y)
 
- at objectmodel.specialize.ll()
+ at specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
     """
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -90,16 +90,16 @@
 
 FIVEARY_CUTOFF = 8
 
+ at specialize.argtype(0)
 def _mask_digit(x):
     return UDIGIT_MASK(x & MASK)
-_mask_digit._annspecialcase_ = 'specialize:argtype(0)'
 
 def _widen_digit(x):
     return rffi.cast(LONG_TYPE, x)
 
+ at specialize.argtype(0)
 def _store_digit(x):
     return rffi.cast(STORE_TYPE, x)
-_store_digit._annspecialcase_ = 'specialize:argtype(0)'
 
 def _load_unsigned_digit(x):
     return rffi.cast(UNSIGNED_TYPE, x)
@@ -175,11 +175,11 @@
         return _load_unsigned_digit(self._digits[x])
     udigit._always_inline_ = True
 
+    @specialize.argtype(2)
     def setdigit(self, x, val):
         val = _mask_digit(val)
         assert val >= 0
         self._digits[x] = _store_digit(val)
-    setdigit._annspecialcase_ = 'specialize:argtype(2)'
     setdigit._always_inline_ = True
 
     def numdigits(self):
@@ -1312,6 +1312,7 @@
 
     return res
 
+ at specialize.argtype(0)
 def digits_from_nonneg_long(l):
     digits = []
     while True:
@@ -1319,8 +1320,8 @@
         l = l >> SHIFT
         if not l:
             return digits[:] # to make it non-resizable
-digits_from_nonneg_long._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def digits_for_most_neg_long(l):
     # This helper only works if 'l' is the most negative integer of its
     # type, which in base 2 looks like: 1000000..0000
@@ -1335,8 +1336,8 @@
     assert l & MASK == l
     digits.append(_store_digit(l))
     return digits[:] # to make it non-resizable
-digits_for_most_neg_long._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def args_from_rarith_int1(x):
     if x > 0:
         return digits_from_nonneg_long(x), 1
@@ -1348,11 +1349,10 @@
     else:
         # the most negative integer! hacks needed...
         return digits_for_most_neg_long(x), -1
-args_from_rarith_int1._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def args_from_rarith_int(x):
     return args_from_rarith_int1(widen(x))
-args_from_rarith_int._annspecialcase_ = "specialize:argtype(0)"
 # ^^^ specialized by the precise type of 'x', which is typically a r_xxx
 #     instance from rlib.rarithmetic
 
@@ -1909,6 +1909,7 @@
         i += 1
     return borrow
 
+ at specialize.argtype(2)
 def _muladd1(a, n, extra=0):
     """Multiply by a single digit and add a single digit, ignoring the sign.
     """
@@ -1926,7 +1927,7 @@
     z.setdigit(i, carry)
     z._normalize()
     return z
-_muladd1._annspecialcase_ = "specialize:argtype(2)"
+
 def _v_lshift(z, a, m, d):
     """ Shift digit vector a[0:m] d bits left, with 0 <= d < SHIFT. Put
         * result in z[0:m], and return the d bits shifted out of the top.
@@ -2178,6 +2179,7 @@
         ad = -ad
     return ad
 
+ at specialize.arg(0)
 def _loghelper(func, arg):
     """
     A decent logarithm is easy to compute even for huge bigints, but libm can't
@@ -2195,7 +2197,6 @@
     # CAUTION:  e*SHIFT may overflow using int arithmetic,
     # so force use of double. */
     return func(x) + (e * float(SHIFT) * func(2.0))
-_loghelper._annspecialcase_ = 'specialize:arg(0)'
 
 # ____________________________________________________________
 
@@ -2519,6 +2520,7 @@
     return output.build()
 
 
+ at specialize.arg(1)
 def _bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
@@ -2598,8 +2600,8 @@
         return z
 
     return z.invert()
-_bitwise._annspecialcase_ = "specialize:arg(1)"
-
+
+ at specialize.arg(1)
 def _int_bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
@@ -2682,7 +2684,6 @@
         return z
 
     return z.invert()
-_int_bitwise._annspecialcase_ = "specialize:arg(1)"
 
 ULONGLONG_BOUND = r_ulonglong(1L << (r_longlong.BITS-1))
 LONGLONG_MIN = r_longlong(-(1L << (r_longlong.BITS-1)))
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.nonconst import NonConstant
 from rpython.rlib.rarithmetic import intmask
 
@@ -239,12 +239,12 @@
     _, _VirtualProtect_safe = winexternal('VirtualProtect',
                                   [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD],
                                   BOOL)
+    @specialize.ll()
     def VirtualProtect(addr, size, mode, oldmode_ptr):
         return _VirtualProtect_safe(addr,
                                rffi.cast(rffi.SIZE_T, size),
                                rffi.cast(DWORD, mode),
                                oldmode_ptr)
-    VirtualProtect._annspecialcase_ = 'specialize:ll'
     VirtualFree, VirtualFree_safe = winexternal('VirtualFree',
                               [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL)
 
diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -7,6 +7,7 @@
 from struct import unpack
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 from rpython.rlib.rstruct.error import StructError
+from rpython.rlib.objectmodel import specialize
 
 class MasterReader(object):
     def __init__(self, s):
@@ -99,14 +100,14 @@
         self._create_unpacking_func()
         return True
 
+ at specialize.memo()
 def create_unpacker(unpack_str):
     fmtiter = FrozenUnpackIterator(unpack_str)
     fmtiter.interpret(unpack_str)
     assert fmtiter._freeze_()
     return fmtiter
-create_unpacker._annspecialcase_ = 'specialize:memo'
 
+ at specialize.arg(0)
 def runpack(fmt, input):
     unpacker = create_unpacker(fmt)
     return unpacker.unpack(input)
-runpack._annspecialcase_ = 'specialize:arg(0)'

From pypy.commits at gmail.com  Wed Sep 21 16:24:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 13:24:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: Check in
 extradoc/talk/bucharest2016/jit-backend-8vhY1ArTsh.txt in the
Message-ID: <57e2ec88.c336c20a.19b0c.16aa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87292:1e5c1b905a1c
Date: 2016-09-21 22:24 +0200
http://bitbucket.org/pypy/pypy/changeset/1e5c1b905a1c/

Log:	Check in extradoc/talk/bucharest2016/jit-backend-8vhY1ArTsh.txt in
	the docs, as a draft description of our low-level JIT assembler
	backends

diff --git a/rpython/doc/jit/backend.rst b/rpython/doc/jit/backend.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/jit/backend.rst
@@ -0,0 +1,263 @@
+=========================
+PyPy's assembler backends
+=========================
+
+Draft notes about the organization of assembler backends in the PyPy JIT, in 2016
+=================================================================================
+
+
+input: linear sequence of instructions, called a "trace".
+
+A trace is a sequence of instructions in SSA form.  Most instructions
+correspond to one or a few CPU-level instructions.  There are a few
+meta-instructions like `label` and debugging stuff.  All branching is
+done with guards, which are instructions that check that a condition is
+true and exit the trace if not.  A failing guard can have a new trace
+added to it later, called a "bridge".  A patched guard becomes a direct
+`Jcond` instruction going to the bridge, with no indirection, no
+register spilling, etc.
+
+A trace ends with either a `return` or a `jump to label`.  The target
+label is either inside the same trace, or in some older one.  For
+historical reasons we call a "loop" a trace that is not a bridge.  The
+machine code that we generate is organized as a forest of trees; the
+trunk of the tree is a "loop", and the branches are all bridges
+(branching off the trunk or off another branch).
+
+* every trunk or branch that ends in a `jump to label` can target a
+  label from a different tree, too.
+
+* the whole process of assembling a loop or a branch is basically
+  single-threaded, so no synchronization issue there (including to patch
+  older generated instructions).
+
+* the generated assembler has got a "frame" in %rbp, which is actually
+  not on the stack at all, but is a GC object (called a "jitframe").
+  Spilling goes there.
+
+* the guards are `Jcond` to a very small piece of generated code, which
+  is basically pushing a couple of constants on the stack and then
+  jumping to the general guard-recovery code.  That code will save the
+  registers into the jitframe and then exit the whole generated
+  function.  The caller of that generated function checks how it
+  finished: if it finished by hitting a guard, then the caller is
+  responsible for calling the "blackhole interpreter".  This is the part
+  of the front-end that recovers from failing guards and finishes
+  running the frame (including, possibly, by jumping again into
+  generated assembler).
+
+
+Details about the JITting process:
+
+* front-end and optimization pass
+
+* rewrite (includes gc related transformation as well as simplifactions)
+
+* assembler generation
+
+
+Front-end and optimization pass
+-------------------------------
+
+Not discussed here in detail.  This produces loops and bridges using an
+instruction set that is "high-level" in some sense: it contains
+intructions like "new"/"new_array", and
+"setfield"/"setarrayitem"/"setinteriorfield" which describe the action
+of storing a value in a precise field of the structure or array.  For
+example, the "setfield" action might require implicitly a GC write
+barrier.  This is the high-level trace that we send to the following
+step.
+
+
+Rewrite
+-------
+
+A mostly but not completely CPU-independent phase: lowers some
+instructions.  For example, the variants of "new" are lowered to
+"malloc" and a few "gc_store": it bumps the pointer of the GC and then
+sets a few fields explicitly in the newly allocated structure.  The
+"setfield" is replaced with a "cond_gc_wb_call" (conditional call to the
+write barrier) if needed, followed by a "gc_store".
+
+The "gc_store" instruction can be encoded in a single MOV assembler
+instruction, but is not as flexible as a MOV.  The address is always
+specified as "some GC pointer + an offset".  We don't have the notion of
+interior pointer for GC objects.
+
+A different instruction, "gc_store_indexed", offers additional operands,
+which can be mapped to a single MOV instruction using forms like
+`[rax+8*rcx+24]`.
+
+Some other complex instructions pass through to the backend, which must
+deal with them: for example, "card marking" in the GC.  (Writing an
+object pointer inside an array would require walking the whole array
+later to find "young" references. Instead of that, we flip a bit for
+every range of 128 entries.  This is a common GC optimization.)  Setting
+the card bit of a GC object requires a sequence of assembler
+instructions that depends too much on the target CPU to be expressed
+explicitly here (moreover, it contains a few branches, which are hard to
+express at this level).
+
+
+Assembly
+--------
+
+No fancy code generation technique, but greedy forward pass that tries
+to avoid some pitfalls
+
+
+Handling instructions
+~~~~~~~~~~~~~~~~~~~~~
+
+* One by one (forward direction).   Each instruction asks the register
+  allocator to ensure that some arguments are in registers (not in the
+  jitframe); asks for a register to put its result into; and asks for
+  additional scratch registers that will be freed at  the end of the
+  instruction.  There is a special case for boolean variables: they are
+  stored in the condition code flags instead of being materialized as a
+  0/1 value.  (They are materialized later, except in the common case
+  where they are only used by the next `guard_false` or `guard_true` and
+  then forgotten.)
+
+* Instruction arguments are loaded into a register on demand.  This
+  makes the backend quite easy to write, but leads do some bad
+  decisions.
+
+
+Linear scan register allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Although it's always a linear trace that we consider, we don't use
+advanced techniques for register allocation: we do forward, on-demand
+allocation as the backend produces the assembler.  When it asks for a
+register to put some value into, we give it any free register, without
+consideration for what will be done with it later.  We compute the
+longevity of all variables, but only use it when choosing which register
+to spill (we spill the variable with the longest longevity).
+
+This works to some extend because it is well integrated with the earlier
+optimization pass. Loops are unrolled once by the optimization pass to
+allow more powerful optimizations---the optimization pass itself is the
+place that benefits the most, but it also has benefits here in the
+assembly pass.  These are:
+
+* The first peeling initializes the register binding on the first use.
+
+* This leads to an already allocated register of the trace loop.
+
+* As well as allocated registers when exiting bridges
+
+[Try to better allocate registers to match the ABI (minor to non benefit
+in the current state)]
+
+
+More complex mappings
+~~~~~~~~~~~~~~~~~~~~~
+
+Some instructions generate more complex code.  These are either or both of:
+
+* complex instructions generating some local control flow, like
+  "cond_gc_wb_call" (for write barriers), "call_assembler" (a call
+  followed by a few checks).
+
+* instructions that invoke custom assembler helpers, like the slow-path
+  of write barriers or the slow-path of allocations.  These slow-paths
+  are typically generated too, so that we are not constrained by the
+  usual calling conventions.
+
+
+GC pointers
+~~~~~~~~~~~
+
+Around most CALL instructions, we need to record a description of where
+the GC pointers are (registers and stack frame).  This is needed in case
+the CALL invokes a garbage collection.  The GC pointers can move; the
+pointers in the registers and stack frame are updated by the GC.  That's
+a reason for why we don't have explicit interior pointers.
+
+GC pointers can appear as constants in the trace.  We are busy changing
+that to use a constant table and `MOV REG, (%RIP+offset)`.  The
+"constant" in the table is actually updated by the GC if the object
+move.
+
+
+Vectorization
+~~~~~~~~~~~~~
+
+Optimization developed to use SIMD instructions for trace loops. Primary
+idea was to use it as an optimization of micro numpy. It has several
+passes on the already optimized trace.
+
+Shortly explained: It builds dependencies for an unrolled trace loop,
+gathering pairs/packs of operations that could be executed in parallel
+and finally schedules the operations.
+
+What did it add to the code base:
+
+* Dependencies can be constructed
+
+* Code motion of guards to relax dependencies
+
+* Scheduler to reorder trace
+
+* Array bound check removal (especially for unrolled traces)
+
+What can it do:
+
+* Transform vector loops (element wise operations)
+
+* Accumulation (`reduce([...],operator,0)`). Requires Operation to be
+  associative and commutative
+
+* SSE 4.1 as "vector backend"
+
+
+We do not
+~~~~~~~~~
+
+* Keep tracing data around to reoptimize the trace tree. (Once a trace
+  is compiled, minimal data is kept.)  This is one reason (there are
+  others in the front-end) for the following result: JIT-compiling a
+  small loop with two common paths ends up as one "loop" and one bridge
+  assembled, and the bridge-following path is slightly less efficient.
+  This is notably because this bridge is assembled with two constraints:
+  the input registers are fixed (from the guard), and the output
+  registers are fixed (from the jump target); usually these two sets of
+  fixed registers are different, and copying around is needed.
+
+* We don't join trace tails: we only assemble *trees*.
+
+* We don't do any reordering (neither of trace instructions nor of
+  individual assembler instructions)
+
+* We don't do any cross-instruction optimization that makes sense only
+  for the backend and can't easily be expressed at a higher level.  I'm
+  sure there are tons of examples of that, but e.g. loading a large
+  constant in a register that will survive for several instructions;
+  moving out of loops *parts* of some instruction like the address
+  calculation; etc. etc.
+
+* Other optimization opportunities I can think about: look at the
+  function prologue/epilogue; look at the overhead (small but not zero)
+  at the start of a bridge.  Also check if the way guards are
+  implemented makes sense.  Also, we generate large-ish sequences of
+  assembler instructions with tons of `Jcond` that are almost never
+  followed; any optimization opportunity there?  (They all go forward,
+  if it changes anything.)  In theory we could also replace some of
+  these with a signal handler on segfault (e.g. `guard_nonnull_class`).
+
+
+a GCC or LLVM backend?
+~~~~~~~~~~~~~~~~~~~~~~
+
+At least for comparison we'd like a JIT backend that emits its code
+using GCC or LLVM (irrespective of the time it would take).  But it's
+hard to map reasonably well the guards to the C language or to LLVM IR.
+The problems are: (1) we have many guards, we would like to avoid having
+many paths that each do a full
+saving-all-local-variables-that-are-still-alive; (2) it's hard to patch
+a guard when a bridge is compiled from it; (3) instructions like a CALL
+need to expose the local variables that are GC pointers; CALL_MAY_FORCE
+need to expose *all* local variables for optional off-line
+reconstruction of the interpreter state.
+
diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst
--- a/rpython/doc/jit/index.rst
+++ b/rpython/doc/jit/index.rst
@@ -26,6 +26,7 @@
    optimizer
    virtualizable
    vectorization
+   backend
 
 - :doc:`Overview <overview>`: motivating our approach
 
@@ -34,5 +35,8 @@
 - :doc:`Optimizer <optimizer>`: the step between tracing and writing
   machine code
 
-- :doc:`Virtulizable <virtualizable>` how virtualizables work and what they are
-  (in other words how to make frames more efficient).
+- :doc:`Virtualizable <virtualizable>`: how virtualizables work and what
+  they are (in other words how to make frames more efficient).
+
+- :doc:`Assembler backend <backend>`: draft notes about the organization
+  of the assembler backends

From pypy.commits at gmail.com  Wed Sep 21 16:33:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 13:33:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57e2eea1.898b1c0a.b0412.c0a8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87293:f854ee56f616
Date: 2016-09-21 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/f854ee56f616/

Log:	hg merge default

diff --git a/rpython/doc/jit/backend.rst b/rpython/doc/jit/backend.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/jit/backend.rst
@@ -0,0 +1,263 @@
+=========================
+PyPy's assembler backends
+=========================
+
+Draft notes about the organization of assembler backends in the PyPy JIT, in 2016
+=================================================================================
+
+
+input: linear sequence of instructions, called a "trace".
+
+A trace is a sequence of instructions in SSA form.  Most instructions
+correspond to one or a few CPU-level instructions.  There are a few
+meta-instructions like `label` and debugging stuff.  All branching is
+done with guards, which are instructions that check that a condition is
+true and exit the trace if not.  A failing guard can have a new trace
+added to it later, called a "bridge".  A patched guard becomes a direct
+`Jcond` instruction going to the bridge, with no indirection, no
+register spilling, etc.
+
+A trace ends with either a `return` or a `jump to label`.  The target
+label is either inside the same trace, or in some older one.  For
+historical reasons we call a "loop" a trace that is not a bridge.  The
+machine code that we generate is organized as a forest of trees; the
+trunk of the tree is a "loop", and the branches are all bridges
+(branching off the trunk or off another branch).
+
+* every trunk or branch that ends in a `jump to label` can target a
+  label from a different tree, too.
+
+* the whole process of assembling a loop or a branch is basically
+  single-threaded, so no synchronization issue there (including to patch
+  older generated instructions).
+
+* the generated assembler has got a "frame" in %rbp, which is actually
+  not on the stack at all, but is a GC object (called a "jitframe").
+  Spilling goes there.
+
+* the guards are `Jcond` to a very small piece of generated code, which
+  is basically pushing a couple of constants on the stack and then
+  jumping to the general guard-recovery code.  That code will save the
+  registers into the jitframe and then exit the whole generated
+  function.  The caller of that generated function checks how it
+  finished: if it finished by hitting a guard, then the caller is
+  responsible for calling the "blackhole interpreter".  This is the part
+  of the front-end that recovers from failing guards and finishes
+  running the frame (including, possibly, by jumping again into
+  generated assembler).
+
+
+Details about the JITting process:
+
+* front-end and optimization pass
+
+* rewrite (includes gc related transformation as well as simplifactions)
+
+* assembler generation
+
+
+Front-end and optimization pass
+-------------------------------
+
+Not discussed here in detail.  This produces loops and bridges using an
+instruction set that is "high-level" in some sense: it contains
+intructions like "new"/"new_array", and
+"setfield"/"setarrayitem"/"setinteriorfield" which describe the action
+of storing a value in a precise field of the structure or array.  For
+example, the "setfield" action might require implicitly a GC write
+barrier.  This is the high-level trace that we send to the following
+step.
+
+
+Rewrite
+-------
+
+A mostly but not completely CPU-independent phase: lowers some
+instructions.  For example, the variants of "new" are lowered to
+"malloc" and a few "gc_store": it bumps the pointer of the GC and then
+sets a few fields explicitly in the newly allocated structure.  The
+"setfield" is replaced with a "cond_gc_wb_call" (conditional call to the
+write barrier) if needed, followed by a "gc_store".
+
+The "gc_store" instruction can be encoded in a single MOV assembler
+instruction, but is not as flexible as a MOV.  The address is always
+specified as "some GC pointer + an offset".  We don't have the notion of
+interior pointer for GC objects.
+
+A different instruction, "gc_store_indexed", offers additional operands,
+which can be mapped to a single MOV instruction using forms like
+`[rax+8*rcx+24]`.
+
+Some other complex instructions pass through to the backend, which must
+deal with them: for example, "card marking" in the GC.  (Writing an
+object pointer inside an array would require walking the whole array
+later to find "young" references. Instead of that, we flip a bit for
+every range of 128 entries.  This is a common GC optimization.)  Setting
+the card bit of a GC object requires a sequence of assembler
+instructions that depends too much on the target CPU to be expressed
+explicitly here (moreover, it contains a few branches, which are hard to
+express at this level).
+
+
+Assembly
+--------
+
+No fancy code generation technique, but greedy forward pass that tries
+to avoid some pitfalls
+
+
+Handling instructions
+~~~~~~~~~~~~~~~~~~~~~
+
+* One by one (forward direction).   Each instruction asks the register
+  allocator to ensure that some arguments are in registers (not in the
+  jitframe); asks for a register to put its result into; and asks for
+  additional scratch registers that will be freed at  the end of the
+  instruction.  There is a special case for boolean variables: they are
+  stored in the condition code flags instead of being materialized as a
+  0/1 value.  (They are materialized later, except in the common case
+  where they are only used by the next `guard_false` or `guard_true` and
+  then forgotten.)
+
+* Instruction arguments are loaded into a register on demand.  This
+  makes the backend quite easy to write, but leads do some bad
+  decisions.
+
+
+Linear scan register allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Although it's always a linear trace that we consider, we don't use
+advanced techniques for register allocation: we do forward, on-demand
+allocation as the backend produces the assembler.  When it asks for a
+register to put some value into, we give it any free register, without
+consideration for what will be done with it later.  We compute the
+longevity of all variables, but only use it when choosing which register
+to spill (we spill the variable with the longest longevity).
+
+This works to some extend because it is well integrated with the earlier
+optimization pass. Loops are unrolled once by the optimization pass to
+allow more powerful optimizations---the optimization pass itself is the
+place that benefits the most, but it also has benefits here in the
+assembly pass.  These are:
+
+* The first peeling initializes the register binding on the first use.
+
+* This leads to an already allocated register of the trace loop.
+
+* As well as allocated registers when exiting bridges
+
+[Try to better allocate registers to match the ABI (minor to non benefit
+in the current state)]
+
+
+More complex mappings
+~~~~~~~~~~~~~~~~~~~~~
+
+Some instructions generate more complex code.  These are either or both of:
+
+* complex instructions generating some local control flow, like
+  "cond_gc_wb_call" (for write barriers), "call_assembler" (a call
+  followed by a few checks).
+
+* instructions that invoke custom assembler helpers, like the slow-path
+  of write barriers or the slow-path of allocations.  These slow-paths
+  are typically generated too, so that we are not constrained by the
+  usual calling conventions.
+
+
+GC pointers
+~~~~~~~~~~~
+
+Around most CALL instructions, we need to record a description of where
+the GC pointers are (registers and stack frame).  This is needed in case
+the CALL invokes a garbage collection.  The GC pointers can move; the
+pointers in the registers and stack frame are updated by the GC.  That's
+a reason for why we don't have explicit interior pointers.
+
+GC pointers can appear as constants in the trace.  We are busy changing
+that to use a constant table and `MOV REG, (%RIP+offset)`.  The
+"constant" in the table is actually updated by the GC if the object
+move.
+
+
+Vectorization
+~~~~~~~~~~~~~
+
+Optimization developed to use SIMD instructions for trace loops. Primary
+idea was to use it as an optimization of micro numpy. It has several
+passes on the already optimized trace.
+
+Shortly explained: It builds dependencies for an unrolled trace loop,
+gathering pairs/packs of operations that could be executed in parallel
+and finally schedules the operations.
+
+What did it add to the code base:
+
+* Dependencies can be constructed
+
+* Code motion of guards to relax dependencies
+
+* Scheduler to reorder trace
+
+* Array bound check removal (especially for unrolled traces)
+
+What can it do:
+
+* Transform vector loops (element wise operations)
+
+* Accumulation (`reduce([...],operator,0)`). Requires Operation to be
+  associative and commutative
+
+* SSE 4.1 as "vector backend"
+
+
+We do not
+~~~~~~~~~
+
+* Keep tracing data around to reoptimize the trace tree. (Once a trace
+  is compiled, minimal data is kept.)  This is one reason (there are
+  others in the front-end) for the following result: JIT-compiling a
+  small loop with two common paths ends up as one "loop" and one bridge
+  assembled, and the bridge-following path is slightly less efficient.
+  This is notably because this bridge is assembled with two constraints:
+  the input registers are fixed (from the guard), and the output
+  registers are fixed (from the jump target); usually these two sets of
+  fixed registers are different, and copying around is needed.
+
+* We don't join trace tails: we only assemble *trees*.
+
+* We don't do any reordering (neither of trace instructions nor of
+  individual assembler instructions)
+
+* We don't do any cross-instruction optimization that makes sense only
+  for the backend and can't easily be expressed at a higher level.  I'm
+  sure there are tons of examples of that, but e.g. loading a large
+  constant in a register that will survive for several instructions;
+  moving out of loops *parts* of some instruction like the address
+  calculation; etc. etc.
+
+* Other optimization opportunities I can think about: look at the
+  function prologue/epilogue; look at the overhead (small but not zero)
+  at the start of a bridge.  Also check if the way guards are
+  implemented makes sense.  Also, we generate large-ish sequences of
+  assembler instructions with tons of `Jcond` that are almost never
+  followed; any optimization opportunity there?  (They all go forward,
+  if it changes anything.)  In theory we could also replace some of
+  these with a signal handler on segfault (e.g. `guard_nonnull_class`).
+
+
+a GCC or LLVM backend?
+~~~~~~~~~~~~~~~~~~~~~~
+
+At least for comparison we'd like a JIT backend that emits its code
+using GCC or LLVM (irrespective of the time it would take).  But it's
+hard to map reasonably well the guards to the C language or to LLVM IR.
+The problems are: (1) we have many guards, we would like to avoid having
+many paths that each do a full
+saving-all-local-variables-that-are-still-alive; (2) it's hard to patch
+a guard when a bridge is compiled from it; (3) instructions like a CALL
+need to expose the local variables that are GC pointers; CALL_MAY_FORCE
+need to expose *all* local variables for optional off-line
+reconstruction of the interpreter state.
+
diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst
--- a/rpython/doc/jit/index.rst
+++ b/rpython/doc/jit/index.rst
@@ -26,6 +26,7 @@
    optimizer
    virtualizable
    vectorization
+   backend
 
 - :doc:`Overview <overview>`: motivating our approach
 
@@ -34,5 +35,8 @@
 - :doc:`Optimizer <optimizer>`: the step between tracing and writing
   machine code
 
-- :doc:`Virtulizable <virtualizable>` how virtualizables work and what they are
-  (in other words how to make frames more efficient).
+- :doc:`Virtualizable <virtualizable>`: how virtualizables work and what
+  they are (in other words how to make frames more efficient).
+
+- :doc:`Assembler backend <backend>`: draft notes about the organization
+  of the assembler backends
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -710,6 +710,12 @@
 
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
+
+    def filter(self, builder):
+        if not builder.cpu.supports_cond_call_value and \
+           self.opnum == rop.COND_CALL_VALUE_I:
+            raise CannotProduceOperation
+
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.test.test_random import check_random_function, Random
 from rpython.jit.backend.test.test_ll_random import LLtypeOperationBuilder
 from rpython.jit.backend.detect_cpu import getcpuclass
+from rpython.jit.metainterp.resoperation import rop
 import platform
 
 CPU = getcpuclass()
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -359,12 +359,13 @@
     tpe.members[n] = lltype.nullptr(FFI_TYPE_P.TO)
     return tpe
 
+ at specialize.memo()
 def cast_type_to_ffitype(tp):
     """ This function returns ffi representation of rpython type tp
     """
     return TYPE_MAP[tp]
-cast_type_to_ffitype._annspecialcase_ = 'specialize:memo'
 
+ at specialize.argtype(1)
 def push_arg_as_ffiptr(ffitp, arg, ll_buf):
     # This is for primitive types.  Note that the exact type of 'arg' may be
     # different from the expected 'c_size'.  To cope with that, we fall back
@@ -396,7 +397,6 @@
                 arg >>= 8
         else:
             raise AssertionError
-push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)'
 
 
 # type defs for callback and closure userdata
@@ -470,12 +470,12 @@
 FUNCFLAG_USE_ERRNO = 8
 FUNCFLAG_USE_LASTERROR = 16
 
+ at specialize.arg(1)     # hack :-/
 def get_call_conv(flags, from_jit):
     if _WIN32 and not _WIN64 and (flags & FUNCFLAG_CDECL == 0):
         return FFI_STDCALL
     else:
         return FFI_DEFAULT_ABI
-get_call_conv._annspecialcase_ = 'specialize:arg(1)'     # hack :-/
 
 
 class AbstractFuncPtr(object):
@@ -599,6 +599,7 @@
         else:
             self.restype_size = -1
 
+    @specialize.argtype(1)
     def push_arg(self, value):
         #if self.pushed_args == self.argnum:
         #    raise TypeError("Too many arguments, eats %d, pushed %d" %
@@ -618,7 +619,6 @@
         push_arg_as_ffiptr(self.argtypes[self.pushed_args], value,
                            self.ll_args[self.pushed_args])
         self.pushed_args += 1
-    push_arg._annspecialcase_ = 'specialize:argtype(1)'
 
     def _check_args(self):
         if self.pushed_args < self.argnum:
@@ -627,6 +627,7 @@
     def _clean_args(self):
         self.pushed_args = 0
 
+    @specialize.arg(1)
     def call(self, RES_TP):
         self._check_args()
         ffires = c_ffi_call(self.ll_cif, self.funcsym,
@@ -645,7 +646,6 @@
         self._clean_args()
         check_fficall_result(ffires, self.flags)
         return res
-    call._annspecialcase_ = 'specialize:arg(1)'
 
     def __del__(self):
         if self.ll_args:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -280,6 +280,7 @@
 
 
 @oopspec("jit.isconstant(value)")
+ at specialize.call_location()
 def isconstant(value):
     """
     While tracing, returns whether or not the value is currently known to be
@@ -289,9 +290,9 @@
     This is for advanced usage only.
     """
     return NonConstant(False)
-isconstant._annspecialcase_ = "specialize:call_location"
 
 @oopspec("jit.isvirtual(value)")
+ at specialize.call_location()
 def isvirtual(value):
     """
     Returns if this value is virtual, while tracing, it's relatively
@@ -300,7 +301,6 @@
     This is for advanced usage only.
     """
     return NonConstant(False)
-isvirtual._annspecialcase_ = "specialize:call_location"
 
 @specialize.call_location()
 def loop_unrolling_heuristic(lst, size, cutoff=2):
@@ -401,28 +401,27 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Signed, _we_are_jitted)
 
-
+ at oopspec('jit.current_trace_length()')
 def current_trace_length():
     """During JIT tracing, returns the current trace length (as a constant).
     If not tracing, returns -1."""
     if NonConstant(False):
         return 73
     return -1
-current_trace_length.oopspec = 'jit.current_trace_length()'
 
+ at oopspec('jit.debug(string, arg1, arg2, arg3, arg4)')
 def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1,
                       arg3=-sys.maxint-1, arg4=-sys.maxint-1):
     """When JITted, cause an extra operation JIT_DEBUG to appear in
     the graphs.  Should not be left after debugging."""
     keepalive_until_here(string) # otherwise the whole function call is removed
-jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)'
 
+ at oopspec('jit.assert_green(value)')
+ at specialize.argtype(0)
 def assert_green(value):
     """Very strong assert: checks that 'value' is a green
     (a JIT compile-time constant)."""
     keepalive_until_here(value)
-assert_green._annspecialcase_ = 'specialize:argtype(0)'
-assert_green.oopspec = 'jit.assert_green(value)'
 
 class AssertGreenFailed(Exception):
     pass
@@ -457,6 +456,7 @@
 # ____________________________________________________________
 # VRefs
 
+ at oopspec('virtual_ref(x)')
 @specialize.argtype(0)
 def virtual_ref(x):
     """Creates a 'vref' object that contains a reference to 'x'.  Calls
@@ -467,14 +467,13 @@
     dereferenced (by the call syntax 'vref()'), it returns 'x', which is
     then forced."""
     return DirectJitVRef(x)
-virtual_ref.oopspec = 'virtual_ref(x)'
 
+ at oopspec('virtual_ref_finish(x)')
 @specialize.argtype(1)
 def virtual_ref_finish(vref, x):
     """See docstring in virtual_ref(x)"""
     keepalive_until_here(x)   # otherwise the whole function call is removed
     _virtual_ref_finish(vref, x)
-virtual_ref_finish.oopspec = 'virtual_ref_finish(x)'
 
 def non_virtual_ref(x):
     """Creates a 'vref' that just returns x when called; nothing more special.
@@ -831,6 +830,7 @@
     jit_opencoder_model
     """
 
+ at specialize.arg(0)
 def set_user_param(driver, text):
     """Set the tunable JIT parameters from a user-supplied string
     following the format 'param=value,param=value', or 'off' to
@@ -866,7 +866,6 @@
                     break
             else:
                 raise ValueError
-set_user_param._annspecialcase_ = 'specialize:arg(0)'
 
 # ____________________________________________________________
 #
diff --git a/rpython/rlib/listsort.py b/rpython/rlib/listsort.py
--- a/rpython/rlib/listsort.py
+++ b/rpython/rlib/listsort.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rlib.objectmodel import specialize
 
 
 ## ------------------------------------------------------------------------
@@ -141,6 +142,12 @@
         # or, IOW, the first k elements of a should precede key, and the last
         # n-k should follow key.
 
+        # hint for the annotator: the argument 'rightmost' is always passed in as
+        # a constant (either True or False), so we can specialize the function for
+        # the two cases.  (This is actually needed for technical reasons: the
+        # variable 'lower' must contain a known method, which is the case in each
+        # specialized version but not in the unspecialized one.)
+        @specialize.arg(4)
         def gallop(self, key, a, hint, rightmost):
             assert 0 <= hint < a.len
             if rightmost:
@@ -212,12 +219,6 @@
             assert lastofs == ofs         # so a[ofs-1] < key <= a[ofs]
             return ofs
 
-        # hint for the annotator: the argument 'rightmost' is always passed in as
-        # a constant (either True or False), so we can specialize the function for
-        # the two cases.  (This is actually needed for technical reasons: the
-        # variable 'lower' must contain a known method, which is the case in each
-        # specialized version but not in the unspecialized one.)
-        gallop._annspecialcase_ = "specialize:arg(4)"
 
         # ____________________________________________________________
 
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -37,6 +37,7 @@
 from rpython.rlib import objectmodel
 from rpython.flowspace.model import Constant, const
 from rpython.flowspace.specialcase import register_flow_sc
+from rpython.rlib.objectmodel import specialize
 
 """
 Long-term target:
@@ -135,14 +136,15 @@
     # We deal directly with overflow there anyway.
     return r_longlonglong(n)
 
+ at specialize.argtype(0)
 def widen(n):
     from rpython.rtyper.lltypesystem import lltype
     if _should_widen_type(lltype.typeOf(n)):
         return intmask(n)
     else:
         return n
-widen._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def _should_widen_type(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Bool:
@@ -153,19 +155,18 @@
     assert issubclass(r_class, base_int)
     return r_class.BITS < LONG_BIT or (
         r_class.BITS == LONG_BIT and r_class.SIGNED)
-_should_widen_type._annspecialcase_ = 'specialize:memo'
 
 # the replacement for sys.maxint
 maxint = int(LONG_TEST - 1)
 # for now, it should be equal to sys.maxint on all supported platforms
 assert maxint == sys.maxint
 
+ at specialize.argtype(0)
 def is_valid_int(r):
     if objectmodel.we_are_translated():
         return isinstance(r, int)
     return isinstance(r, (base_int, int, long, bool)) and (
         -maxint - 1 <= r <= maxint)
-is_valid_int._annspecialcase_ = 'specialize:argtype(0)'
 
 def ovfcheck(r):
     "NOT_RPYTHON"
@@ -225,12 +226,12 @@
         return build_int(None, self_type.SIGNED, max(self_type.BITS, other_type.BITS))
     raise AssertionError("Merging these types (%s, %s) is not supported" % (self_type, other_type))
 
+ at specialize.memo()
 def signedtype(t):
     if t in (bool, int, long):
         return True
     else:
         return t.SIGNED
-signedtype._annspecialcase_ = 'specialize:memo'
 
 def normalizedinttype(t):
     if t is int:
@@ -241,11 +242,12 @@
         assert t.BITS <= r_longlong.BITS
         return build_int(None, t.SIGNED, r_longlong.BITS)
 
+ at specialize.argtype(0)
 def most_neg_value_of_same_type(x):
     from rpython.rtyper.lltypesystem import lltype
     return most_neg_value_of(lltype.typeOf(x))
-most_neg_value_of_same_type._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def most_neg_value_of(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -256,13 +258,13 @@
         return r_class(-(r_class.MASK >> 1) - 1)
     else:
         return r_class(0)
-most_neg_value_of._annspecialcase_ = 'specialize:memo'
 
+ at specialize.argtype(0)
 def most_pos_value_of_same_type(x):
     from rpython.rtyper.lltypesystem import lltype
     return most_pos_value_of(lltype.typeOf(x))
-most_pos_value_of_same_type._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def most_pos_value_of(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -273,8 +275,8 @@
         return r_class(r_class.MASK >> 1)
     else:
         return r_class(r_class.MASK)
-most_pos_value_of._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def is_signed_integer_type(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -284,7 +286,6 @@
         return r_class.SIGNED
     except KeyError:
         return False   # not an integer type
-is_signed_integer_type._annspecialcase_ = 'specialize:memo'
 
 def highest_bit(n):
     """
@@ -676,7 +677,7 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_mod(lltype.Signed, x, y)
 
- at objectmodel.specialize.ll()
+ at specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
     """
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -90,16 +90,16 @@
 
 FIVEARY_CUTOFF = 8
 
+ at specialize.argtype(0)
 def _mask_digit(x):
     return UDIGIT_MASK(x & MASK)
-_mask_digit._annspecialcase_ = 'specialize:argtype(0)'
 
 def _widen_digit(x):
     return rffi.cast(LONG_TYPE, x)
 
+ at specialize.argtype(0)
 def _store_digit(x):
     return rffi.cast(STORE_TYPE, x)
-_store_digit._annspecialcase_ = 'specialize:argtype(0)'
 
 def _load_unsigned_digit(x):
     return rffi.cast(UNSIGNED_TYPE, x)
@@ -175,11 +175,11 @@
         return _load_unsigned_digit(self._digits[x])
     udigit._always_inline_ = True
 
+    @specialize.argtype(2)
     def setdigit(self, x, val):
         val = _mask_digit(val)
         assert val >= 0
         self._digits[x] = _store_digit(val)
-    setdigit._annspecialcase_ = 'specialize:argtype(2)'
     setdigit._always_inline_ = True
 
     def numdigits(self):
@@ -1312,6 +1312,7 @@
 
     return res
 
+ at specialize.argtype(0)
 def digits_from_nonneg_long(l):
     digits = []
     while True:
@@ -1319,8 +1320,8 @@
         l = l >> SHIFT
         if not l:
             return digits[:] # to make it non-resizable
-digits_from_nonneg_long._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def digits_for_most_neg_long(l):
     # This helper only works if 'l' is the most negative integer of its
     # type, which in base 2 looks like: 1000000..0000
@@ -1335,8 +1336,8 @@
     assert l & MASK == l
     digits.append(_store_digit(l))
     return digits[:] # to make it non-resizable
-digits_for_most_neg_long._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def args_from_rarith_int1(x):
     if x > 0:
         return digits_from_nonneg_long(x), 1
@@ -1348,11 +1349,10 @@
     else:
         # the most negative integer! hacks needed...
         return digits_for_most_neg_long(x), -1
-args_from_rarith_int1._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def args_from_rarith_int(x):
     return args_from_rarith_int1(widen(x))
-args_from_rarith_int._annspecialcase_ = "specialize:argtype(0)"
 # ^^^ specialized by the precise type of 'x', which is typically a r_xxx
 #     instance from rlib.rarithmetic
 
@@ -1909,6 +1909,7 @@
         i += 1
     return borrow
 
+ at specialize.argtype(2)
 def _muladd1(a, n, extra=0):
     """Multiply by a single digit and add a single digit, ignoring the sign.
     """
@@ -1926,7 +1927,7 @@
     z.setdigit(i, carry)
     z._normalize()
     return z
-_muladd1._annspecialcase_ = "specialize:argtype(2)"
+
 def _v_lshift(z, a, m, d):
     """ Shift digit vector a[0:m] d bits left, with 0 <= d < SHIFT. Put
         * result in z[0:m], and return the d bits shifted out of the top.
@@ -2178,6 +2179,7 @@
         ad = -ad
     return ad
 
+ at specialize.arg(0)
 def _loghelper(func, arg):
     """
     A decent logarithm is easy to compute even for huge bigints, but libm can't
@@ -2195,7 +2197,6 @@
     # CAUTION:  e*SHIFT may overflow using int arithmetic,
     # so force use of double. */
     return func(x) + (e * float(SHIFT) * func(2.0))
-_loghelper._annspecialcase_ = 'specialize:arg(0)'
 
 # ____________________________________________________________
 
@@ -2519,6 +2520,7 @@
     return output.build()
 
 
+ at specialize.arg(1)
 def _bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
@@ -2598,8 +2600,8 @@
         return z
 
     return z.invert()
-_bitwise._annspecialcase_ = "specialize:arg(1)"
-
+
+ at specialize.arg(1)
 def _int_bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
@@ -2682,7 +2684,6 @@
         return z
 
     return z.invert()
-_int_bitwise._annspecialcase_ = "specialize:arg(1)"
 
 ULONGLONG_BOUND = r_ulonglong(1L << (r_longlong.BITS-1))
 LONGLONG_MIN = r_longlong(-(1L << (r_longlong.BITS-1)))
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.nonconst import NonConstant
 from rpython.rlib.rarithmetic import intmask
 
@@ -239,12 +239,12 @@
     _, _VirtualProtect_safe = winexternal('VirtualProtect',
                                   [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD],
                                   BOOL)
+    @specialize.ll()
     def VirtualProtect(addr, size, mode, oldmode_ptr):
         return _VirtualProtect_safe(addr,
                                rffi.cast(rffi.SIZE_T, size),
                                rffi.cast(DWORD, mode),
                                oldmode_ptr)
-    VirtualProtect._annspecialcase_ = 'specialize:ll'
     VirtualFree, VirtualFree_safe = winexternal('VirtualFree',
                               [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL)
 
diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -7,6 +7,7 @@
 from struct import unpack
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 from rpython.rlib.rstruct.error import StructError
+from rpython.rlib.objectmodel import specialize
 
 class MasterReader(object):
     def __init__(self, s):
@@ -99,14 +100,14 @@
         self._create_unpacking_func()
         return True
 
+ at specialize.memo()
 def create_unpacker(unpack_str):
     fmtiter = FrozenUnpackIterator(unpack_str)
     fmtiter.interpret(unpack_str)
     assert fmtiter._freeze_()
     return fmtiter
-create_unpacker._annspecialcase_ = 'specialize:memo'
 
+ at specialize.arg(0)
 def runpack(fmt, input):
     unpacker = create_unpacker(fmt)
     return unpacker.unpack(input)
-runpack._annspecialcase_ = 'specialize:arg(0)'

From pypy.commits at gmail.com  Wed Sep 21 16:34:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 13:34:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: Remove again unused import
Message-ID: <57e2eedb.29a9c20a.149f9.09eb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87294:b96f8757abef
Date: 2016-09-21 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/b96f8757abef/

Log:	Remove again unused import

diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -1,7 +1,6 @@
 from rpython.jit.backend.test.test_random import check_random_function, Random
 from rpython.jit.backend.test.test_ll_random import LLtypeOperationBuilder
 from rpython.jit.backend.detect_cpu import getcpuclass
-from rpython.jit.metainterp.resoperation import rop
 import platform
 
 CPU = getcpuclass()

From pypy.commits at gmail.com  Wed Sep 21 18:26:55 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 21 Sep 2016 15:26:55 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Add some missing
 _immutable_fields_ annotations for BuiltinCode objects
Message-ID: <57e3092f.081dc20a.c35a6.339b@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87295:659c9d93b42d
Date: 2016-09-21 18:26 -0400
http://bitbucket.org/pypy/pypy/changeset/659c9d93b42d/

Log:	Add some missing _immutable_fields_ annotations for BuiltinCode
	objects

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -786,6 +786,8 @@
 class BuiltinCode1(BuiltinCode):
     fast_natural_arity = 1
 
+    _immutable_fields_ = ['fastfunc_1']
+
     def fastcall_1(self, space, w_func, w1):
         try:
             w_result = self.fastfunc_1(space, w1)
@@ -805,6 +807,8 @@
 class BuiltinCode2(BuiltinCode):
     fast_natural_arity = 2
 
+    _immutable_fields_ = ['fastfunc_2']
+
     def fastcall_2(self, space, w_func, w1, w2):
         try:
             w_result = self.fastfunc_2(space, w1, w2)
@@ -824,6 +828,8 @@
 class BuiltinCode3(BuiltinCode):
     fast_natural_arity = 3
 
+    _immutable_fields_ = ['fastfunc_3']
+
     def fastcall_3(self, space, func, w1, w2, w3):
         try:
             w_result = self.fastfunc_3(space, w1, w2, w3)
@@ -843,6 +849,8 @@
 class BuiltinCode4(BuiltinCode):
     fast_natural_arity = 4
 
+    _immutable_fields_ = ['fastfunc_4']
+
     def fastcall_4(self, space, func, w1, w2, w3, w4):
         try:
             w_result = self.fastfunc_4(space, w1, w2, w3, w4)

From pypy.commits at gmail.com  Wed Sep 21 19:39:21 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 21 Sep 2016 16:39:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix tests for audioop.byteswap(),
 and test_wave passes.
Message-ID: <57e31a29.0cce1c0a.792f7.e541@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r87296:f7828bc4b08d
Date: 2016-09-22 01:38 +0200
http://bitbucket.org/pypy/pypy/changeset/f7828bc4b08d/

Log:	Fix tests for audioop.byteswap(), and test_wave passes.

diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -592,4 +592,5 @@
         rv[i] = cp[base]
         if base == next_bump:
             base += bump
+            next_bump += size
     return ffi.buffer(rv)[:]

From pypy.commits at gmail.com  Wed Sep 21 20:24:32 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 21 Sep 2016 17:24:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add memoryview.nbytes.
Message-ID: <57e324c0.516d1c0a.454d1.69c3@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r87297:0dd312748a34
Date: 2016-09-22 02:23 +0200
http://bitbucket.org/pypy/pypy/changeset/0dd312748a34/

Log:	Add memoryview.nbytes.

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -387,6 +387,10 @@
         self._check_released(space)
         return space.wrap(self.getlength())
 
+    def w_get_nbytes(self, space):
+        self._check_released(space)
+        return space.wrap(self.buf.getlength())
+
     def w_get_format(self, space):
         self._check_released(space)
         return space.wrap(self.getformat())
@@ -684,6 +688,7 @@
     format      = GetSetProperty(W_MemoryView.w_get_format),
     itemsize    = GetSetProperty(W_MemoryView.w_get_itemsize),
     ndim        = GetSetProperty(W_MemoryView.w_get_ndim),
+    nbytes        = GetSetProperty(W_MemoryView.w_get_nbytes),
     readonly    = GetSetProperty(W_MemoryView.w_is_readonly),
     shape       = GetSetProperty(W_MemoryView.w_get_shape),
     strides     = GetSetProperty(W_MemoryView.w_get_strides),
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -55,6 +55,7 @@
         assert v.shape == (100,)
         assert v.ndim == 1
         assert v.strides == (1,)
+        assert v.nbytes == 100
 
     def test_suboffsets(self):
         v = memoryview(b"a"*100)
@@ -127,6 +128,7 @@
         raises(ValueError, "v[0]")
         raises(ValueError, "v[0] = b'a'")
         raises(ValueError, "v.format")
+        raises(ValueError, "v.nbytes")
         raises(ValueError, "v.itemsize")
         raises(ValueError, "v.ndim")
         raises(ValueError, "v.readonly")
@@ -155,6 +157,7 @@
         assert len(m) == 10
         assert m.shape == (10,)
         assert len(m.tobytes()) == 40
+        assert m.nbytes == 40
         assert m[0] == 0
         m[0] = 1
         assert m[0] == 1
@@ -167,6 +170,7 @@
         assert slice.itemsize == 4
         assert len(slice) == 6
         assert len(slice.tobytes()) == 24
+        assert slice.nbytes == 24
         assert slice[0] == 2
         slice[0] = 1
         assert slice[0] == 1

From pypy.commits at gmail.com  Wed Sep 21 21:12:08 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 21 Sep 2016 18:12:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: socket.py now depends on the 'select'
 module.
Message-ID: <57e32fe8.c398c20a.f5f21.62f9@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r87298:09b437dc07a9
Date: 2016-09-22 02:35 +0200
http://bitbucket.org/pypy/pypy/changeset/09b437dc07a9/

Log:	socket.py now depends on the 'select' module.

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -300,7 +300,7 @@
 
 
 class AppTestSocket:
-    spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct'])
+    spaceconfig = dict(usemodules=['_socket', '_weakref', 'struct', 'select'])
 
     def setup_class(cls):
         cls.space = space
@@ -726,7 +726,7 @@
 
 class AppTestSocketTCP:
     HOST = 'localhost'
-    spaceconfig = {'usemodules': ['_socket', 'array']}
+    spaceconfig = {'usemodules': ['_socket', 'array', 'select']}
 
     def setup_method(self, method):
         w_HOST = self.space.wrap(self.HOST)
@@ -870,7 +870,7 @@
 
 
 class AppTestErrno:
-    spaceconfig = {'usemodules': ['_socket']}
+    spaceconfig = {'usemodules': ['_socket', 'select']}
 
     def test_errno(self):
         from socket import socket, AF_INET, SOCK_STREAM, error
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -2,7 +2,8 @@
 import os
 
 class AppTestSSL:
-    spaceconfig = dict(usemodules=('_ssl', '_socket', 'struct', 'binascii', 'thread'))
+    spaceconfig = dict(usemodules=('_ssl', '_socket', 'select', 'struct',
+                                   'binascii', 'thread'))
 
     def setup_class(cls):
         cls.w_nullbytecert = cls.space.wrap(os.path.join(
@@ -154,7 +155,7 @@
 class AppTestConnectedSSL:
     spaceconfig = {
         "usemodules": ['_ssl', '_socket', 'struct', 'array', 'binascii',
-                       'unicodedata'],
+                       'unicodedata', 'select'],
     }
 
     def setup_method(self, method):

From pypy.commits at gmail.com  Wed Sep 21 21:12:10 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 21 Sep 2016 18:12:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add a minimal _ssl.MemoryBIO to allow
 "import ssl"
Message-ID: <57e32fea.8aabc20a.1d096.502a@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r87299:3c1b46d02aad
Date: 2016-09-22 02:41 +0200
http://bitbucket.org/pypy/pypy/changeset/3c1b46d02aad/

Log:	Add a minimal _ssl.MemoryBIO to allow "import ssl"

diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
--- a/pypy/module/_ssl/__init__.py
+++ b/pypy/module/_ssl/__init__.py
@@ -22,6 +22,7 @@
 
         '_SSLSocket': 'interp_ssl.SSLSocket',
         '_SSLContext': 'interp_ssl.SSLContext',
+        'MemoryBIO': 'interp_ssl.MemoryBIO',
     }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1858,3 +1858,10 @@
         w_convert_path(space, libssl_X509_get_default_cert_dir_env()),
         w_convert_path(space, libssl_X509_get_default_cert_dir()),
         ])
+
+
+class MemoryBIO(W_Root):
+    pass
+MemoryBIO.typedef = TypeDef(
+    "_ssl.MemoryBIO",
+)

From pypy.commits at gmail.com  Wed Sep 21 21:12:11 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 21 Sep 2016 18:12:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix many tests in module/_ssl
Message-ID: <57e32feb.87adc20a.7f06f.57a3@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r87300:d58ded0f71fb
Date: 2016-09-22 03:08 +0200
http://bitbucket.org/pypy/pypy/changeset/d58ded0f71fb/

Log:	Fix many tests in module/_ssl

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -311,7 +311,8 @@
 
 
 class SSLSocket(W_Root):
-    def __init__(self, space):
+    def __init__(self, space, w_ctx):
+        self.w_ctx = w_ctx
         self.w_socket = None
         self.ssl = lltype.nullptr(SSL.TO)
         self.peer_cert = lltype.nullptr(X509.TO)
@@ -691,6 +692,16 @@
         self.w_ctx = w_ctx
         libssl_SSL_set_SSL_CTX(self.ssl, ctx.ctx)
 
+    def descr_get_owner(self, space):
+        if self.w_owner:
+            w_owner = self.w_owner()
+            if w_owner:
+                return w_owner
+        return space.w_None
+
+    def descr_set_owner(self, space, w_owner):
+        self.w_owner = weakref.ref(w_owner)
+
 
 SSLSocket.typedef = TypeDef("_ssl._SSLSocket",
     write = interp2app(SSLSocket.write),
@@ -707,6 +718,8 @@
     tls_unique_cb = interp2app(SSLSocket.tls_unique_cb_w),
     context=GetSetProperty(SSLSocket.descr_get_context,
                            SSLSocket.descr_set_context),
+    owner=GetSetProperty(SSLSocket.descr_get_owner,
+                           SSLSocket.descr_set_owner),
 )
 
 def _certificate_to_der(space, certificate):
@@ -999,14 +1012,14 @@
             libssl_sk_DIST_POINT_free(dps)
     return space.newtuple(cdp_w[:])
 
-def new_sslobject(space, ctx, w_sock, side, server_hostname):
-    ss = SSLSocket(space)
+def new_sslobject(space, w_ctx, w_sock, side, server_hostname):
+    ss = SSLSocket(space, w_ctx)
 
     sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
     w_timeout = space.call_method(w_sock, "gettimeout")
     has_timeout = not space.is_none(w_timeout)
 
-    ss.ssl = libssl_SSL_new(ctx) # new ssl struct
+    ss.ssl = libssl_SSL_new(w_ctx.ctx) # new ssl struct
     libssl_SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL
     # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address
     # of a str object may be changed by the garbage collector.
@@ -1261,7 +1274,20 @@
     index = rffi.cast(lltype.Signed, libssl_SSL_get_app_data(ssl))
     w_ssl = SOCKET_STORAGE.get(index)
     assert isinstance(w_ssl, SSLSocket)
-    w_ssl_socket = w_ssl  # So far. Need to change in 3.3.
+    # The servername callback expects an argument that represents the current
+    # SSL connection and that has a .context attribute that can be changed to
+    # identify the requested hostname. Since the official API is the Python
+    # level API we want to pass the callback a Python level object rather than
+    # a _ssl.SSLSocket instance. If there's an "owner" (typically an
+    # SSLObject) that will be passed. Otherwise if there's a socket then that
+    # will be passed. If both do not exist only then the C-level object is
+    # passed.
+    if w_ssl.w_owner:
+        w_ssl_socket = w_ssl.w_owner()
+    elif w_ssl.w_socket:
+        w_ssl_socket = w_ssl.w_socket()
+    else:
+        w_ssl_socket = w_ssl
     if space.is_none(w_ssl_socket):
         ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
         return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
@@ -1392,7 +1418,7 @@
                         "server_hostname is not supported by your OpenSSL "
                         "library")
 
-        return new_sslobject(space, self.ctx, w_sock, server_side, hostname)
+        return new_sslobject(space, self, w_sock, server_side, hostname)
 
     def session_stats_w(self, space):
         w_stats = space.newdict()
@@ -1796,9 +1822,8 @@
                                SSLContext.descr_set_verify_mode),
     verify_flags=GetSetProperty(SSLContext.descr_get_verify_flags,
                                 SSLContext.descr_set_verify_flags),
-    # XXX: For use by 3.4 ssl.py only
-    #check_hostname=GetSetProperty(SSLContext.descr_get_check_hostname,
-    #                              SSLContext.descr_set_check_hostname),
+    check_hostname=GetSetProperty(SSLContext.descr_get_check_hostname,
+                                  SSLContext.descr_set_check_hostname),
 )
 
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -428,7 +428,7 @@
 
 class AppTestSSLError:
     spaceconfig = dict(usemodules=('_ssl', '_socket', 'binascii', 'thread',
-                                   'struct'))
+                                   'struct', 'select'))
 
     def setup_class(cls):
         tmpfile = udir / "tmpfile.pem"

From pypy.commits at gmail.com  Wed Sep 21 23:01:20 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Wed, 21 Sep 2016 20:01:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: attempt to adapt missing parts of
 29d14733e007 to cffi
Message-ID: <57e34980.0f1f1c0a.35756.07d4@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r87301:16035fa70024
Date: 2016-09-21 20:00 -0700
http://bitbucket.org/pypy/pypy/changeset/16035fa70024/

Log:	attempt to adapt missing parts of 29d14733e007 to cffi

diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -83,6 +83,7 @@
 BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
 HANDLE WINAPI GetStdHandle(DWORD);
+DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD);
 """)
 
 # --------------------
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -162,18 +162,19 @@
         return int(_ffi.cast("intptr_t", res))
 
 def CloseHandle(handle):
-    res = _CloseHandle(handle)
+    res = _kernel32.CloseHandle(_ffi.cast("HANDLE", handle))
 
     if not res:
         raise _WinError()
 
 def GetModuleFileName(module):
-    buf = ctypes.create_unicode_buffer(_MAX_PATH)
-    res = _GetModuleFileNameW(module, buf, _MAX_PATH)
+    buf = _ffi.new("wchar_t[]", _MAX_PATH)
+    res = _kernel32.GetModuleFileNameW(_ffi.cast("HANDLE", module),
+                                       buf, _MAX_PATH)
 
     if not res:
         raise _WinError()
-    return buf.value
+    return _ffi.string(buf)
 
 STD_INPUT_HANDLE = -10
 STD_OUTPUT_HANDLE = -11

From pypy.commits at gmail.com  Thu Sep 22 02:33:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Sep 2016 23:33:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Duplicate
Message-ID: <57e37b3d.9102c20a.5d128.1cc0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87302:1fec3502fdc3
Date: 2016-09-22 08:30 +0200
http://bitbucket.org/pypy/pypy/changeset/1fec3502fdc3/

Log:	Duplicate

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -20,7 +20,6 @@
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.util import get_positive_index
 
-NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d"
 
 
 class W_BytearrayObject(W_Root):

From pypy.commits at gmail.com  Thu Sep 22 03:03:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 00:03:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Try to blindly fix some casting issues
 inside interp_time, showing up
Message-ID: <57e3824d.04141c0a.25833.466e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87303:e6d9f404e80c
Date: 2016-09-22 08:55 +0200
http://bitbucket.org/pypy/pypy/changeset/e6d9f404e80c/

Log:	Try to blindly fix some casting issues inside interp_time, showing
	up e.g. on OS/X translations.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror, exception_from_saved_errno
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat
+from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat, widen
 from rpython.rlib.rtime import (TIMEB, c_ftime,
                                 GETTIMEOFDAY_NO_TZ, TIMEVAL,
                                 HAVE_GETTIMEOFDAY, HAVE_FTIME)
@@ -290,12 +290,14 @@
                 if rffi.cast(rffi.LONG, errcode) == 0:
                     if w_info is not None:
                         _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
-                    return space.wrap(timeval.c_tv_sec + timeval.c_tv_usec * 1e-6)
+                    return space.wrap(
+                        widen(timeval.c_tv_sec) +
+                        int(timeval.c_tv_usec) * 1e-6)
         if HAVE_FTIME:
             with lltype.scoped_alloc(TIMEB) as t:
                 c_ftime(t)
-                result = (float(intmask(t.c_time)) +
-                          float(intmask(t.c_millitm)) * 0.001)
+                result = (widen(t.c_time) +
+                          int(t.c_millitm) * 0.001)
                 if w_info is not None:
                     _setinfo(space, w_info, "ftime()", 1e-3,
                              False, True) 
@@ -535,7 +537,7 @@
         if not allowNone:
             raise oefmt(space.w_TypeError, "tuple expected")
         # default to the current local time
-        tt = rffi.r_time_t(int(pytime.time()))
+        tt = rffi.cast(rffi.TIME_T, pytime.time())
         t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
         t_ref[0] = tt
         pbuf = c_localtime(t_ref)
@@ -737,7 +739,7 @@
 
 if HAS_CLOCK_GETTIME:
     def _timespec_to_seconds(timespec):
-        return int(timespec.c_tv_sec) + int(timespec.c_tv_nsec) * 1e-9
+        return widen(timespec.c_tv_sec) + int(timespec.c_tv_nsec) * 1e-9
 
     @unwrap_spec(clk_id='c_int')
     def clock_gettime(space, clk_id):
@@ -751,8 +753,9 @@
     @unwrap_spec(clk_id='c_int', secs=float)
     def clock_settime(space, clk_id, secs):
         with lltype.scoped_alloc(TIMESPEC) as timespec:
-            frac = math.fmod(secs, 1.0)
-            rffi.setintfield(timespec, 'c_tv_sec', int(secs))
+            integer_secs = rffi.cast(TIMESPEC.c_tv_sec, secs)
+            frac = secs - widen(integer_secs)
+            rffi.setintfield(timespec, 'c_tv_sec', integer_secs)
             rffi.setintfield(timespec, 'c_tv_nsec', int(frac * 1e9))
             ret = c_clock_settime(clk_id, timespec)
             if ret != 0:
@@ -1051,16 +1054,15 @@
             return win_perf_counter(space, w_info=w_info)
         except ValueError:
             pass
-    value = _clock()
-    # Is this casting correct?
-    if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)):
+    value = widen(_clock())
+    if value == widen(rffi.cast(rposix.CLOCK_T, -1)):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")
     if w_info is not None:
         _setinfo(space, w_info,
                  "clock()", 1.0 / CLOCKS_PER_SEC, True, False)
-    return space.wrap((1.0 * value) / CLOCKS_PER_SEC)
+    return space.wrap(float(value) / CLOCKS_PER_SEC)
 
 
 def _setinfo(space, w_info, impl, res, mono, adj):

From pypy.commits at gmail.com  Thu Sep 22 03:06:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 22 Sep 2016 00:06:50 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: translation issues
Message-ID: <57e3830a.c70a1c0a.45dfe.47bb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87304:2b045fa07844
Date: 2016-09-22 09:06 +0200
http://bitbucket.org/pypy/pypy/changeset/2b045fa07844/

Log:	translation issues

diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -13,11 +13,6 @@
   (e.g. those in the NumPyPy module).
 * --jit vec_all=1: turns on the vectorization for any jit driver. See parameters for
   the filtering heuristics of traces.
-* --jit vec_ratio=2: A number from 0 to 10 that represents a real number (vec_ratio / 10).
-  This filters traces if vec_all is enabled. N is the trace count then the number of
-  vector transformable operations (add_int -> vec_add_int) M, the following must hold:
-  M / N >= (vec_ratio / 10)
-* --jit vec_length=60: The maximum number of trace instructions the vectorizer filters for.
 
 Features
 --------
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -186,10 +186,8 @@
         if op.is_primitive_array_access():
             at_least_one_array_access = True
 
-        if warmstate.vec_ratio > 0.0:
-            # blacklist
-            if rop.is_call(op.opnum) or rop.is_call_assembler(op.opnum):
-                return True
+        if rop.is_call(op.opnum) or rop.is_call_assembler(op.opnum):
+            return True
 
         if rop.is_guard(op.opnum):
             guard_count += 1
@@ -197,15 +195,6 @@
     if not at_least_one_array_access:
         return True
 
-    if resop_count > warmstate.vec_length:
-        return True
-
-    if (float(vector_instr)/float(resop_count)) < warmstate.vec_ratio:
-        return True
-
-    if float(guard_count)/float(resop_count) > warmstate.vec_guard_ratio:
-        return True
-
     return False
 
 class VectorizingOptimizer(Optimizer):
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -739,7 +739,7 @@
                 i += 1
             lltype.free(vector_a, flavor='raw')
             return breaks
-        res = self.meta_interp(f, [22], vec_all=True, vec_guard_ratio=5)
+        res = self.meta_interp(f, [22], vec_all=True)
         assert res == f(22)
 
     def run_unpack(self, unpack, vector_type, assignments, float=True):

From pypy.commits at gmail.com  Thu Sep 22 03:45:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 00:45:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: next attempt
Message-ID: <57e38c0d.43e61c0a.64418.96e1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87305:5fe4864fbac4
Date: 2016-09-22 09:43 +0200
http://bitbucket.org/pypy/pypy/changeset/5fe4864fbac4/

Log:	next attempt

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -292,12 +292,12 @@
                         _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
                     return space.wrap(
                         widen(timeval.c_tv_sec) +
-                        int(timeval.c_tv_usec) * 1e-6)
+                        widen(timeval.c_tv_usec) * 1e-6)
         if HAVE_FTIME:
             with lltype.scoped_alloc(TIMEB) as t:
                 c_ftime(t)
                 result = (widen(t.c_time) +
-                          int(t.c_millitm) * 0.001)
+                          widen(t.c_millitm) * 0.001)
                 if w_info is not None:
                     _setinfo(space, w_info, "ftime()", 1e-3,
                              False, True) 
@@ -739,7 +739,7 @@
 
 if HAS_CLOCK_GETTIME:
     def _timespec_to_seconds(timespec):
-        return widen(timespec.c_tv_sec) + int(timespec.c_tv_nsec) * 1e-9
+        return widen(timespec.c_tv_sec) + widen(timespec.c_tv_nsec) * 1e-9
 
     @unwrap_spec(clk_id='c_int')
     def clock_gettime(space, clk_id):

From pypy.commits at gmail.com  Thu Sep 22 04:08:41 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 22 Sep 2016 01:08:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix translation
Message-ID: <57e39189.a699c20a.501d5.4358@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r87306:827ca1cc3dfb
Date: 2016-09-22 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/827ca1cc3dfb/

Log:	Fix translation

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -314,6 +314,7 @@
     def __init__(self, space, w_ctx):
         self.w_ctx = w_ctx
         self.w_socket = None
+        self.w_owner = None
         self.ssl = lltype.nullptr(SSL.TO)
         self.peer_cert = lltype.nullptr(X509.TO)
         self.shutdown_seen_zero = False
@@ -693,13 +694,14 @@
         libssl_SSL_set_SSL_CTX(self.ssl, ctx.ctx)
 
     def descr_get_owner(self, space):
-        if self.w_owner:
+        if self.w_owner is not None:
             w_owner = self.w_owner()
             if w_owner:
                 return w_owner
         return space.w_None
 
     def descr_set_owner(self, space, w_owner):
+        assert w_owner is not None
         self.w_owner = weakref.ref(w_owner)
 
 
@@ -1282,9 +1284,9 @@
     # SSLObject) that will be passed. Otherwise if there's a socket then that
     # will be passed. If both do not exist only then the C-level object is
     # passed.
-    if w_ssl.w_owner:
+    if w_ssl.w_owner is not None:
         w_ssl_socket = w_ssl.w_owner()
-    elif w_ssl.w_socket:
+    elif w_ssl.w_socket is not None:
         w_ssl_socket = w_ssl.w_socket()
     else:
         w_ssl_socket = w_ssl

From pypy.commits at gmail.com  Thu Sep 22 05:12:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 02:12:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: memoryview indexing returns integers,
 not bytes
Message-ID: <57e3a074.448e1c0a.45ebb.769e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87307:01fa23342efc
Date: 2016-09-22 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/01fa23342efc/

Log:	memoryview indexing returns integers, not bytes

diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1126,8 +1126,8 @@
         a[3] = b'x'
         b = memoryview(a)
         assert len(b) == 10
-        assert b[3] == b'x'
-        b[6] = b'y'
+        assert b[3] == ord(b'x')
+        b[6] = ord(b'y')
         assert a[6] == b'y'
         b[3:5] = b'zt'
         assert a[3] == b'z'
@@ -1135,9 +1135,9 @@
 
         b = memoryview(a)
         assert len(b) == 10
-        assert b[3] == b'z'
-        b[3] = b'x'
-        assert b[3] == b'x'
+        assert b[3] == ord(b'z')
+        b[3] = ord(b'x')
+        assert b[3] == ord(b'x')
 
     def test_pypy_raw_address(self):
         import _rawffi

From pypy.commits at gmail.com  Thu Sep 22 05:16:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 02:16:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Aaaargh
Message-ID: <57e3a163.516d1c0a.454d1.f8c6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87308:39f6d2a2c675
Date: 2016-09-22 11:15 +0200
http://bitbucket.org/pypy/pypy/changeset/39f6d2a2c675/

Log:	Aaaargh

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1970,7 +1970,7 @@
     'UnicodeWarning',
     'ValueError',
     'Warning',
-    'ZeroDivisionError'
+    'ZeroDivisionError',
     'RuntimeWarning',
     'PendingDeprecationWarning',
     'UserWarning',

From pypy.commits at gmail.com  Thu Sep 22 05:30:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 02:30:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57e3a4b7.0d8d1c0a.ae587.7eec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87309:7f7862ca4a74
Date: 2016-09-22 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/7f7862ca4a74/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 3083 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -13,6 +13,7 @@
 import sys
 import os
 import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -62,8 +63,7 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -75,8 +75,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,7 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break
+                break        
 
     if args:
         vals = []
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -83,6 +83,7 @@
 BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
 HANDLE WINAPI GetStdHandle(DWORD);
+DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD);
 """)
 
 # --------------------
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -162,18 +162,19 @@
         return int(_ffi.cast("intptr_t", res))
 
 def CloseHandle(handle):
-    res = _CloseHandle(handle)
+    res = _kernel32.CloseHandle(_ffi.cast("HANDLE", handle))
 
     if not res:
         raise _WinError()
 
 def GetModuleFileName(module):
-    buf = ctypes.create_unicode_buffer(_MAX_PATH)
-    res = _GetModuleFileNameW(module, buf, _MAX_PATH)
+    buf = _ffi.new("wchar_t[]", _MAX_PATH)
+    res = _kernel32.GetModuleFileNameW(_ffi.cast("HANDLE", module),
+                                       buf, _MAX_PATH)
 
     if not res:
         raise _WinError()
-    return buf.value
+    return _ffi.string(buf)
 
 STD_INPUT_HANDLE = -10
 STD_OUTPUT_HANDLE = -11
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.2
+Version: 1.8.4
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.2"
-__version_info__ = (1, 8, 2)
+__version__ = "1.8.4"
+__version_info__ = (1, 8, 4)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.2"
+                               "\ncompiled with cffi version: 1.8.4"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -332,7 +332,7 @@
                         realtype = model.unknown_ptr_type(decl.name)
                     else:
                         realtype, quals = self._get_type_and_quals(
-                            decl.type, name=decl.name)
+                            decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
                 else:
                     raise api.CDefError("unrecognized construct", decl)
@@ -781,11 +781,14 @@
                 exprnode.name in self._int_constants):
             return self._int_constants[exprnode.name]
         #
-        if partial_length_ok:
-            if (isinstance(exprnode, pycparser.c_ast.ID) and
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
                 self._partial_length = True
                 return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
         #
         raise api.FFIError(":%d: unsupported expression: expected a "
                            "simple numeric constant" % exprnode.coord.line)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -587,8 +587,11 @@
     # ----------
     # typedefs
 
+    def _typedef_type(self, tp, name):
+        return self._global_type(tp, "(*(%s *)0)" % (name,))
+
     def _generate_cpy_typedef_collecttype(self, tp, name):
-        self._do_collect_type(tp)
+        self._do_collect_type(self._typedef_type(tp, name))
 
     def _generate_cpy_typedef_decl(self, tp, name):
         pass
@@ -598,6 +601,7 @@
         self._lsts["typename"].append(TypenameExpr(name, type_index))
 
     def _generate_cpy_typedef_ctx(self, tp, name):
+        tp = self._typedef_type(tp, name)
         self._typedef_ctx(tp, name)
         if getattr(tp, "origin", None) == "unknown_type":
             self._struct_ctx(tp, tp.name, approxname=None)
diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt
--- a/pypy/doc/config/translation.profopt.txt
+++ b/pypy/doc/config/translation.profopt.txt
@@ -3,3 +3,14 @@
 RPython program) to gather profile data. Example for pypy-c: "-c 'from
 richards import main;main(); from test import pystone;
 pystone.main()'"
+
+NOTE: be aware of what this does in JIT-enabled executables.  What it
+does is instrument and later optimize the C code that happens to run in
+the example you specify, ignoring any execution of the JIT-generated
+assembler.  That means that you have to choose the example wisely.  If
+it is something that will just generate assembler and stay there, there
+is little value.  If it is something that exercises heavily library
+routines that are anyway written in C, then it will optimize that.  Most
+interesting would be something that causes a lot of JIT-compilation,
+like running a medium-sized test suite several times in a row, in order
+to optimize the warm-up in general.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,8 @@
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -246,6 +246,10 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+            if config.translation.profopt is not None:
+                raise Exception("Cannot use the --profopt option "
+                                "when --shared is on (it is by default). "
+                                "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1961,12 +1961,10 @@
     'NotImplementedError',
     'OSError',
     'OverflowError',
-    'PendingDeprecationWarning',
     'ReferenceError',
     'ResourceWarning',
     'RecursionError',
     'RuntimeError',
-    'RuntimeWarning',
     'StopIteration',
     'SyntaxError',
     'SyntaxWarning',
@@ -1980,10 +1978,12 @@
     'UnicodeError',
     'UnicodeTranslateError',
     'UnicodeWarning',
-    'UserWarning',
     'ValueError',
     'Warning',
-    'ZeroDivisionError'
+    'ZeroDivisionError',
+    'RuntimeWarning',
+    'PendingDeprecationWarning',
+    'UserWarning',
 ]
 
 if sys.platform.startswith("win"):
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,12 +57,14 @@
         def f():
             yield 2
         g = f()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
 
     def test_throw3(self):
@@ -236,7 +238,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -583,3 +583,21 @@
         assert a2ref() is None, "stack not cleared"
         #
         raises(StopIteration, next, gen)
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.2"
+VERSION = "1.8.4"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer
 from pypy.module._cffi_backend import ctypeprim
 
 
@@ -22,6 +22,7 @@
     is_nonfunc_pointer_or_array = True
 
     def __init__(self, space, ctptr, length, arraysize, extra):
+        assert isinstance(ctptr, W_CTypePointer)
         W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
                                    ctptr.ctitem)
         self.length = length
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -35,8 +35,7 @@
         assert isinstance(ellipsis, bool)
         extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi)
         size = rffi.sizeof(rffi.VOIDP)
-        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult,
-                                could_cast_anything=False)
+        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult)
         self.fargs = fargs
         self.ellipsis = ellipsis
         self.abi = abi
@@ -59,6 +58,16 @@
                     lltype.free(self.cif_descr, flavor='raw')
                     self.cif_descr = lltype.nullptr(CIF_DESCRIPTION)
 
+    def is_unichar_ptr_or_array(self):
+        return False
+
+    def is_char_or_unichar_ptr_or_array(self):
+        return False
+
+    def string(self, cdataobj, maxlen):
+        # Can't use ffi.string() on a function pointer
+        return W_CType.string(self, cdataobj, maxlen)
+
     def new_ctypefunc_completing_argtypes(self, args_w):
         space = self.space
         nargs_declared = len(self.fargs)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -19,7 +19,6 @@
     # XXX this could be improved with an elidable method get_size()
     # that raises in case it's still -1...
 
-    cast_anything = False
     is_primitive_integer = False
     is_nonfunc_pointer_or_array = False
     is_indirect_arg_for_call_python = False
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -120,7 +120,6 @@
 
 class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar):
     _attrs_ = []
-    cast_anything = True
 
     def cast_to_int(self, cdata):
         return self.space.wrap(ord(cdata[0]))
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -16,12 +16,11 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _attrs_            = ['ctitem', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'accept_str', 'length']
     length = -1
 
-    def __init__(self, space, size, extra, extra_position, ctitem,
-                 could_cast_anything=True):
+    def __init__(self, space, size, extra, extra_position, ctitem):
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
@@ -29,10 +28,11 @@
         #  - for arrays, it is the array item type
         #  - for functions, it is the return type
         self.ctitem = ctitem
-        self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.accept_str = (self.can_cast_anything or
-                            (ctitem.is_primitive_integer and
-                             ctitem.size == rffi.sizeof(lltype.Char)))
+        self.accept_str = (self.is_nonfunc_pointer_or_array and
+                (isinstance(ctitem, ctypevoid.W_CTypeVoid) or
+                 isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or
+                 (ctitem.is_primitive_integer and
+                  ctitem.size == rffi.sizeof(lltype.Char))))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -139,7 +139,10 @@
 
 class W_CTypePtrBase(W_CTypePtrOrArray):
     # base class for both pointers and pointers-to-functions
-    _attrs_ = []
+    _attrs_ = ['is_void_ptr', 'is_voidchar_ptr']
+    _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr']
+    is_void_ptr = False
+    is_voidchar_ptr = False
 
     def convert_to_object(self, cdata):
         ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0]
@@ -156,7 +159,16 @@
             else:
                 raise self._convert_error("compatible pointer", w_ob)
         if self is not other:
-            if not (self.can_cast_anything or other.can_cast_anything):
+            if self.is_void_ptr or other.is_void_ptr:
+                pass     # cast from or to 'void *'
+            elif self.is_voidchar_ptr or other.is_voidchar_ptr:
+                space = self.space
+                msg = ("implicit cast from '%s' to '%s' "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" % (other.name, self.name))
+                space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1)
+            else:
                 raise self._convert_error("compatible pointer", w_ob)
 
         rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
@@ -167,8 +179,8 @@
 
 
 class W_CTypePointer(W_CTypePtrBase):
-    _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types']
-    _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr']
+    _attrs_ = ['is_file', 'cache_array_type', '_array_types']
+    _immutable_fields_ = ['is_file', 'cache_array_type?']
     kind = "pointer"
     cache_array_type = None
     is_nonfunc_pointer_or_array = True
@@ -183,6 +195,8 @@
         self.is_file = (ctitem.name == "struct _IO_FILE" or
                         ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
+        self.is_voidchar_ptr = (self.is_void_ptr or
+                           isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar))
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init, allocator):
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -7,7 +7,6 @@
 
 class W_CTypeVoid(W_CType):
     _attrs_ = []
-    cast_anything = True
     kind = "void"
 
     def __init__(self, space):
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -32,8 +32,8 @@
 @unwrap_spec(w_cdata=cdataobj.W_CData)
 def from_handle(space, w_cdata):
     ctype = w_cdata.ctype
-    if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or
-        not ctype.can_cast_anything):
+    if (not isinstance(ctype, ctypeptr.W_CTypePointer) or
+        not ctype.is_voidchar_ptr):
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3665,3 +3665,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -503,3 +503,15 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_bug_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        q = ffi.new("char[]", b"abcd")
+        p = ffi.cast("char(*)(void)", q)
+        raises(TypeError, ffi.string, p)
+
+    def test_negative_array_size(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -8,7 +8,7 @@
 
 @unwrap_spec(cdef=str, module_name=str, source=str)
 def prepare(space, cdef, module_name, source, w_includes=None,
-            w_extra_source=None):
+            w_extra_source=None, w_min_version=None):
     try:
         import cffi
         from cffi import FFI            # <== the system one, which
@@ -16,8 +16,13 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 4, 0):
-        py.test.skip("system cffi module needs to be at least 1.4.0")
+    if w_min_version is None:
+        min_version = (1, 4, 0)
+    else:
+        min_version = tuple(space.unwrap(w_min_version))
+    if cffi.__version_info__ < min_version:
+        py.test.skip("system cffi module needs to be at least %s, got %s" % (
+            min_version, cffi.__version_info__))
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -1790,3 +1795,28 @@
                                 "void f(void) { }")
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
+
+    def test_typedef_array_dotdotdot(self):
+        ffi, lib = self.prepare("""
+            typedef int foo_t[...], bar_t[...];
+            int gv[...];
+            typedef int mat_t[...][...];
+            typedef int vmat_t[][...];
+            """,
+            "test_typedef_array_dotdotdot", """
+            typedef int foo_t[50], bar_t[50];
+            int gv[23];
+            typedef int mat_t[6][7];
+            typedef int vmat_t[][8];
+        """, min_version=(1, 8, 4))
+        assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+        assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+        assert len(ffi.new("foo_t")) == 50
+        assert len(ffi.new("bar_t")) == 50
+        assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+        assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+        assert len(ffi.new("mat_t")) == 6
+        assert len(ffi.new("mat_t")[3]) == 7
+        raises(ffi.error, ffi.sizeof, "vmat_t")
+        p = ffi.new("vmat_t", 4)
+        assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1126,8 +1126,8 @@
         a[3] = b'x'
         b = memoryview(a)
         assert len(b) == 10
-        assert b[3] == b'x'
-        b[6] = b'y'
+        assert b[3] == ord(b'x')
+        b[6] = ord(b'y')
         assert a[6] == b'y'
         b[3:5] = b'zt'
         assert a[3] == b'z'
@@ -1135,9 +1135,9 @@
 
         b = memoryview(a)
         assert len(b) == 10
-        assert b[3] == b'z'
-        b[3] = b'x'
-        assert b[3] == b'x'
+        assert b[3] == ord(b'z')
+        b[3] = ord(b'x')
+        assert b[3] == ord(b'x')
 
     def test_pypy_raw_address(self):
         import _rawffi
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -33,28 +33,27 @@
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if isinstance(w_initializer, W_ArrayBase):
-                    a.extend(w_initializer, True)
-                elif space.type(w_initializer) is space.w_list:
-                    a.descr_fromlist(space, w_initializer)
-                else:
-                    try:
-                        buf = space.bufferstr_w(w_initializer)
-                    except OperationError as e:
-                        if not e.match(space, space.w_TypeError):
-                            raise
-                        a.extend(w_initializer, True)
-                    else:
-                        a.descr_frombytes(space, buf)
             break
     else:
         raise oefmt(space.w_ValueError,
                     "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f "
                     "or d)")
 
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        if isinstance(w_initializer, W_ArrayBase):
+            a.extend(w_initializer, True)
+        elif space.type(w_initializer) is space.w_list:
+            a.descr_fromlist(space, w_initializer)
+        else:
+            try:
+                buf = space.bufferstr_w(w_initializer)
+            except OperationError as e:
+                if not e.match(space, space.w_TypeError):
+                    raise
+                a.extend(w_initializer, True)
+            else:
+                a.descr_frombytes(space, buf)
     return a
 
 
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -5,7 +5,6 @@
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long", "w_int")
@@ -27,25 +26,25 @@
     """Return a new PyLongObject object from a C size_t, or NULL on
     failure.
     """
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.LONGLONG], PyObject)
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
     """Return a new PyLongObject object from a C unsigned long, or
     NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONGLONG], PyObject)
 def PyLong_FromUnsignedLongLong(space, val):
     """Return a new PyLongObject object from a C unsigned long long,
     or NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([PyObject], rffi.ULONG, error=-1)
 def PyLong_AsUnsignedLong(space, w_long):
@@ -212,7 +211,10 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    value = rffi.cast(ADDR, p)    # signed integer
+    if value < 0:
+        return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
+    return space.wrap(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -1,5 +1,6 @@
 import sys, py
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import maxint
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
@@ -7,18 +8,20 @@
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
-        value = api.PyLong_FromLong(3)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == 3
+        w_value = api.PyLong_FromLong(3)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == 3
 
-        value = api.PyLong_FromLong(sys.maxint)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == sys.maxint
+        w_value = api.PyLong_FromLong(sys.maxint)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == sys.maxint
 
     def test_aslong(self, space, api):
         w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+        assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsLong(w_value)
         assert value == (sys.maxint - 1)
 
@@ -34,12 +37,16 @@
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsSsize_t(w_value)
         assert value == 2
-        assert space.eq_w(w_value, api.PyLong_FromSsize_t(2))
+        w_val2 = api.PyLong_FromSsize_t(2)
+        assert isinstance(w_val2, W_LongObject)
+        assert space.eq_w(w_value, w_val2)
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
+        assert space.isinstance_w(w_value, space.w_int)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -102,9 +109,22 @@
 
     def test_as_voidptr(self, space, api):
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
-        assert space.unwrap(w_l) == 0L
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == 0
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
 
+        p = rffi.cast(rffi.VOIDP, maxint)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == maxint
+        assert api.PyLong_AsVoidPtr(w_l) == p
+
+        p = rffi.cast(rffi.VOIDP, -maxint-1)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == maxint+1
+        assert api.PyLong_AsVoidPtr(w_l) == p
+
     def test_sign_and_bits(self, space, api):
         if space.is_true(space.lt(space.sys.get('version_info'),
                                   space.wrap((2, 7)))):
@@ -133,23 +153,58 @@
         module = self.import_extension('foo', [
             ("from_unsignedlong", "METH_NOARGS",
              """
-                 return PyLong_FromUnsignedLong((unsigned long)-1);
+                 PyObject * obj;
+                 obj = PyLong_FromUnsignedLong((unsigned long)-1);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
         import sys
         assert module.from_unsignedlong() == 2 * sys.maxsize + 1
 
     def test_fromlonglong(self):
         module = self.import_extension('foo', [
-            ("from_longlong", "METH_NOARGS",
+            ("from_longlong", "METH_VARARGS",
              """
-                 return PyLong_FromLongLong((long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """),
-            ("from_unsignedlonglong", "METH_NOARGS",
+            ("from_unsignedlonglong", "METH_VARARGS",
              """
-                 return PyLong_FromUnsignedLongLong((unsigned long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromUnsignedLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
-        assert module.from_longlong() == -1
-        assert module.from_unsignedlonglong() == (1<<64) - 1
+        assert module.from_longlong(-1) == -1
+        assert module.from_longlong(0) == 0
+        assert module.from_unsignedlonglong(0) == 0
+        assert module.from_unsignedlonglong(-1) == (1<<64) - 1
 
     def test_from_size_t(self):
         module = self.import_extension('foo', [
@@ -228,7 +283,7 @@
                     return PyLong_FromLong(3);
                  if (str + strlen(str) != end)
                     return PyLong_FromLong(4);
-                 return PyLong_FromLong(0); 
+                 return PyLong_FromLong(0);
              """)])
         assert module.from_str() == 0
 
@@ -237,10 +292,15 @@
             ("has_sub", "METH_NOARGS",
              """
                 PyObject *ret, *obj = PyLong_FromLong(42);
-                if (obj->ob_type->tp_as_number->nb_subtract)
-                    ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                if (obj->ob_type != &PyLong_Type)
+                    ret = PyLong_FromLong(-2);
                 else
-                    ret = PyLong_FromLong(-1);
+                {
+                    if (obj->ob_type->tp_as_number->nb_subtract)
+                        ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                    else
+                        ret = PyLong_FromLong(-1);
+                }
                 Py_DECREF(obj);
                 return ret;
              """),
@@ -279,4 +339,4 @@
         assert module.has_pow() == 0
         assert module.has_hex() == '0x2aL'
         assert module.has_oct() == '052L'
-                
+
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,7 +3,7 @@
 import py, pytest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
-only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 def test_pragma_version():
     from pypy.module.sys.version import CPYTHON_VERSION
@@ -46,10 +46,18 @@
     def test_pypy_versions(self):
         import sys
         init = """
+        static struct PyModuleDef moduledef = {
+                PyModuleDef_HEAD_INIT,
+                "foo",          /* m_name */
+                NULL,           /* m_doc */
+                -1,             /* m_size */
+                NULL            /* m_methods */
+            };
         if (Py_IsInitialized()) {
-            PyObject *m = Py_InitModule("foo", NULL);
+            PyObject *m = PyModule_Create(&moduledef);
             PyModule_AddStringConstant(m, "pypy_version", PYPY_VERSION);
             PyModule_AddIntConstant(m, "pypy_version_num", PYPY_VERSION_NUM);
+            return m;
         }
         Py_RETURN_NONE;
         """
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -40,9 +40,6 @@
     st_atime = structseqfield(11, "time of last access")
     st_mtime = structseqfield(12, "time of last modification")
     st_ctime = structseqfield(13, "time of last change")
-    st_atime_ns = structseqfield(14, "time of last access in nanoseconds")
-    st_mtime_ns = structseqfield(15, "time of last modification in nanoseconds")
-    st_ctime_ns = structseqfield(16, "time of last change in nanoseconds")
 
     if "st_blksize" in posix._statfields:
         st_blksize = structseqfield(20, "blocksize for filesystem I/O")
@@ -64,6 +61,21 @@
         if self.st_ctime is None:
             self.__dict__['st_ctime'] = self[9]
 
+    @property
+    def st_atime_ns(self):
+        "time of last access in nanoseconds"
+        return int(self[7]) * 1000000000 + self.nsec_atime
+
+    @property
+    def st_mtime_ns(self):
+        "time of last modification in nanoseconds"
+        return int(self[8]) * 1000000000 + self.nsec_mtime
+
+    @property
+    def st_ctime_ns(self):
+        "time of last change in nanoseconds"
+        return int(self[9]) * 1000000000 + self.nsec_ctime
+
 
 class statvfs_result(metaclass=structseqtype):
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -364,56 +364,50 @@
 
 STAT_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS))
 
-N_INDEXABLE_FIELDS = 10
-
-def _time_ns_from_float(ftime):
-    "Convert a floating-point time (in seconds) into a (s, ns) pair of ints"
-    fracpart, intpart = modf(ftime)
-    if fracpart < 0:
-        fracpart += 1.
-        intpart -= 1.
-    return int(intpart), int(fracpart * 1e9)
-
- at specialize.arg(4)
-def _fill_time(space, lst, index, w_keywords, attrname, ftime):
-    stat_float_times = space.fromcache(StatState).stat_float_times
-    seconds, fractional_ns = _time_ns_from_float(ftime)
-    lst[index] = space.wrap(seconds)
-    if stat_float_times:
-        space.setitem(w_keywords, space.wrap(attrname), space.wrap(ftime))
-    else:
-        space.setitem(w_keywords, space.wrap(attrname), space.wrap(seconds))
-    w_billion = space.wrap(1000000000)
-    w_total_ns = space.add(space.mul(space.wrap(seconds), w_billion),
-                           space.wrap(fractional_ns))
-    space.setitem(w_keywords, space.wrap(attrname + '_ns'), w_total_ns)
-
-STANDARD_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS[:7]))
-EXTRA_FIELDS = unrolling_iterable(rposix_stat.STAT_FIELDS[10:])
+STATVFS_FIELDS = unrolling_iterable(enumerate(rposix_stat.STATVFS_FIELDS))
 
 def build_stat_result(space, st):
-    lst = [None] * N_INDEXABLE_FIELDS
+    FIELDS = STAT_FIELDS    # also when not translating at all
+    lst = [None] * rposix_stat.N_INDEXABLE_FIELDS
     w_keywords = space.newdict()
-    for (i, (name, TYPE)) in STANDARD_FIELDS:
-        value = getattr(st, name)
-        w_value = space.wrap(value)
-        lst[i] = w_value
+    stat_float_times = space.fromcache(StatState).stat_float_times
+    for i, (name, TYPE) in FIELDS:
+        if i < rposix_stat.N_INDEXABLE_FIELDS:
+            # get the first 10 items by indexing; this gives us
+            # 'st_Xtime' as an integer, too
+            w_value = space.wrap(st[i])
+            lst[i] = w_value
+        else:
+            try:
+                value = getattr(st, name)
+            except AttributeError:
+                # untranslated, there is no nsec_Xtime attribute
+                assert name.startswith('nsec_')
+                value = rposix_stat.get_stat_ns_as_bigint(st, name[5:])
+                value = value.tolong() % 1000000000
+            w_value = space.wrap(value)
+            space.setitem(w_keywords, space.wrap(name), w_value)
 
-    _fill_time(space, lst, 7, w_keywords, 'st_atime', st.st_atime)
-    _fill_time(space, lst, 8, w_keywords, 'st_mtime', st.st_mtime)
-    _fill_time(space, lst, 9, w_keywords, 'st_ctime', st.st_ctime)
+    # Note: 'w_keywords' contains the three attributes 'nsec_Xtime'.
+    # We have an app-level property in app_posix.stat_result to
+    # compute the full 'st_Xtime_ns' value.
 
-    for name, TYPE in EXTRA_FIELDS:
-        value = getattr(st, name)
-        w_value = space.wrap(value)
-        space.setitem(w_keywords, space.wrap(name), w_value)
+    # non-rounded values for name-based access
+    if stat_float_times:
+        space.setitem(w_keywords,
+                      space.wrap('st_atime'), space.wrap(st.st_atime))
+        space.setitem(w_keywords,
+                      space.wrap('st_mtime'), space.wrap(st.st_mtime))
+        space.setitem(w_keywords,
+                      space.wrap('st_ctime'), space.wrap(st.st_ctime))
+    #else:
+    #   filled by the __init__ method
 
     w_tuple = space.newtuple(lst)
     w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
                                   space.wrap('stat_result'))
     return space.call_function(w_stat_result, w_tuple, w_keywords)
 
-STATVFS_FIELDS = unrolling_iterable(enumerate(rposix_stat.STATVFS_FIELDS))
 
 def build_statvfs_result(space, st):
     vals_w = [None] * len(rposix_stat.STATVFS_FIELDS)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -126,9 +126,9 @@
         assert st[4] == st.st_uid
         assert st[5] == st.st_gid
         assert st[6] == st.st_size
-        assert st[7] == int(st.st_atime)
-        assert st[8] == int(st.st_mtime)
-        assert st[9] == int(st.st_ctime)
+        assert st[7] == int(st.st_atime)   # in complete corner cases, rounding
+        assert st[8] == int(st.st_mtime)   # here could maybe get the wrong
+        assert st[9] == int(st.st_ctime)   # integer...
 
         assert stat.S_IMODE(st.st_mode) & stat.S_IRUSR
         assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR
@@ -138,15 +138,17 @@
         assert st.st_size == 14
         assert st.st_nlink == 1
 
-        #if sys.platform.startswith('linux'):
-        #    # expects non-integer timestamps - it's unlikely that they are
-        #    # all three integers
-        #    assert ((st.st_atime, st.st_mtime, st.st_ctime) !=
-        #            (st[7],       st[8],       st[9]))
-        #    assert st.st_blksize * st.st_blocks >= st.st_size
         if sys.platform.startswith('linux'):
+            assert isinstance(st.st_atime, float)
+            assert isinstance(st.st_mtime, float)
+            assert isinstance(st.st_ctime, float)
             assert hasattr(st, 'st_rdev')
 
+        assert isinstance(st.st_atime_ns, int)
+        assert abs(st.st_atime_ns - 1e9*st.st_atime) < 500
+        assert abs(st.st_mtime_ns - 1e9*st.st_mtime) < 500
+        assert abs(st.st_ctime_ns - 1e9*st.st_ctime) < 500
+
     def test_stat_float_times(self):
         path = self.path
         posix = self.posix
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -242,13 +242,33 @@
     from rpython.rtyper.lltypesystem import lltype, rffi
     return space.wrap(rffi.cast(lltype.Signed, handle))
 
+getsizeof_missing = """sys.getsizeof() is not implemented on PyPy.
+
+A memory profiler using this function is most likely to give results
+inconsistent with reality on PyPy.  It would be possible to have
+sys.getsizeof() return a number (with enough work), but that may or
+may not represent how much memory the object uses.  It doesn't even
+make really sense to ask how much *one* object uses, in isolation
+with the rest of the system.  For example, instances have maps,
+which are often shared across many instances; in this case the maps
+would probably be ignored by an implementation of sys.getsizeof(),
+but their overhead is important in some cases if they are many
+instances with unique maps.  Conversely, equal strings may share
+their internal string data even if they are different objects---or
+empty containers may share parts of their internals as long as they
+are empty.  Even stranger, some lists create objects as you read
+them; if you try to estimate the size in memory of range(10**6) as
+the sum of all items' size, that operation will by itself create one
+million integer objects that never existed in the first place.
+"""
+
 def getsizeof(space, w_object, w_default=None):
-    """Not implemented on PyPy."""
     if w_default is None:
-        raise oefmt(space.w_TypeError,
-                    "sys.getsizeof() not implemented on PyPy")
+        raise oefmt(space.w_TypeError, getsizeof_missing)
     return w_default
 
+getsizeof.__doc__ = getsizeof_missing
+
 def intern(space, w_str):
     """``Intern'' the given string.  This enters the string in the (global)
 table of interned strings whose purpose is to speed up dictionary lookups.
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -480,3 +480,7 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_negative_array_size(self):
+        ffi = FFI()
+        py.test.raises(ValueError, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -503,3 +503,7 @@
     assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
     p = ffi.new("int[]", [-123456789])
     assert ffi.unpack(p, 1) == [-123456789]
+
+def test_negative_array_size():
+    ffi = _cffi1_backend.FFI()
+    py.test.raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1981,3 +1981,29 @@
         static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
+
+def test_typedef_array_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef int foo_t[...], bar_t[...];
+        int gv[...];
+        typedef int mat_t[...][...];
+        typedef int vmat_t[][...];
+        """)
+    lib = verify(ffi, "test_typedef_array_dotdotdot", """
+        typedef int foo_t[50], bar_t[50];
+        int gv[23];
+        typedef int mat_t[6][7];
+        typedef int vmat_t[][8];
+    """)
+    assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+    assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+    assert len(ffi.new("foo_t")) == 50
+    assert len(ffi.new("bar_t")) == 50
+    assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+    assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+    assert len(ffi.new("mat_t")) == 6
+    assert len(ffi.new("mat_t")[3]) == 7
+    py.test.raises(ffi.error, ffi.sizeof, "vmat_t")
+    p = ffi.new("vmat_t", 4)
+    assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror, exception_from_saved_errno
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
-from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat
+from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat, widen
 from rpython.rlib.rtime import (TIMEB, c_ftime,
                                 GETTIMEOFDAY_NO_TZ, TIMEVAL,
                                 HAVE_GETTIMEOFDAY, HAVE_FTIME)
@@ -290,12 +290,14 @@
                 if rffi.cast(rffi.LONG, errcode) == 0:
                     if w_info is not None:
                         _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True)
-                    return space.wrap(timeval.c_tv_sec + timeval.c_tv_usec * 1e-6)
+                    return space.wrap(
+                        widen(timeval.c_tv_sec) +
+                        widen(timeval.c_tv_usec) * 1e-6)
         if HAVE_FTIME:
             with lltype.scoped_alloc(TIMEB) as t:
                 c_ftime(t)
-                result = (float(intmask(t.c_time)) +
-                          float(intmask(t.c_millitm)) * 0.001)
+                result = (widen(t.c_time) +
+                          widen(t.c_millitm) * 0.001)
                 if w_info is not None:
                     _setinfo(space, w_info, "ftime()", 1e-3,
                              False, True) 
@@ -535,7 +537,7 @@
         if not allowNone:
             raise oefmt(space.w_TypeError, "tuple expected")
         # default to the current local time
-        tt = rffi.r_time_t(int(pytime.time()))
+        tt = rffi.cast(rffi.TIME_T, pytime.time())
         t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw')
         t_ref[0] = tt
         pbuf = c_localtime(t_ref)
@@ -737,7 +739,7 @@
 
 if HAS_CLOCK_GETTIME:
     def _timespec_to_seconds(timespec):
-        return int(timespec.c_tv_sec) + int(timespec.c_tv_nsec) * 1e-9
+        return widen(timespec.c_tv_sec) + widen(timespec.c_tv_nsec) * 1e-9
 
     @unwrap_spec(clk_id='c_int')
     def clock_gettime(space, clk_id):
@@ -751,8 +753,9 @@
     @unwrap_spec(clk_id='c_int', secs=float)
     def clock_settime(space, clk_id, secs):
         with lltype.scoped_alloc(TIMESPEC) as timespec:
-            frac = math.fmod(secs, 1.0)
-            rffi.setintfield(timespec, 'c_tv_sec', int(secs))
+            integer_secs = rffi.cast(TIMESPEC.c_tv_sec, secs)
+            frac = secs - widen(integer_secs)
+            rffi.setintfield(timespec, 'c_tv_sec', integer_secs)
             rffi.setintfield(timespec, 'c_tv_nsec', int(frac * 1e9))
             ret = c_clock_settime(clk_id, timespec)
             if ret != 0:
@@ -1051,16 +1054,15 @@
             return win_perf_counter(space, w_info=w_info)
         except ValueError:
             pass
-    value = _clock()
-    # Is this casting correct?
-    if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)):
+    value = widen(_clock())
+    if value == widen(rffi.cast(rposix.CLOCK_T, -1)):
         raise oefmt(space.w_RuntimeError,
                     "the processor time used is not available or its value"
                     "cannot be represented")
     if w_info is not None:
         _setinfo(space, w_info,
                  "clock()", 1.0 / CLOCKS_PER_SEC, True, False)
-    return space.wrap((1.0 * value) / CLOCKS_PER_SEC)
+    return space.wrap(float(value) / CLOCKS_PER_SEC)
 
 
 def _setinfo(space, w_info, impl, res, mono, adj):
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -23,7 +23,6 @@
 from pypy.objspace.std.util import get_positive_index
 from pypy.objspace.std.formatting import mod_format, FORMAT_BYTEARRAY
 
-NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d"
 
 
 class W_BytearrayObject(W_Root):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -288,6 +288,10 @@
             return W_SmallLongObject.fromint(val)
         return W_LongObject.fromint(self, val)
 
+    @specialize.argtype(1)
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+        return W_LongObject.fromrarith_int(val)
+
     def newlong_from_rbigint(self, val):
         return newlong(self, val)
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -26,7 +26,6 @@
         space.raises_w(space.w_OverflowError, space.float_w, w_big)
 
     def test_rint_variants(self):
-        py.test.skip("XXX broken!")
         from rpython.rtyper.tool.rfficache import platform
         space = self.space
         for r in platform.numbertype_to_rclass.values():
@@ -37,8 +36,8 @@
             for x in values:
                 if not r.SIGNED:
                     x &= r.MASK
-                w_obj = space.wrap(r(x))
-                assert space.bigint_w(w_obj).eq(rbigint.fromint(x))
+                w_obj = space.newlong_from_rarith_int(r(x))
+                assert space.bigint_w(w_obj).eq(rbigint.fromlong(x))
 
 
 class AppTestLong:
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -805,9 +805,7 @@
         class AA(object):
             __slots__ = ('a',)
         aa = AA()
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "aa.__class__ = A")
+        aa.__class__ = A
         raises(TypeError, "aa.__class__ = object")
         class Z1(A):
             pass
@@ -869,9 +867,13 @@
             __slots__ = ['a', 'b']
         class Order2(object):
             __slots__ = ['b', 'a']
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "Order1().__class__ = Order2")
+        Order1().__class__ = Order2
+
+        # like CPython, the order of slot names doesn't matter
+        x = Order1()
+        x.a, x.b = 1, 2
+        x.__class__ = Order2
+        assert (x.a, x.b) == (1, 2)
 
         class U1(object):
             __slots__ = ['a', 'b']
@@ -881,10 +883,11 @@
             __slots__ = ['a', 'b']
         class V2(V1):
             __slots__ = ['c', 'd', 'e']
-        # the following line does not work on CPython >= 2.6 either.
-        # that's just obscure.  Really really.  So we just ignore
-        # the whole issue until someone comes complaining.  Then we'll
-        # just kill slots altogether apart from maybe doing a few checks.
+        # the following line does not work on CPython either: we can't
+        # change a class if the old and new class have different layouts
+        # that look compatible but aren't, because they don't have the
+        # same base-layout class (even if these base classes are
+        # themselves compatible)...  obscure.
         raises(TypeError, "U2().__class__ = V2")
 
     def test_name(self):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -103,9 +103,10 @@
     """
     _immutable_ = True
 
-    def __init__(self, typedef, nslots, base_layout=None):
+    def __init__(self, typedef, nslots, newslotnames=[], base_layout=None):
         self.typedef = typedef
         self.nslots = nslots
+        self.newslotnames = newslotnames[:]    # make a fixed-size list
         self.base_layout = base_layout
 
     def issublayout(self, parent):
@@ -115,6 +116,12 @@
                 return False
         return True
 
+    def expand(self, hasdict, weakrefable):
+        """Turn this Layout into a tuple.  If two classes get equal
+        tuples, it means their instances have a fully compatible layout."""
+        return (self.typedef, self.newslotnames, self.base_layout,
+                hasdict, weakrefable)
+
 
 # possible values of compares_by_identity_status
 UNKNOWN = 0
@@ -287,8 +294,7 @@
 
     # compute a tuple that fully describes the instance layout
     def get_full_instance_layout(self):
-        layout = self.layout
-        return (layout, self.hasdict, self.weakrefable)
+        return self.layout.expand(self.hasdict, self.weakrefable)
 
     def compute_default_mro(self):
         return compute_C3_mro(self.space, self)
@@ -1022,11 +1028,15 @@
         w_self.weakrefable = w_self.weakrefable or w_base.weakrefable
     return hasoldstylebase
 
+
 def create_all_slots(w_self, hasoldstylebase, w_bestbase, force_new_layout):
+    from pypy.objspace.std.listobject import StringSort
+
     base_layout = w_bestbase.layout
     index_next_extra_slot = base_layout.nslots
     space = w_self.space
     dict_w = w_self.dict_w
+    newslotnames = []
     if '__slots__' not in dict_w:
         wantdict = True
         wantweakref = True
@@ -1052,9 +1062,22 @@
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
             else:
-                index_next_extra_slot = create_slot(w_self, w_slot_name,
-                                                    slot_name,
-                                                    index_next_extra_slot)
+                newslotnames.append(slot_name)
+        # Sort the list of names collected so far
+        sorter = StringSort(newslotnames, len(newslotnames))
+        sorter.sort()
+        # Try to create all slots in order.  The creation of some of
+        # them might silently fail; then we delete the name from the
+        # list.  At the end, 'index_next_extra_slot' has been advanced
+        # by the final length of 'newslotnames'.
+        i = 0
+        while i < len(newslotnames):
+            if create_slot(w_self, newslotnames[i], index_next_extra_slot):
+                index_next_extra_slot += 1
+                i += 1
+            else:
+                del newslotnames[i]
+    #
     wantdict = wantdict or hasoldstylebase
     if wantdict:
         create_dict_slot(w_self)
@@ -1063,35 +1086,34 @@
     if '__del__' in dict_w:
         w_self.hasuserdel = True
     #
+    assert index_next_extra_slot == base_layout.nslots + len(newslotnames)
     if index_next_extra_slot == base_layout.nslots and not force_new_layout:
         return base_layout
     else:
         return Layout(base_layout.typedef, index_next_extra_slot,
-                      base_layout=base_layout)
+                      newslotnames, base_layout=base_layout)
 
-def create_slot(w_self, w_slot_name, slot_name, index_next_extra_slot):
+def create_slot(w_self, slot_name, index_next_extra_slot):
     space = w_self.space
     if not valid_slot_name(slot_name):
         raise oefmt(space.w_TypeError, "__slots__ must be identifiers")
     # create member
     slot_name = mangle(slot_name, w_self.name)
-    if slot_name in w_self.dict_w:
-        w_prev = w_self.dict_w[slot_name]
-        if isinstance(w_prev, Member) and w_prev.w_cls is w_self:
-            pass   # special case: duplicate __slots__ entry, ignored
-                   # (e.g. occurs in datetime.py, fwiw)
-        else:
-            raise oefmt(space.w_ValueError,
-                        "%R in __slots__ conflicts with class variable",
-                        w_slot_name)
-    else:
+    if slot_name not in w_self.dict_w:
         # Force interning of slot names.
         slot_name = space.str_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
         member = Member(index_next_extra_slot, slot_name, w_self)
-        index_next_extra_slot += 1
         w_self.dict_w[slot_name] = space.wrap(member)
-    return index_next_extra_slot
+        return True
+    else:
+        w_prev = w_self.dict_w[slot_name]
+        if isinstance(w_prev, Member) and w_prev.w_cls is w_self:
+            return False   # special case: duplicate __slots__ entry, ignored
+                           # (e.g. occurs in datetime.py, fwiw)
+        raise oefmt(space.w_ValueError,
+                    "'%8' in __slots__ conflicts with class variable",
+                    slot_name)
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:
diff --git a/rpython/doc/jit/backend.rst b/rpython/doc/jit/backend.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/jit/backend.rst
@@ -0,0 +1,263 @@
+=========================
+PyPy's assembler backends
+=========================
+
+Draft notes about the organization of assembler backends in the PyPy JIT, in 2016
+=================================================================================
+
+
+input: linear sequence of instructions, called a "trace".
+
+A trace is a sequence of instructions in SSA form.  Most instructions
+correspond to one or a few CPU-level instructions.  There are a few
+meta-instructions like `label` and debugging stuff.  All branching is
+done with guards, which are instructions that check that a condition is
+true and exit the trace if not.  A failing guard can have a new trace
+added to it later, called a "bridge".  A patched guard becomes a direct
+`Jcond` instruction going to the bridge, with no indirection, no
+register spilling, etc.
+
+A trace ends with either a `return` or a `jump to label`.  The target
+label is either inside the same trace, or in some older one.  For
+historical reasons we call a "loop" a trace that is not a bridge.  The
+machine code that we generate is organized as a forest of trees; the
+trunk of the tree is a "loop", and the branches are all bridges
+(branching off the trunk or off another branch).
+
+* every trunk or branch that ends in a `jump to label` can target a
+  label from a different tree, too.
+
+* the whole process of assembling a loop or a branch is basically
+  single-threaded, so no synchronization issue there (including to patch
+  older generated instructions).
+
+* the generated assembler has got a "frame" in %rbp, which is actually
+  not on the stack at all, but is a GC object (called a "jitframe").
+  Spilling goes there.
+
+* the guards are `Jcond` to a very small piece of generated code, which
+  is basically pushing a couple of constants on the stack and then
+  jumping to the general guard-recovery code.  That code will save the
+  registers into the jitframe and then exit the whole generated
+  function.  The caller of that generated function checks how it
+  finished: if it finished by hitting a guard, then the caller is
+  responsible for calling the "blackhole interpreter".  This is the part
+  of the front-end that recovers from failing guards and finishes
+  running the frame (including, possibly, by jumping again into
+  generated assembler).
+
+
+Details about the JITting process:
+
+* front-end and optimization pass
+
+* rewrite (includes gc related transformation as well as simplifactions)
+
+* assembler generation
+
+
+Front-end and optimization pass
+-------------------------------
+
+Not discussed here in detail.  This produces loops and bridges using an
+instruction set that is "high-level" in some sense: it contains
+intructions like "new"/"new_array", and
+"setfield"/"setarrayitem"/"setinteriorfield" which describe the action
+of storing a value in a precise field of the structure or array.  For
+example, the "setfield" action might require implicitly a GC write
+barrier.  This is the high-level trace that we send to the following
+step.
+
+
+Rewrite
+-------
+
+A mostly but not completely CPU-independent phase: lowers some
+instructions.  For example, the variants of "new" are lowered to
+"malloc" and a few "gc_store": it bumps the pointer of the GC and then
+sets a few fields explicitly in the newly allocated structure.  The
+"setfield" is replaced with a "cond_gc_wb_call" (conditional call to the
+write barrier) if needed, followed by a "gc_store".
+
+The "gc_store" instruction can be encoded in a single MOV assembler
+instruction, but is not as flexible as a MOV.  The address is always
+specified as "some GC pointer + an offset".  We don't have the notion of
+interior pointer for GC objects.
+
+A different instruction, "gc_store_indexed", offers additional operands,
+which can be mapped to a single MOV instruction using forms like
+`[rax+8*rcx+24]`.
+
+Some other complex instructions pass through to the backend, which must
+deal with them: for example, "card marking" in the GC.  (Writing an
+object pointer inside an array would require walking the whole array
+later to find "young" references. Instead of that, we flip a bit for
+every range of 128 entries.  This is a common GC optimization.)  Setting
+the card bit of a GC object requires a sequence of assembler
+instructions that depends too much on the target CPU to be expressed
+explicitly here (moreover, it contains a few branches, which are hard to
+express at this level).
+
+
+Assembly
+--------
+
+No fancy code generation technique, but greedy forward pass that tries
+to avoid some pitfalls
+
+
+Handling instructions
+~~~~~~~~~~~~~~~~~~~~~
+
+* One by one (forward direction).   Each instruction asks the register
+  allocator to ensure that some arguments are in registers (not in the
+  jitframe); asks for a register to put its result into; and asks for
+  additional scratch registers that will be freed at  the end of the
+  instruction.  There is a special case for boolean variables: they are
+  stored in the condition code flags instead of being materialized as a
+  0/1 value.  (They are materialized later, except in the common case
+  where they are only used by the next `guard_false` or `guard_true` and
+  then forgotten.)
+
+* Instruction arguments are loaded into a register on demand.  This
+  makes the backend quite easy to write, but leads do some bad
+  decisions.
+
+
+Linear scan register allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Although it's always a linear trace that we consider, we don't use
+advanced techniques for register allocation: we do forward, on-demand
+allocation as the backend produces the assembler.  When it asks for a
+register to put some value into, we give it any free register, without
+consideration for what will be done with it later.  We compute the
+longevity of all variables, but only use it when choosing which register
+to spill (we spill the variable with the longest longevity).
+
+This works to some extend because it is well integrated with the earlier
+optimization pass. Loops are unrolled once by the optimization pass to
+allow more powerful optimizations---the optimization pass itself is the
+place that benefits the most, but it also has benefits here in the
+assembly pass.  These are:
+
+* The first peeling initializes the register binding on the first use.
+
+* This leads to an already allocated register of the trace loop.
+
+* As well as allocated registers when exiting bridges
+
+[Try to better allocate registers to match the ABI (minor to non benefit
+in the current state)]
+
+
+More complex mappings
+~~~~~~~~~~~~~~~~~~~~~
+
+Some instructions generate more complex code.  These are either or both of:
+
+* complex instructions generating some local control flow, like
+  "cond_gc_wb_call" (for write barriers), "call_assembler" (a call
+  followed by a few checks).
+
+* instructions that invoke custom assembler helpers, like the slow-path
+  of write barriers or the slow-path of allocations.  These slow-paths
+  are typically generated too, so that we are not constrained by the
+  usual calling conventions.
+
+
+GC pointers
+~~~~~~~~~~~
+
+Around most CALL instructions, we need to record a description of where
+the GC pointers are (registers and stack frame).  This is needed in case
+the CALL invokes a garbage collection.  The GC pointers can move; the
+pointers in the registers and stack frame are updated by the GC.  That's
+a reason for why we don't have explicit interior pointers.
+
+GC pointers can appear as constants in the trace.  We are busy changing
+that to use a constant table and `MOV REG, (%RIP+offset)`.  The
+"constant" in the table is actually updated by the GC if the object
+move.
+
+
+Vectorization
+~~~~~~~~~~~~~
+
+Optimization developed to use SIMD instructions for trace loops. Primary
+idea was to use it as an optimization of micro numpy. It has several
+passes on the already optimized trace.
+
+Shortly explained: It builds dependencies for an unrolled trace loop,
+gathering pairs/packs of operations that could be executed in parallel
+and finally schedules the operations.
+
+What did it add to the code base:
+
+* Dependencies can be constructed
+
+* Code motion of guards to relax dependencies
+
+* Scheduler to reorder trace
+
+* Array bound check removal (especially for unrolled traces)
+
+What can it do:
+
+* Transform vector loops (element wise operations)
+
+* Accumulation (`reduce([...],operator,0)`). Requires Operation to be
+  associative and commutative
+
+* SSE 4.1 as "vector backend"
+
+
+We do not
+~~~~~~~~~
+
+* Keep tracing data around to reoptimize the trace tree. (Once a trace
+  is compiled, minimal data is kept.)  This is one reason (there are
+  others in the front-end) for the following result: JIT-compiling a
+  small loop with two common paths ends up as one "loop" and one bridge
+  assembled, and the bridge-following path is slightly less efficient.
+  This is notably because this bridge is assembled with two constraints:
+  the input registers are fixed (from the guard), and the output
+  registers are fixed (from the jump target); usually these two sets of
+  fixed registers are different, and copying around is needed.
+
+* We don't join trace tails: we only assemble *trees*.
+
+* We don't do any reordering (neither of trace instructions nor of
+  individual assembler instructions)
+
+* We don't do any cross-instruction optimization that makes sense only
+  for the backend and can't easily be expressed at a higher level.  I'm
+  sure there are tons of examples of that, but e.g. loading a large
+  constant in a register that will survive for several instructions;
+  moving out of loops *parts* of some instruction like the address
+  calculation; etc. etc.
+
+* Other optimization opportunities I can think about: look at the
+  function prologue/epilogue; look at the overhead (small but not zero)
+  at the start of a bridge.  Also check if the way guards are
+  implemented makes sense.  Also, we generate large-ish sequences of
+  assembler instructions with tons of `Jcond` that are almost never
+  followed; any optimization opportunity there?  (They all go forward,
+  if it changes anything.)  In theory we could also replace some of
+  these with a signal handler on segfault (e.g. `guard_nonnull_class`).
+
+
+a GCC or LLVM backend?
+~~~~~~~~~~~~~~~~~~~~~~
+
+At least for comparison we'd like a JIT backend that emits its code
+using GCC or LLVM (irrespective of the time it would take).  But it's
+hard to map reasonably well the guards to the C language or to LLVM IR.
+The problems are: (1) we have many guards, we would like to avoid having
+many paths that each do a full
+saving-all-local-variables-that-are-still-alive; (2) it's hard to patch
+a guard when a bridge is compiled from it; (3) instructions like a CALL
+need to expose the local variables that are GC pointers; CALL_MAY_FORCE
+need to expose *all* local variables for optional off-line
+reconstruction of the interpreter state.
+
diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst
--- a/rpython/doc/jit/index.rst
+++ b/rpython/doc/jit/index.rst
@@ -26,6 +26,7 @@
    optimizer
    virtualizable
    vectorization
+   backend
 
 - :doc:`Overview <overview>`: motivating our approach
 
@@ -34,5 +35,8 @@
 - :doc:`Optimizer <optimizer>`: the step between tracing and writing
   machine code
 
-- :doc:`Virtulizable <virtualizable>` how virtualizables work and what they are
-  (in other words how to make frames more efficient).
+- :doc:`Virtualizable <virtualizable>`: how virtualizables work and what
+  they are (in other words how to make frames more efficient).
+
+- :doc:`Assembler backend <backend>`: draft notes about the organization
+  of the assembler backends
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -710,6 +710,12 @@
 
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
+
+    def filter(self, builder):
+        if not builder.cpu.supports_cond_call_value and \
+           self.opnum == rop.COND_CALL_VALUE_I:
+            raise CannotProduceOperation
+
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.test.test_random import check_random_function, Random
 from rpython.jit.backend.test.test_ll_random import LLtypeOperationBuilder
 from rpython.jit.backend.detect_cpu import getcpuclass
+from rpython.jit.metainterp.resoperation import rop
 import platform
 
 CPU = getcpuclass()
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -200,8 +200,12 @@
             or v.concretetype != lltype.Bool):
             return False
         for op in block.operations[::-1]:
-            if v in op.args:
-                return False   # variable is also used in cur block
+            # check if variable is used in block
+            for arg in op.args:
+                if arg == v:
+                    return False
+                if isinstance(arg, ListOfKind) and v in arg.content:
+                    return False
             if v is op.result:
                 if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne',
                                      'int_gt', 'int_ge',
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -243,6 +243,20 @@
         assert block.exitswitch == (opname, v1, '-live-before')
         assert block.exits == exits
 
+def test_optimize_goto_if_not__argument_to_call():
+    for opname in ['ptr_iszero', 'ptr_nonzero']:
+        v1 = Variable()
+        v3 = Variable(); v3.concretetype = lltype.Bool
+        v4 = Variable()
+        block = Block([v1])
+        callop = SpaceOperation('residual_call_r_i',
+                ["fake", ListOfKind('int', [v3])], v4)
+        block.operations = [SpaceOperation(opname, [v1], v3), callop]
+        block.exitswitch = v3
+        block.exits = exits = [FakeLink(False), FakeLink(True)]
+        res = Transformer().optimize_goto_if_not(block)
+        assert not res
+
 def test_symmetric():
     ops = {'int_add': 'int_add',
            'int_or': 'int_or',
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -90,18 +90,23 @@
         assert vs.make_inputargs(args, optimizer) == []
 
     def test_make_inputargs_2(self):
-        # Ensure that make_inputargs properly errors with VirtualStatesCantMatch
-        # when the type information for a virtual field conflicts. In practice the
-        # expected and given field always share a common subclass.
-        # This check is needed as not all paths to make_inputargs in unroll.py
-        # are guarded by a call to generate_guards.
+        # Ensure that make_inputargs does not error when the lengths of the fields
+        # for the runtime box does not match what the virtual state expected.
+        # This can occur in unroll.py, as not all paths to make_inputargs are
+        # guareded with a generalization_of check. The property is validated
+        # subsequently in all cases, so we just need to ensure that this case does
+        # not cause segfaults.
         optimizer = FakeOptimizer(self.cpu)
         classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
-        innervalue1 = info.InstancePtrInfo(known_class=classbox1, is_virtual=True, descr=self.valuedescr.get_parent_descr())
+        innervalue1 = info.InstancePtrInfo(
+                known_class=classbox1, is_virtual=True,
+                descr=self.valuedescr.get_parent_descr())
         for field in self.valuedescr.get_parent_descr().get_all_fielddescrs():
             innervalue1.setfield(field, None, ConstInt(42))
         classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3))
-        innervalue2 = info.InstancePtrInfo(known_class=classbox2, is_virtual=True, descr=self.valuedescr3.get_parent_descr())
+        innervalue2 = info.InstancePtrInfo(
+                known_class=classbox2, is_virtual=True,
+                descr=self.valuedescr3.get_parent_descr())
         for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs():
             innervalue2.setfield(field, None, ConstInt(42))
 
@@ -111,10 +116,14 @@
         nodebox2.set_forwarded(innervalue2)
 
         constr = VirtualStateConstructor(optimizer)
-        vs = constr.get_virtual_state([nodebox1])
+        vs1 = constr.get_virtual_state([nodebox1])
+        constr = VirtualStateConstructor(optimizer)
+        vs2 = constr.get_virtual_state([nodebox2])
 
-        with py.test.raises(VirtualStatesCantMatch):
-            args = vs.make_inputargs([nodebox2], optimizer, force_boxes=True)
+        # This should succeed with no exceptions
+        vs1.make_inputargs([nodebox2], optimizer, force_boxes=False)
+        assert not vs1.generalization_of(vs2, optimizer)
+        assert not vs2.generalization_of(vs1, optimizer)
 
     def test_position_generalization(self):
         def postest(info1, info2):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -167,7 +167,8 @@
                [self.get_box_replacement(x) for x in end_jump.getarglist()],
                self.optimizer, force_boxes=True)
             for arg in args:
-                self.optimizer.force_box(arg)
+                if arg is not None:
+                    self.optimizer.force_box(arg)
         except VirtualStatesCantMatch:
             raise InvalidLoop("Virtual states did not match "
                               "after picking the virtual state, when forcing"
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -177,14 +177,6 @@
     def _generalization_of_structpart(self, other):
         raise NotImplementedError
 
-    @staticmethod
-    def descr_issubclass(descr1, descr2, optimizer):
-        if not descr1.is_object() or not descr2.is_object():
-            return True
-        vtable1 = descr1.get_vtable()
-        vtable2 = descr2.get_vtable()
-        return optimizer._check_subclass(vtable1, vtable2)
-
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
@@ -193,13 +185,12 @@
         else:
             assert isinstance(info, AbstractStructPtrInfo)

From pypy.commits at gmail.com  Thu Sep 22 05:30:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 02:30:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: backport 146f531d6c86
Message-ID: <57e3a4b9.c4f6c20a.6a4d9.63bf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87310:29e007fd6daa
Date: 2016-09-22 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/29e007fd6daa/

Log:	backport 146f531d6c86

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1391,3 +1391,7 @@
         assert not self.compares_by_identity(X)
         del X.__eq__
         assert self.compares_by_identity(X)
+
+    def test_duplicate_slot_name(self):
+        class X:   # does not raise
+            __slots__ = 'a', 'a'
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1107,7 +1107,10 @@
         w_self.dict_w[slot_name] = space.wrap(member)
         return True
     else:
-        # never returns False, but that's to minimize the diff with pypy2
+        w_prev = w_self.dict_w[slot_name]
+        if isinstance(w_prev, Member) and w_prev.w_cls is w_self:
+            return False   # special case: duplicate __slots__ entry, ignored
+                           # (e.g. occurs in datetime.py, fwiw)
         raise oefmt(space.w_ValueError,
                     "'%8' in __slots__ conflicts with class variable",
                     slot_name)

From pypy.commits at gmail.com  Thu Sep 22 05:30:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 02:30:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57e3a4ba.861b1c0a.6d3a0.9d38@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87311:0ac497474106
Date: 2016-09-22 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/0ac497474106/

Log:	hg merge py3k


From pypy.commits at gmail.com  Thu Sep 22 05:43:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 02:43:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Skip this import test which I don't
 understand at all, and don't really
Message-ID: <57e3a7db.a468c20a.e0336.6466@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87312:dbbc9b12af87
Date: 2016-09-22 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/dbbc9b12af87/

Log:	Skip this import test which I don't understand at all, and don't
	really want to: it works fine on a translated pypy-c with -A

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -64,6 +64,7 @@
                     infinite_reload = "import infinite_reload, imp; imp.reload(infinite_reload)",
                     del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n",
                     _md5 = "hello_world = 42\n",
+                    _pypyjson = "hello_world = 42\n",
                     gc = "should_never_be_seen = 42\n",
                     )
     root.ensure("packagenamespace", dir=1)    # empty, no __init__.py
@@ -180,7 +181,7 @@
 
 class AppTestImport(BaseImportTest):
     spaceconfig = {
-        "usemodules": ['_md5', 'time', 'struct'],
+        "usemodules": ['_md5', 'time', 'struct', '_pypyjson'],
     }
 
     def setup_class(cls):
@@ -672,18 +673,19 @@
         del sys.modules['gc']
 
     def test_shadow_extension_1(self):
-        if self.runappdirect: skip("hard to test: module is already imported")
-        # 'import _md5' is supposed to find _md5.py if there is
+        if not self.runappdirect:
+            skip("I don't understand why it fails, but it works in -A "
+                 "on top of a translated PyPy.  Good enough...")
+        # 'import _pypyjson' is supposed to find _pypyjson.py if there is
         # one in sys.path.
         import sys
-        assert '_md5' not in sys.modules
+        assert '_pypyjson' not in sys.modules
         try:
-            import _md5
-            assert hasattr(_md5, 'hello_world')
-            assert not hasattr(_md5, 'md5')
-            assert '(built-in)' not in repr(_md5)
+            import _pypyjson
+            assert hasattr(_pypyjson, 'hello_world')
+            assert '(built-in)' not in repr(_pypyjson)
         finally:
-            sys.modules.pop('_md5', None)
+            sys.modules.pop('_pypyjson', None)
 
     def test_shadow_extension_2(self):
         if self.runappdirect: skip("hard to test: module is already imported")

From pypy.commits at gmail.com  Thu Sep 22 06:07:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 03:07:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix for sandbox tests
Message-ID: <57e3ad6d.411d1c0a.aea37.8e60@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87313:89527d477337
Date: 2016-09-22 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/89527d477337/

Log:	Fix for sandbox tests

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -147,13 +147,20 @@
         # (we ignore the extra values here for simplicity and portability)
         def stat_result_reduce(st):
             return (st[0], st[1], st[2], st[3], st[4],
-                    st[5], st[6], st[7], st[8], st[9],
-                    st[-3], st[-2], st[-1])
+                    st[5], st[6], st.st_atime, st.st_mtime, st.st_ctime)
 
         def stat_result_recreate(tup):
-            return make_stat_result(tup[:10] + extra_zeroes + tup[-3:])
+            atime, mtime, ctime = tup[7:]
+            result = tup[:7]
+            result += (int(atime), int(mtime), int(ctime))
+            result += extra_zeroes
+            result += (int((atime - result[7]) * 1e9),
+                       int((mtime - result[8]) * 1e9),
+                       int((ctime - result[9]) * 1e9))
+            return make_stat_result(result)
         s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE)
-                                       for name, TYPE in PORTABLE_STAT_FIELDS])
+                                    for name, TYPE in PORTABLE_STAT_FIELDS[:7]]
+                                 + 3 * [lltype_to_annotation(lltype.Float)])
         extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS) - 3)
         return s_reduced, stat_result_reduce, stat_result_recreate
 

From pypy.commits at gmail.com  Thu Sep 22 06:08:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 03:08:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57e3ada7.8a13c20a.66d39.724c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87314:03f142546f64
Date: 2016-09-22 12:07 +0200
http://bitbucket.org/pypy/pypy/changeset/03f142546f64/

Log:	hg merge default

diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -1,7 +1,6 @@
 from rpython.jit.backend.test.test_random import check_random_function, Random
 from rpython.jit.backend.test.test_ll_random import LLtypeOperationBuilder
 from rpython.jit.backend.detect_cpu import getcpuclass
-from rpython.jit.metainterp.resoperation import rop
 import platform
 
 CPU = getcpuclass()
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -147,13 +147,20 @@
         # (we ignore the extra values here for simplicity and portability)
         def stat_result_reduce(st):
             return (st[0], st[1], st[2], st[3], st[4],
-                    st[5], st[6], st[7], st[8], st[9],
-                    st[-3], st[-2], st[-1])
+                    st[5], st[6], st.st_atime, st.st_mtime, st.st_ctime)
 
         def stat_result_recreate(tup):
-            return make_stat_result(tup[:10] + extra_zeroes + tup[-3:])
+            atime, mtime, ctime = tup[7:]
+            result = tup[:7]
+            result += (int(atime), int(mtime), int(ctime))
+            result += extra_zeroes
+            result += (int((atime - result[7]) * 1e9),
+                       int((mtime - result[8]) * 1e9),
+                       int((ctime - result[9]) * 1e9))
+            return make_stat_result(result)
         s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE)
-                                       for name, TYPE in PORTABLE_STAT_FIELDS])
+                                    for name, TYPE in PORTABLE_STAT_FIELDS[:7]]
+                                 + 3 * [lltype_to_annotation(lltype.Float)])
         extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS) - 3)
         return s_reduced, stat_result_reduce, stat_result_recreate
 

From pypy.commits at gmail.com  Thu Sep 22 06:08:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 03:08:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57e3ada8.05d71c0a.b149d.9042@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87315:9d7f27e06a5c
Date: 2016-09-22 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/9d7f27e06a5c/

Log:	hg merge py3k

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -64,6 +64,7 @@
                     infinite_reload = "import infinite_reload, imp; imp.reload(infinite_reload)",
                     del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n",
                     _md5 = "hello_world = 42\n",
+                    _pypyjson = "hello_world = 42\n",
                     gc = "should_never_be_seen = 42\n",
                     )
     root.ensure("packagenamespace", dir=1)    # empty, no __init__.py
@@ -180,7 +181,7 @@
 
 class AppTestImport(BaseImportTest):
     spaceconfig = {
-        "usemodules": ['_md5', 'time', 'struct'],
+        "usemodules": ['_md5', 'time', 'struct', '_pypyjson'],
     }
 
     def setup_class(cls):
@@ -672,18 +673,19 @@
         del sys.modules['gc']
 
     def test_shadow_extension_1(self):
-        if self.runappdirect: skip("hard to test: module is already imported")
-        # 'import _md5' is supposed to find _md5.py if there is
+        if not self.runappdirect:
+            skip("I don't understand why it fails, but it works in -A "
+                 "on top of a translated PyPy.  Good enough...")
+        # 'import _pypyjson' is supposed to find _pypyjson.py if there is
         # one in sys.path.
         import sys
-        assert '_md5' not in sys.modules
+        assert '_pypyjson' not in sys.modules
         try:
-            import _md5
-            assert hasattr(_md5, 'hello_world')
-            assert not hasattr(_md5, 'md5')
-            assert '(built-in)' not in repr(_md5)
+            import _pypyjson
+            assert hasattr(_pypyjson, 'hello_world')
+            assert '(built-in)' not in repr(_pypyjson)
         finally:
-            sys.modules.pop('_md5', None)
+            sys.modules.pop('_pypyjson', None)
 
     def test_shadow_extension_2(self):
         if self.runappdirect: skip("hard to test: module is already imported")
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -1,7 +1,6 @@
 from rpython.jit.backend.test.test_random import check_random_function, Random
 from rpython.jit.backend.test.test_ll_random import LLtypeOperationBuilder
 from rpython.jit.backend.detect_cpu import getcpuclass
-from rpython.jit.metainterp.resoperation import rop
 import platform
 
 CPU = getcpuclass()
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -147,13 +147,20 @@
         # (we ignore the extra values here for simplicity and portability)
         def stat_result_reduce(st):
             return (st[0], st[1], st[2], st[3], st[4],
-                    st[5], st[6], st[7], st[8], st[9],
-                    st[-3], st[-2], st[-1])
+                    st[5], st[6], st.st_atime, st.st_mtime, st.st_ctime)
 
         def stat_result_recreate(tup):
-            return make_stat_result(tup[:10] + extra_zeroes + tup[-3:])
+            atime, mtime, ctime = tup[7:]
+            result = tup[:7]
+            result += (int(atime), int(mtime), int(ctime))
+            result += extra_zeroes
+            result += (int((atime - result[7]) * 1e9),
+                       int((mtime - result[8]) * 1e9),
+                       int((ctime - result[9]) * 1e9))
+            return make_stat_result(result)
         s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE)
-                                       for name, TYPE in PORTABLE_STAT_FIELDS])
+                                    for name, TYPE in PORTABLE_STAT_FIELDS[:7]]
+                                 + 3 * [lltype_to_annotation(lltype.Float)])
         extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS) - 3)
         return s_reduced, stat_result_reduce, stat_result_recreate
 

From pypy.commits at gmail.com  Thu Sep 22 08:39:30 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 22 Sep 2016 05:39:30 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: test used old vec_raw_store/load
Message-ID: <57e3d102.a970c20a.6630c.b1f6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87316:eb0230392430
Date: 2016-09-22 09:21 +0200
http://bitbucket.org/pypy/pypy/changeset/eb0230392430/

Log:	test used old vec_raw_store/load

diff --git a/rpython/jit/metainterp/test/test_resoperation.py b/rpython/jit/metainterp/test/test_resoperation.py
--- a/rpython/jit/metainterp/test/test_resoperation.py
+++ b/rpython/jit/metainterp/test/test_resoperation.py
@@ -117,7 +117,8 @@
     args = [rop.InputArgInt(), ConstInt(0)]
     baseop = rop.ResOperation(rop.rop.RAW_LOAD_I, args, descr=descr)
     baseop.set_forwarded(rop.VectorizationInfo(baseop))
-    op = rop.VecOperation(rop.rop.VEC_RAW_LOAD_I, args, baseop, 4, descr=descr)
+    op = rop.VecOperation(rop.rop.VEC_LOAD_I, args + [ConstInt(1), ConstInt(0)],
+                          baseop, 4, descr=descr)
     assert (op.type, op.datatype, op.bytesize, op.is_vector()) == ('i', 'i', 4, True)
 
 def test_vec_store():
@@ -126,7 +127,8 @@
     args = [rop.InputArgRef(), ConstInt(0), vec]
     baseop = rop.ResOperation(rop.rop.RAW_STORE,  args, descr=descr)
     baseop.set_forwarded(rop.VectorizationInfo(baseop))
-    op = rop.VecOperation(rop.rop.VEC_RAW_STORE, args, baseop, 2, descr=descr)
+    op = rop.VecOperation(rop.rop.VEC_STORE, args + [ConstInt(1), ConstInt(0)],
+                          baseop, 2, descr=descr)
     assert (op.type, op.datatype, op.bytesize, op.is_vector()) == ('v', 'v', 8, True)
 
 def test_vec_guard():

From pypy.commits at gmail.com  Thu Sep 22 08:39:32 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 22 Sep 2016 05:39:32 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57e3d104.6286c20a.dd2c3.b17c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87317:ce0e1fb75fc8
Date: 2016-09-22 09:23 +0200
http://bitbucket.org/pypy/pypy/changeset/ce0e1fb75fc8/

Log:	merge default

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -507,7 +507,7 @@
     def test_bug_1(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
-        q = ffi.new("char[]", "abcd")
+        q = ffi.new("char[]", b"abcd")
         p = ffi.cast("char(*)(void)", q)
         raises(TypeError, ffi.string, p)
 
diff --git a/rpython/doc/jit/backend.rst b/rpython/doc/jit/backend.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/jit/backend.rst
@@ -0,0 +1,263 @@
+=========================
+PyPy's assembler backends
+=========================
+
+Draft notes about the organization of assembler backends in the PyPy JIT, in 2016
+=================================================================================
+
+
+input: linear sequence of instructions, called a "trace".
+
+A trace is a sequence of instructions in SSA form.  Most instructions
+correspond to one or a few CPU-level instructions.  There are a few
+meta-instructions like `label` and debugging stuff.  All branching is
+done with guards, which are instructions that check that a condition is
+true and exit the trace if not.  A failing guard can have a new trace
+added to it later, called a "bridge".  A patched guard becomes a direct
+`Jcond` instruction going to the bridge, with no indirection, no
+register spilling, etc.
+
+A trace ends with either a `return` or a `jump to label`.  The target
+label is either inside the same trace, or in some older one.  For
+historical reasons we call a "loop" a trace that is not a bridge.  The
+machine code that we generate is organized as a forest of trees; the
+trunk of the tree is a "loop", and the branches are all bridges
+(branching off the trunk or off another branch).
+
+* every trunk or branch that ends in a `jump to label` can target a
+  label from a different tree, too.
+
+* the whole process of assembling a loop or a branch is basically
+  single-threaded, so no synchronization issue there (including to patch
+  older generated instructions).
+
+* the generated assembler has got a "frame" in %rbp, which is actually
+  not on the stack at all, but is a GC object (called a "jitframe").
+  Spilling goes there.
+
+* the guards are `Jcond` to a very small piece of generated code, which
+  is basically pushing a couple of constants on the stack and then
+  jumping to the general guard-recovery code.  That code will save the
+  registers into the jitframe and then exit the whole generated
+  function.  The caller of that generated function checks how it
+  finished: if it finished by hitting a guard, then the caller is
+  responsible for calling the "blackhole interpreter".  This is the part
+  of the front-end that recovers from failing guards and finishes
+  running the frame (including, possibly, by jumping again into
+  generated assembler).
+
+
+Details about the JITting process:
+
+* front-end and optimization pass
+
+* rewrite (includes gc related transformation as well as simplifactions)
+
+* assembler generation
+
+
+Front-end and optimization pass
+-------------------------------
+
+Not discussed here in detail.  This produces loops and bridges using an
+instruction set that is "high-level" in some sense: it contains
+intructions like "new"/"new_array", and
+"setfield"/"setarrayitem"/"setinteriorfield" which describe the action
+of storing a value in a precise field of the structure or array.  For
+example, the "setfield" action might require implicitly a GC write
+barrier.  This is the high-level trace that we send to the following
+step.
+
+
+Rewrite
+-------
+
+A mostly but not completely CPU-independent phase: lowers some
+instructions.  For example, the variants of "new" are lowered to
+"malloc" and a few "gc_store": it bumps the pointer of the GC and then
+sets a few fields explicitly in the newly allocated structure.  The
+"setfield" is replaced with a "cond_gc_wb_call" (conditional call to the
+write barrier) if needed, followed by a "gc_store".
+
+The "gc_store" instruction can be encoded in a single MOV assembler
+instruction, but is not as flexible as a MOV.  The address is always
+specified as "some GC pointer + an offset".  We don't have the notion of
+interior pointer for GC objects.
+
+A different instruction, "gc_store_indexed", offers additional operands,
+which can be mapped to a single MOV instruction using forms like
+`[rax+8*rcx+24]`.
+
+Some other complex instructions pass through to the backend, which must
+deal with them: for example, "card marking" in the GC.  (Writing an
+object pointer inside an array would require walking the whole array
+later to find "young" references. Instead of that, we flip a bit for
+every range of 128 entries.  This is a common GC optimization.)  Setting
+the card bit of a GC object requires a sequence of assembler
+instructions that depends too much on the target CPU to be expressed
+explicitly here (moreover, it contains a few branches, which are hard to
+express at this level).
+
+
+Assembly
+--------
+
+No fancy code generation technique, but greedy forward pass that tries
+to avoid some pitfalls
+
+
+Handling instructions
+~~~~~~~~~~~~~~~~~~~~~
+
+* One by one (forward direction).   Each instruction asks the register
+  allocator to ensure that some arguments are in registers (not in the
+  jitframe); asks for a register to put its result into; and asks for
+  additional scratch registers that will be freed at  the end of the
+  instruction.  There is a special case for boolean variables: they are
+  stored in the condition code flags instead of being materialized as a
+  0/1 value.  (They are materialized later, except in the common case
+  where they are only used by the next `guard_false` or `guard_true` and
+  then forgotten.)
+
+* Instruction arguments are loaded into a register on demand.  This
+  makes the backend quite easy to write, but leads do some bad
+  decisions.
+
+
+Linear scan register allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Although it's always a linear trace that we consider, we don't use
+advanced techniques for register allocation: we do forward, on-demand
+allocation as the backend produces the assembler.  When it asks for a
+register to put some value into, we give it any free register, without
+consideration for what will be done with it later.  We compute the
+longevity of all variables, but only use it when choosing which register
+to spill (we spill the variable with the longest longevity).
+
+This works to some extend because it is well integrated with the earlier
+optimization pass. Loops are unrolled once by the optimization pass to
+allow more powerful optimizations---the optimization pass itself is the
+place that benefits the most, but it also has benefits here in the
+assembly pass.  These are:
+
+* The first peeling initializes the register binding on the first use.
+
+* This leads to an already allocated register of the trace loop.
+
+* As well as allocated registers when exiting bridges
+
+[Try to better allocate registers to match the ABI (minor to non benefit
+in the current state)]
+
+
+More complex mappings
+~~~~~~~~~~~~~~~~~~~~~
+
+Some instructions generate more complex code.  These are either or both of:
+
+* complex instructions generating some local control flow, like
+  "cond_gc_wb_call" (for write barriers), "call_assembler" (a call
+  followed by a few checks).
+
+* instructions that invoke custom assembler helpers, like the slow-path
+  of write barriers or the slow-path of allocations.  These slow-paths
+  are typically generated too, so that we are not constrained by the
+  usual calling conventions.
+
+
+GC pointers
+~~~~~~~~~~~
+
+Around most CALL instructions, we need to record a description of where
+the GC pointers are (registers and stack frame).  This is needed in case
+the CALL invokes a garbage collection.  The GC pointers can move; the
+pointers in the registers and stack frame are updated by the GC.  That's
+a reason for why we don't have explicit interior pointers.
+
+GC pointers can appear as constants in the trace.  We are busy changing
+that to use a constant table and `MOV REG, (%RIP+offset)`.  The
+"constant" in the table is actually updated by the GC if the object
+move.
+
+
+Vectorization
+~~~~~~~~~~~~~
+
+Optimization developed to use SIMD instructions for trace loops. Primary
+idea was to use it as an optimization of micro numpy. It has several
+passes on the already optimized trace.
+
+Shortly explained: It builds dependencies for an unrolled trace loop,
+gathering pairs/packs of operations that could be executed in parallel
+and finally schedules the operations.
+
+What did it add to the code base:
+
+* Dependencies can be constructed
+
+* Code motion of guards to relax dependencies
+
+* Scheduler to reorder trace
+
+* Array bound check removal (especially for unrolled traces)
+
+What can it do:
+
+* Transform vector loops (element wise operations)
+
+* Accumulation (`reduce([...],operator,0)`). Requires Operation to be
+  associative and commutative
+
+* SSE 4.1 as "vector backend"
+
+
+We do not
+~~~~~~~~~
+
+* Keep tracing data around to reoptimize the trace tree. (Once a trace
+  is compiled, minimal data is kept.)  This is one reason (there are
+  others in the front-end) for the following result: JIT-compiling a
+  small loop with two common paths ends up as one "loop" and one bridge
+  assembled, and the bridge-following path is slightly less efficient.
+  This is notably because this bridge is assembled with two constraints:
+  the input registers are fixed (from the guard), and the output
+  registers are fixed (from the jump target); usually these two sets of
+  fixed registers are different, and copying around is needed.
+
+* We don't join trace tails: we only assemble *trees*.
+
+* We don't do any reordering (neither of trace instructions nor of
+  individual assembler instructions)
+
+* We don't do any cross-instruction optimization that makes sense only
+  for the backend and can't easily be expressed at a higher level.  I'm
+  sure there are tons of examples of that, but e.g. loading a large
+  constant in a register that will survive for several instructions;
+  moving out of loops *parts* of some instruction like the address
+  calculation; etc. etc.
+
+* Other optimization opportunities I can think about: look at the
+  function prologue/epilogue; look at the overhead (small but not zero)
+  at the start of a bridge.  Also check if the way guards are
+  implemented makes sense.  Also, we generate large-ish sequences of
+  assembler instructions with tons of `Jcond` that are almost never
+  followed; any optimization opportunity there?  (They all go forward,
+  if it changes anything.)  In theory we could also replace some of
+  these with a signal handler on segfault (e.g. `guard_nonnull_class`).
+
+
+a GCC or LLVM backend?
+~~~~~~~~~~~~~~~~~~~~~~
+
+At least for comparison we'd like a JIT backend that emits its code
+using GCC or LLVM (irrespective of the time it would take).  But it's
+hard to map reasonably well the guards to the C language or to LLVM IR.
+The problems are: (1) we have many guards, we would like to avoid having
+many paths that each do a full
+saving-all-local-variables-that-are-still-alive; (2) it's hard to patch
+a guard when a bridge is compiled from it; (3) instructions like a CALL
+need to expose the local variables that are GC pointers; CALL_MAY_FORCE
+need to expose *all* local variables for optional off-line
+reconstruction of the interpreter state.
+
diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst
--- a/rpython/doc/jit/index.rst
+++ b/rpython/doc/jit/index.rst
@@ -26,6 +26,7 @@
    optimizer
    virtualizable
    vectorization
+   backend
 
 - :doc:`Overview <overview>`: motivating our approach
 
@@ -34,5 +35,8 @@
 - :doc:`Optimizer <optimizer>`: the step between tracing and writing
   machine code
 
-- :doc:`Virtulizable <virtualizable>` how virtualizables work and what they are
-  (in other words how to make frames more efficient).
+- :doc:`Virtualizable <virtualizable>`: how virtualizables work and what
+  they are (in other words how to make frames more efficient).
+
+- :doc:`Assembler backend <backend>`: draft notes about the organization
+  of the assembler backends
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -710,6 +710,12 @@
 
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
+
+    def filter(self, builder):
+        if not builder.cpu.supports_cond_call_value and \
+           self.opnum == rop.COND_CALL_VALUE_I:
+            raise CannotProduceOperation
+
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -359,12 +359,13 @@
     tpe.members[n] = lltype.nullptr(FFI_TYPE_P.TO)
     return tpe
 
+ at specialize.memo()
 def cast_type_to_ffitype(tp):
     """ This function returns ffi representation of rpython type tp
     """
     return TYPE_MAP[tp]
-cast_type_to_ffitype._annspecialcase_ = 'specialize:memo'
 
+ at specialize.argtype(1)
 def push_arg_as_ffiptr(ffitp, arg, ll_buf):
     # This is for primitive types.  Note that the exact type of 'arg' may be
     # different from the expected 'c_size'.  To cope with that, we fall back
@@ -396,7 +397,6 @@
                 arg >>= 8
         else:
             raise AssertionError
-push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)'
 
 
 # type defs for callback and closure userdata
@@ -470,12 +470,12 @@
 FUNCFLAG_USE_ERRNO = 8
 FUNCFLAG_USE_LASTERROR = 16
 
+ at specialize.arg(1)     # hack :-/
 def get_call_conv(flags, from_jit):
     if _WIN32 and not _WIN64 and (flags & FUNCFLAG_CDECL == 0):
         return FFI_STDCALL
     else:
         return FFI_DEFAULT_ABI
-get_call_conv._annspecialcase_ = 'specialize:arg(1)'     # hack :-/
 
 
 class AbstractFuncPtr(object):
@@ -599,6 +599,7 @@
         else:
             self.restype_size = -1
 
+    @specialize.argtype(1)
     def push_arg(self, value):
         #if self.pushed_args == self.argnum:
         #    raise TypeError("Too many arguments, eats %d, pushed %d" %
@@ -618,7 +619,6 @@
         push_arg_as_ffiptr(self.argtypes[self.pushed_args], value,
                            self.ll_args[self.pushed_args])
         self.pushed_args += 1
-    push_arg._annspecialcase_ = 'specialize:argtype(1)'
 
     def _check_args(self):
         if self.pushed_args < self.argnum:
@@ -627,6 +627,7 @@
     def _clean_args(self):
         self.pushed_args = 0
 
+    @specialize.arg(1)
     def call(self, RES_TP):
         self._check_args()
         ffires = c_ffi_call(self.ll_cif, self.funcsym,
@@ -645,7 +646,6 @@
         self._clean_args()
         check_fficall_result(ffires, self.flags)
         return res
-    call._annspecialcase_ = 'specialize:arg(1)'
 
     def __del__(self):
         if self.ll_args:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -280,6 +280,7 @@
 
 
 @oopspec("jit.isconstant(value)")
+ at specialize.call_location()
 def isconstant(value):
     """
     While tracing, returns whether or not the value is currently known to be
@@ -289,9 +290,9 @@
     This is for advanced usage only.
     """
     return NonConstant(False)
-isconstant._annspecialcase_ = "specialize:call_location"
 
 @oopspec("jit.isvirtual(value)")
+ at specialize.call_location()
 def isvirtual(value):
     """
     Returns if this value is virtual, while tracing, it's relatively
@@ -300,7 +301,6 @@
     This is for advanced usage only.
     """
     return NonConstant(False)
-isvirtual._annspecialcase_ = "specialize:call_location"
 
 @specialize.call_location()
 def loop_unrolling_heuristic(lst, size, cutoff=2):
@@ -401,28 +401,27 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Signed, _we_are_jitted)
 
-
+ at oopspec('jit.current_trace_length()')
 def current_trace_length():
     """During JIT tracing, returns the current trace length (as a constant).
     If not tracing, returns -1."""
     if NonConstant(False):
         return 73
     return -1
-current_trace_length.oopspec = 'jit.current_trace_length()'
 
+ at oopspec('jit.debug(string, arg1, arg2, arg3, arg4)')
 def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1,
                       arg3=-sys.maxint-1, arg4=-sys.maxint-1):
     """When JITted, cause an extra operation JIT_DEBUG to appear in
     the graphs.  Should not be left after debugging."""
     keepalive_until_here(string) # otherwise the whole function call is removed
-jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)'
 
+ at oopspec('jit.assert_green(value)')
+ at specialize.argtype(0)
 def assert_green(value):
     """Very strong assert: checks that 'value' is a green
     (a JIT compile-time constant)."""
     keepalive_until_here(value)
-assert_green._annspecialcase_ = 'specialize:argtype(0)'
-assert_green.oopspec = 'jit.assert_green(value)'
 
 class AssertGreenFailed(Exception):
     pass
@@ -457,6 +456,7 @@
 # ____________________________________________________________
 # VRefs
 
+ at oopspec('virtual_ref(x)')
 @specialize.argtype(0)
 def virtual_ref(x):
     """Creates a 'vref' object that contains a reference to 'x'.  Calls
@@ -467,14 +467,13 @@
     dereferenced (by the call syntax 'vref()'), it returns 'x', which is
     then forced."""
     return DirectJitVRef(x)
-virtual_ref.oopspec = 'virtual_ref(x)'
 
+ at oopspec('virtual_ref_finish(x)')
 @specialize.argtype(1)
 def virtual_ref_finish(vref, x):
     """See docstring in virtual_ref(x)"""
     keepalive_until_here(x)   # otherwise the whole function call is removed
     _virtual_ref_finish(vref, x)
-virtual_ref_finish.oopspec = 'virtual_ref_finish(x)'
 
 def non_virtual_ref(x):
     """Creates a 'vref' that just returns x when called; nothing more special.
@@ -825,6 +824,7 @@
     jit_opencoder_model
     """
 
+ at specialize.arg(0)
 def set_user_param(driver, text):
     """Set the tunable JIT parameters from a user-supplied string
     following the format 'param=value,param=value', or 'off' to
@@ -860,7 +860,6 @@
                     break
             else:
                 raise ValueError
-set_user_param._annspecialcase_ = 'specialize:arg(0)'
 
 # ____________________________________________________________
 #
diff --git a/rpython/rlib/listsort.py b/rpython/rlib/listsort.py
--- a/rpython/rlib/listsort.py
+++ b/rpython/rlib/listsort.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rlib.objectmodel import specialize
 
 
 ## ------------------------------------------------------------------------
@@ -141,6 +142,12 @@
         # or, IOW, the first k elements of a should precede key, and the last
         # n-k should follow key.
 
+        # hint for the annotator: the argument 'rightmost' is always passed in as
+        # a constant (either True or False), so we can specialize the function for
+        # the two cases.  (This is actually needed for technical reasons: the
+        # variable 'lower' must contain a known method, which is the case in each
+        # specialized version but not in the unspecialized one.)
+        @specialize.arg(4)
         def gallop(self, key, a, hint, rightmost):
             assert 0 <= hint < a.len
             if rightmost:
@@ -212,12 +219,6 @@
             assert lastofs == ofs         # so a[ofs-1] < key <= a[ofs]
             return ofs
 
-        # hint for the annotator: the argument 'rightmost' is always passed in as
-        # a constant (either True or False), so we can specialize the function for
-        # the two cases.  (This is actually needed for technical reasons: the
-        # variable 'lower' must contain a known method, which is the case in each
-        # specialized version but not in the unspecialized one.)
-        gallop._annspecialcase_ = "specialize:arg(4)"
 
         # ____________________________________________________________
 
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -37,6 +37,7 @@
 from rpython.rlib import objectmodel
 from rpython.flowspace.model import Constant, const
 from rpython.flowspace.specialcase import register_flow_sc
+from rpython.rlib.objectmodel import specialize
 
 """
 Long-term target:
@@ -135,14 +136,15 @@
     # We deal directly with overflow there anyway.
     return r_longlonglong(n)
 
+ at specialize.argtype(0)
 def widen(n):
     from rpython.rtyper.lltypesystem import lltype
     if _should_widen_type(lltype.typeOf(n)):
         return intmask(n)
     else:
         return n
-widen._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def _should_widen_type(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Bool:
@@ -153,19 +155,18 @@
     assert issubclass(r_class, base_int)
     return r_class.BITS < LONG_BIT or (
         r_class.BITS == LONG_BIT and r_class.SIGNED)
-_should_widen_type._annspecialcase_ = 'specialize:memo'
 
 # the replacement for sys.maxint
 maxint = int(LONG_TEST - 1)
 # for now, it should be equal to sys.maxint on all supported platforms
 assert maxint == sys.maxint
 
+ at specialize.argtype(0)
 def is_valid_int(r):
     if objectmodel.we_are_translated():
         return isinstance(r, int)
     return isinstance(r, (base_int, int, long, bool)) and (
         -maxint - 1 <= r <= maxint)
-is_valid_int._annspecialcase_ = 'specialize:argtype(0)'
 
 def ovfcheck(r):
     "NOT_RPYTHON"
@@ -225,12 +226,12 @@
         return build_int(None, self_type.SIGNED, max(self_type.BITS, other_type.BITS))
     raise AssertionError("Merging these types (%s, %s) is not supported" % (self_type, other_type))
 
+ at specialize.memo()
 def signedtype(t):
     if t in (bool, int, long):
         return True
     else:
         return t.SIGNED
-signedtype._annspecialcase_ = 'specialize:memo'
 
 def normalizedinttype(t):
     if t is int:
@@ -241,11 +242,12 @@
         assert t.BITS <= r_longlong.BITS
         return build_int(None, t.SIGNED, r_longlong.BITS)
 
+ at specialize.argtype(0)
 def most_neg_value_of_same_type(x):
     from rpython.rtyper.lltypesystem import lltype
     return most_neg_value_of(lltype.typeOf(x))
-most_neg_value_of_same_type._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def most_neg_value_of(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -256,13 +258,13 @@
         return r_class(-(r_class.MASK >> 1) - 1)
     else:
         return r_class(0)
-most_neg_value_of._annspecialcase_ = 'specialize:memo'
 
+ at specialize.argtype(0)
 def most_pos_value_of_same_type(x):
     from rpython.rtyper.lltypesystem import lltype
     return most_pos_value_of(lltype.typeOf(x))
-most_pos_value_of_same_type._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def most_pos_value_of(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -273,8 +275,8 @@
         return r_class(r_class.MASK >> 1)
     else:
         return r_class(r_class.MASK)
-most_pos_value_of._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def is_signed_integer_type(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -284,7 +286,6 @@
         return r_class.SIGNED
     except KeyError:
         return False   # not an integer type
-is_signed_integer_type._annspecialcase_ = 'specialize:memo'
 
 def highest_bit(n):
     """
@@ -676,7 +677,7 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_mod(lltype.Signed, x, y)
 
- at objectmodel.specialize.ll()
+ at specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
     """
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -90,16 +90,16 @@
 
 FIVEARY_CUTOFF = 8
 
+ at specialize.argtype(0)
 def _mask_digit(x):
     return UDIGIT_MASK(x & MASK)
-_mask_digit._annspecialcase_ = 'specialize:argtype(0)'
 
 def _widen_digit(x):
     return rffi.cast(LONG_TYPE, x)
 
+ at specialize.argtype(0)
 def _store_digit(x):
     return rffi.cast(STORE_TYPE, x)
-_store_digit._annspecialcase_ = 'specialize:argtype(0)'
 
 def _load_unsigned_digit(x):
     return rffi.cast(UNSIGNED_TYPE, x)
@@ -175,11 +175,11 @@
         return _load_unsigned_digit(self._digits[x])
     udigit._always_inline_ = True
 
+    @specialize.argtype(2)
     def setdigit(self, x, val):
         val = _mask_digit(val)
         assert val >= 0
         self._digits[x] = _store_digit(val)
-    setdigit._annspecialcase_ = 'specialize:argtype(2)'
     setdigit._always_inline_ = True
 
     def numdigits(self):
@@ -1312,6 +1312,7 @@
 
     return res
 
+ at specialize.argtype(0)
 def digits_from_nonneg_long(l):
     digits = []
     while True:
@@ -1319,8 +1320,8 @@
         l = l >> SHIFT
         if not l:
             return digits[:] # to make it non-resizable
-digits_from_nonneg_long._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def digits_for_most_neg_long(l):
     # This helper only works if 'l' is the most negative integer of its
     # type, which in base 2 looks like: 1000000..0000
@@ -1335,8 +1336,8 @@
     assert l & MASK == l
     digits.append(_store_digit(l))
     return digits[:] # to make it non-resizable
-digits_for_most_neg_long._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def args_from_rarith_int1(x):
     if x > 0:
         return digits_from_nonneg_long(x), 1
@@ -1348,11 +1349,10 @@
     else:
         # the most negative integer! hacks needed...
         return digits_for_most_neg_long(x), -1
-args_from_rarith_int1._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def args_from_rarith_int(x):
     return args_from_rarith_int1(widen(x))
-args_from_rarith_int._annspecialcase_ = "specialize:argtype(0)"
 # ^^^ specialized by the precise type of 'x', which is typically a r_xxx
 #     instance from rlib.rarithmetic
 
@@ -1909,6 +1909,7 @@
         i += 1
     return borrow
 
+ at specialize.argtype(2)
 def _muladd1(a, n, extra=0):
     """Multiply by a single digit and add a single digit, ignoring the sign.
     """
@@ -1926,7 +1927,7 @@
     z.setdigit(i, carry)
     z._normalize()
     return z
-_muladd1._annspecialcase_ = "specialize:argtype(2)"
+
 def _v_lshift(z, a, m, d):
     """ Shift digit vector a[0:m] d bits left, with 0 <= d < SHIFT. Put
         * result in z[0:m], and return the d bits shifted out of the top.
@@ -2178,6 +2179,7 @@
         ad = -ad
     return ad
 
+ at specialize.arg(0)
 def _loghelper(func, arg):
     """
     A decent logarithm is easy to compute even for huge bigints, but libm can't
@@ -2195,7 +2197,6 @@
     # CAUTION:  e*SHIFT may overflow using int arithmetic,
     # so force use of double. */
     return func(x) + (e * float(SHIFT) * func(2.0))
-_loghelper._annspecialcase_ = 'specialize:arg(0)'
 
 # ____________________________________________________________
 
@@ -2519,6 +2520,7 @@
     return output.build()
 
 
+ at specialize.arg(1)
 def _bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
@@ -2598,8 +2600,8 @@
         return z
 
     return z.invert()
-_bitwise._annspecialcase_ = "specialize:arg(1)"
-
+
+ at specialize.arg(1)
 def _int_bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
@@ -2682,7 +2684,6 @@
         return z
 
     return z.invert()
-_int_bitwise._annspecialcase_ = "specialize:arg(1)"
 
 ULONGLONG_BOUND = r_ulonglong(1L << (r_longlong.BITS-1))
 LONGLONG_MIN = r_longlong(-(1L << (r_longlong.BITS-1)))
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.nonconst import NonConstant
 from rpython.rlib.rarithmetic import intmask
 
@@ -239,12 +239,12 @@
     _, _VirtualProtect_safe = winexternal('VirtualProtect',
                                   [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD],
                                   BOOL)
+    @specialize.ll()
     def VirtualProtect(addr, size, mode, oldmode_ptr):
         return _VirtualProtect_safe(addr,
                                rffi.cast(rffi.SIZE_T, size),
                                rffi.cast(DWORD, mode),
                                oldmode_ptr)
-    VirtualProtect._annspecialcase_ = 'specialize:ll'
     VirtualFree, VirtualFree_safe = winexternal('VirtualFree',
                               [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL)
 
diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -7,6 +7,7 @@
 from struct import unpack
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 from rpython.rlib.rstruct.error import StructError
+from rpython.rlib.objectmodel import specialize
 
 class MasterReader(object):
     def __init__(self, s):
@@ -99,14 +100,14 @@
         self._create_unpacking_func()
         return True
 
+ at specialize.memo()
 def create_unpacker(unpack_str):
     fmtiter = FrozenUnpackIterator(unpack_str)
     fmtiter.interpret(unpack_str)
     assert fmtiter._freeze_()
     return fmtiter
-create_unpacker._annspecialcase_ = 'specialize:memo'
 
+ at specialize.arg(0)
 def runpack(fmt, input):
     unpacker = create_unpacker(fmt)
     return unpacker.unpack(input)
-runpack._annspecialcase_ = 'specialize:arg(0)'

From pypy.commits at gmail.com  Thu Sep 22 08:39:34 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 22 Sep 2016 05:39:34 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: try to align the loop (should
 improve ppc and s390x load/store performance
Message-ID: <57e3d106.6740c20a.64c87.ad5a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87318:07a0b3bdccbe
Date: 2016-09-22 14:38 +0200
http://bitbucket.org/pypy/pypy/changeset/07a0b3bdccbe/

Log:	try to align the loop (should improve ppc and s390x load/store
	performance

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -879,6 +879,9 @@
     def bh_vec_int_xor(self, vx, vy, count):
         return [int(x) ^ int(y) for x,y in zip(vx,vy)]
 
+    def bh_vec_float_xor(self, vx, vy, count):
+        return [0.0 for x,y in zip(vx,vy)] # just used for clearing the vector register
+
     def bh_vec_cast_float_to_singlefloat(self, vx, count):
         from rpython.rlib.rarithmetic import r_singlefloat
         return [longlong.singlefloat2int(r_singlefloat(longlong.getrealfloat(v)))
diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py
--- a/rpython/jit/backend/llsupport/vector_ext.py
+++ b/rpython/jit/backend/llsupport/vector_ext.py
@@ -218,6 +218,8 @@
 
 class VectorExt(object):
 
+    should_align_unroll = True
+
     def __init__(self):
         self._enabled = False
         self.register_size = 0 # in bytes
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -42,6 +42,9 @@
         return "<TempVector At %s>" % (id(self),)
 
 class X86VectorExt(VectorExt):
+
+    should_align_unroll = True
+
     def setup_once(self, asm):
         if detect_feature.detect_sse4_1():
             self.enable(16, accum=True)
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -943,10 +943,6 @@
     """
     exec py.code.Source(multiplicative_func_source
             .format(name='INT_MUL', op='*', tgt='mul', cop='*')).compile()
-    #exec py.code.Source(multiplicative_func_source
-    #        .format(name='INT_PY_DIV', op='*', tgt='div', cop='/')).compile()
-    #exec py.code.Source(multiplicative_func_source
-    #        .format(name='UINT_FLOORDIV', op='*', tgt='div', cop='/')).compile()
     del multiplicative_func_source
 
     array_access_source = """
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -234,7 +234,10 @@
 
         # unroll
         self.unroll_count = self.get_unroll_count(vsize)
-        self.unroll_loop_iterations(loop, self.unroll_count)
+        align_unroll = self.unroll_count==1 and \
+                       self.vector_ext.should_align_unroll
+        self.unroll_loop_iterations(info, loop, self.unroll_count,
+                                    align_unroll_once=align_unroll)
 
         # vectorize
         graph = DependencyGraph(loop)
@@ -256,23 +259,32 @@
 
         return loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
 
-    def unroll_loop_iterations(self, loop, unroll_count):
-        """ Unroll the loop X times. unroll_count + 1 = unroll_factor """
+    def unroll_loop_iterations(self, info, loop, unroll_count, align_unroll_once=False):
+        """ Unroll the loop `unroll_count` times. There can be an additional unroll step
+            if alignment might benefit """
         numops = len(loop.operations)
 
         renamer = Renamer()
         operations = loop.operations
+        orig_jump_args = loop.jump.getarglist()[:]
+        prohibit_opnums = (rop.GUARD_FUTURE_CONDITION,
+                           rop.GUARD_NOT_INVALIDATED,
+                           rop.DEBUG_MERGE_POINT)
         unrolled = []
-        prohibit_opnums = (rop.GUARD_FUTURE_CONDITION,
-                           rop.GUARD_NOT_INVALIDATED)
-        orig_jump_args = loop.jump.getarglist()[:]
+
+        if align_unroll_once:
+            unroll_count += 1
+
         # it is assumed that #label_args == #jump_args
         label_arg_count = len(orig_jump_args)
+        label = loop.label
+        jump = loop.jump
+        new_label = loop.label
         for u in range(unroll_count):
             # fill the map with the renaming boxes. keys are boxes from the label
             for i in range(label_arg_count):
-                la = loop.label.getarg(i)
-                ja = loop.jump.getarg(i)
+                la = label.getarg(i)
+                ja = jump.getarg(i)
                 ja = renamer.rename_box(ja)
                 if la != ja:
                     renamer.start_renaming(la, ja)
@@ -294,9 +306,16 @@
                 # to be adjusted. rd_snapshot stores the live variables
                 # that are needed to resume.
                 if copied_op.is_guard():
-                    self.copy_guard_descr(renamer, copied_op)
+                    self.copy_guard_descr(renamer, copied_op, align_unroll_once and u == 0)
                 #
                 unrolled.append(copied_op)
+            #
+            if align_unroll_once and u == 0:
+                descr = label.getdescr()
+                args = label.getarglist()[:]
+                new_label = ResOperation(rop.LABEL, args, descr)
+                renamer.rename(new_label)
+            #
 
         # the jump arguments have been changed
         # if label(iX) ... jump(i(X+1)) is called, at the next unrolled loop
@@ -306,7 +325,25 @@
             value = renamer.rename_box(arg)
             loop.jump.setarg(i, value)
         #
-        loop.operations = operations + unrolled
+        loop.label = new_label
+        if align_unroll_once:
+            for op in operations:
+                descr = op.getdescr()
+                if descr:
+                    # the first step of the optimization will overwrite the descr
+                    # with a compile loop version descr
+                    # in the operations to align the loop load/store ops we want the original
+                    # descr saved on the forwarded info
+                    vinfo = copied_op.get_forwarded()
+                    if vinfo:
+                        assert isinstance(vinfo, VectorizationInfo)
+                        descr = vinfo.get_old_descr()
+                        assert descr is not None
+                        op.setdescr(descr)
+            info.extra_same_as += operations
+            loop.operations = unrolled
+        else:
+            loop.operations = operations + unrolled
 
     def copy_guard_descr(self, renamer, copied_op):
         descr = copied_op.getdescr()
@@ -316,6 +353,8 @@
             failargs = renamer.rename_failargs(copied_op, clone=True)
             if not we_are_translated():
                 for arg in failargs:
+                    if arg is None:
+                        continue
                     assert not arg.is_constant()
             copied_op.setfailargs(failargs)
 
@@ -551,8 +590,12 @@
         assert isinstance(op, GuardResOp)
         if op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
             descr = CompileLoopVersionDescr()
+            olddescr = op.getdescr()
+            vinfo = op.get_forwarded()
+            assert isinstance(vinfo, VectorizationInfo)
+            vinfo.set_old_descr(olddescr)
             if op.getdescr():
-                descr.copy_all_attributes_from(op.getdescr())
+                descr.copy_all_attributes_from(olddescr)
             op.setdescr(descr)
         arglistcopy = loop.label.getarglist_copy()
         if not we_are_translated():
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -159,6 +159,7 @@
     count = -1
 
     def __init__(self, op):
+        self.olddescr = None
         if op is None:
             return
         from rpython.jit.metainterp.history import Const
@@ -227,6 +228,12 @@
         self.bytesize = bytesize
         self.signed = signed
 
+    def set_old_descr(self, descr):
+        self.olddescr = descr
+
+    def get_old_descr(self):
+        return self.olddescr
+
 
 class AbstractResOpOrInputArg(AbstractValue):
     _attrs_ = ('_forwarded',)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -117,7 +117,7 @@
         rawstorage = RawStorage()
         va = rawstorage.new(la, type)
         vc = rawstorage.new(None, type, size=l)
-        self.meta_interp(f, [l*size, va, vc])
+        self.meta_interp(f, [l*size, va, vc], vec=True)
 
         for i in range(l):
             c = raw_storage_getitem(type,vc,i*size)
@@ -165,7 +165,7 @@
         va = rawstorage.new(la, type)
         vb = rawstorage.new(lb, type)
         vc = rawstorage.new(None, type, size=l)
-        self.meta_interp(f, [l*size, va, vb, vc])
+        self.meta_interp(f, [l*size, va, vb, vc], vec=True)
 
         for i in range(l):
             c = raw_storage_getitem(type,vc,i*size)
@@ -223,7 +223,7 @@
         va = rawstorage.new(la, type)
         vb = rawstorage.new(lb, type)
         vc = rawstorage.new(None, type, size=l)
-        self.meta_interp(f, [l*size, va, vb, vc])
+        self.meta_interp(f, [l*size, va, vb, vc], vec=True)
 
         for i in range(l):
             c = raw_storage_getitem(type,vc,i*size)
@@ -310,7 +310,7 @@
             lltype.free(va, flavor='raw')
             lltype.free(vb, flavor='raw')
             return res
-        res = self.meta_interp(f, [i])
+        res = self.meta_interp(f, [i], vec=True)
         assert res == f(i) == 3
 
     def test_vec_max(self):
@@ -334,7 +334,7 @@
                 i += 1
             lltype.free(va, flavor='raw')
             return m
-        res = self.meta_interp(f, [30])
+        res = self.meta_interp(f, [30], vec=True)
         assert res == f(30) == 128
 
     @py.test.mark.parametrize('type,func,init,insert,at,count,breaks',
@@ -383,7 +383,7 @@
                 nobreak = True
             lltype.free(va, flavor='raw')
             return not nobreak
-        res = self.meta_interp(f, [count])
+        res = self.meta_interp(f, [count], vec=True)
         assert res == f(count) == breaks
 
     def _vec_reduce(self, strat, func, type, data):
@@ -408,7 +408,7 @@
         accum = data.draw(strat)
         rawstorage = RawStorage()
         va = rawstorage.new(la, type)
-        res = self.meta_interp(f, [accum, l*size, va])
+        res = self.meta_interp(f, [accum, l*size, va], vec=True)
 
         assert isclose(rffi.cast(type, res), f(accum, l*size, va))
 
@@ -439,7 +439,7 @@
             val = va[0]
             lltype.free(va, flavor='raw')
             return val
-        res = self.meta_interp(f, [60])
+        res = self.meta_interp(f, [60], vec=True)
         assert res == f(60) == 34.5
 
     def test_constant_expand_vec_all(self):
@@ -457,7 +457,7 @@
             val = va[0]
             lltype.free(va, flavor='raw')
             return val
-        res = self.meta_interp(f, [60], vec_all=True)
+        res = self.meta_interp(f, [60], vec=True, vec_all=True)
         assert res == f(60) == 34.5
 
     @py.test.mark.parametrize('type,value', [(rffi.DOUBLE, 58.4547),
@@ -476,7 +476,7 @@
             val = va[d//2]
             lltype.free(va, flavor='raw')
             return val
-        res = self.meta_interp(f, [60,value])
+        res = self.meta_interp(f, [60,value], vec=True)
         assert res == f(60,value) == value
 
     @py.test.mark.parametrize('vec,vec_all',[(False,True),(True,False),(True,True),(False,False)])
@@ -540,7 +540,7 @@
             lltype.free(va, flavor='raw')
             lltype.free(vb, flavor='raw')
             return 0
-        res = self.meta_interp(f, [i])
+        res = self.meta_interp(f, [i], vec=True)
         assert res == f(i)
 
     @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3),(25,2.5,0.3)])
@@ -566,7 +566,7 @@
             for i in range(d):
                 s += a[i]
             return s
-        res = self.meta_interp(f, [i,v1,v2], vec_all=True)
+        res = self.meta_interp(f, [i,v1,v2], vec=True, vec_all=True)
         # sum helps to generate the rounding error of floating points
         # return 69.999 ... instead of 70, (v1+v2)*i == 70.0
         assert res == f(i,v1,v2) == sum([v1+v2]*i)
@@ -593,7 +593,7 @@
             free(vector_a)
             free(vector_b)
             return 0
-        res = self.meta_interp(f, [size], vec_all=True)
+        res = self.meta_interp(f, [size], vec=True, vec_all=True)
         assert res == f(size)
 
     def test_max_byte(self):
@@ -616,7 +616,7 @@
                 i += 1
             free(vector_a)
             return max
-        res = self.meta_interp(f, [128], vec_all=True)
+        res = self.meta_interp(f, [128], vec=True, vec_all=True)
         assert res == f(128)
 
 
@@ -673,7 +673,7 @@
                     c += {type_c_loadcast}(vector_c[i])
                 lltype.free(vector_c, flavor='raw')
                 return c
-            res = self.meta_interp(f, [{size}], vec_all=True)
+            res = self.meta_interp(f, [{size}], vec=True, vec_all=True)
             assert res == f({size})
         """
         env = {
@@ -717,7 +717,7 @@
             #    c += vector_c[i]
             lltype.free(vector_c, flavor='raw')
             return 0
-        res = self.meta_interp(f, [22], vec_all=True)
+        res = self.meta_interp(f, [22], vec=True, vec_all=True)
         assert res == f(22)
 
     def test_guard_test_location_assert(self):
@@ -739,7 +739,7 @@
                 i += 1
             lltype.free(vector_a, flavor='raw')
             return breaks
-        res = self.meta_interp(f, [22], vec_all=True)
+        res = self.meta_interp(f, [22], vec=True, vec_all=True)
         assert res == f(22)
 
     def run_unpack(self, unpack, vector_type, assignments, float=True):

From pypy.commits at gmail.com  Thu Sep 22 09:10:00 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 22 Sep 2016 06:10:00 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: one parameter too much
Message-ID: <57e3d828.06d41c0a.9bb00.a6c4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87319:10f3d3387862
Date: 2016-09-22 14:39 +0200
http://bitbucket.org/pypy/pypy/changeset/10f3d3387862/

Log:	one parameter too much

diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -306,7 +306,7 @@
                 # to be adjusted. rd_snapshot stores the live variables
                 # that are needed to resume.
                 if copied_op.is_guard():
-                    self.copy_guard_descr(renamer, copied_op, align_unroll_once and u == 0)
+                    self.copy_guard_descr(renamer, copied_op)
                 #
                 unrolled.append(copied_op)
             #

From pypy.commits at gmail.com  Thu Sep 22 09:10:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 22 Sep 2016 06:10:02 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: removed issue that modified
 list. if the vectorization was bailed this left behind modifications that
 should not have happened
Message-ID: <57e3d82a.e17cc20a.dafbc.b600@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87320:cff9faab2e49
Date: 2016-09-22 15:09 +0200
http://bitbucket.org/pypy/pypy/changeset/cff9faab2e49/

Log:	removed issue that modified list. if the vectorization was bailed
	this left behind modifications that should not have happened

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -326,7 +326,7 @@
         metainterp_sd.logger_ops.log_short_preamble([],
             label_token.short_preamble, metainterp.box_names_memo)
     loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
-                       [loop_info.label_op] + loop_ops)
+                       loop_info.extra_before_label + [loop_info.label_op] + loop_ops)
     if not we_are_translated():
         loop.check_consistency()
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -33,6 +33,7 @@
         self.jump_op = jump_op
         self.quasi_immutable_deps = quasi_immutable_deps
         self.extra_same_as = []
+        self.extra_before_label = []
 
     def final(self):
         return True
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -509,6 +509,7 @@
         self.label_op = label_op
         self.extra_same_as = extra_same_as
         self.quasi_immutable_deps = quasi_immutable_deps
+        self.extra_before_label = []
 
     def final(self):
         return True
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -48,6 +48,7 @@
         self.operations = oplist
         self.jump = jump
         assert self.jump.getopnum() == rop.JUMP
+        self.align_operations = []
 
     def setup_vectorization(self):
         for op in self.operations:
@@ -257,6 +258,8 @@
         state = SchedulerState(self.cpu, graph)
         state.schedule()
 
+        info.extra_before_label = loop.align_operations
+
         return loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
 
     def unroll_loop_iterations(self, info, loop, unroll_count, align_unroll_once=False):
@@ -327,20 +330,7 @@
         #
         loop.label = new_label
         if align_unroll_once:
-            for op in operations:
-                descr = op.getdescr()
-                if descr:
-                    # the first step of the optimization will overwrite the descr
-                    # with a compile loop version descr
-                    # in the operations to align the loop load/store ops we want the original
-                    # descr saved on the forwarded info
-                    vinfo = copied_op.get_forwarded()
-                    if vinfo:
-                        assert isinstance(vinfo, VectorizationInfo)
-                        descr = vinfo.get_old_descr()
-                        assert descr is not None
-                        op.setdescr(descr)
-            info.extra_same_as += operations
+            loop.align_operations = operations
             loop.operations = unrolled
         else:
             loop.operations = operations + unrolled
@@ -591,10 +581,7 @@
         if op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE):
             descr = CompileLoopVersionDescr()
             olddescr = op.getdescr()
-            vinfo = op.get_forwarded()
-            assert isinstance(vinfo, VectorizationInfo)
-            vinfo.set_old_descr(olddescr)
-            if op.getdescr():
+            if olddescr:
                 descr.copy_all_attributes_from(olddescr)
             op.setdescr(descr)
         arglistcopy = loop.label.getarglist_copy()
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -159,7 +159,6 @@
     count = -1
 
     def __init__(self, op):
-        self.olddescr = None
         if op is None:
             return
         from rpython.jit.metainterp.history import Const
@@ -228,12 +227,6 @@
         self.bytesize = bytesize
         self.signed = signed
 
-    def set_old_descr(self, descr):
-        self.olddescr = descr
-
-    def get_old_descr(self):
-        return self.olddescr
-
 
 class AbstractResOpOrInputArg(AbstractValue):
     _attrs_ = ('_forwarded',)

From pypy.commits at gmail.com  Thu Sep 22 09:17:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 06:17:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Likely fix
Message-ID: <57e3d9f0.e2cdc20a.fe841.bbe6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87321:2152d4ea184d
Date: 2016-09-22 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/2152d4ea184d/

Log:	Likely fix

diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -23,7 +23,7 @@
     i = 0
     members = []
     while res.gr_mem[i]:
-        members.append(ffi.string(res.gr_mem[i]))
+        members.append(os.fsdecode(ffi.string(res.gr_mem[i])))
         i += 1
     return struct_group([
         os.fsdecode(ffi.string(res.gr_name)),

From pypy.commits at gmail.com  Thu Sep 22 09:19:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 06:19:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Add merged branches
Message-ID: <57e3da72.948e1c0a.3fc8f.ea6f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87322:0b5c7849917a
Date: 2016-09-22 15:19 +0200
http://bitbucket.org/pypy/pypy/changeset/0b5c7849917a/

Log:	Add merged branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,12 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
+.. branch: mappingproxy
+.. branch: py3k-finish_time
+.. branch: py3k-kwonly-builtin
+.. branch: py3k_add_terminal_size
+.. branch: testing-cleanup-py3k
+
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
 

From pypy.commits at gmail.com  Thu Sep 22 09:30:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 06:30:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: In the py3k branch,
 use 'python3.3' to run -A tests, not 'python3'.
Message-ID: <57e3dce0.94071c0a.1fc40.deaa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87323:1953c9927f3b
Date: 2016-09-22 15:29 +0200
http://bitbucket.org/pypy/pypy/changeset/1953c9927f3b/

Log:	In the py3k branch, use 'python3.3' to run -A tests, not 'python3'.
	Some tests fail if the latter is a more recent version.

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -5,7 +5,8 @@
     # some tests fail otherwise
     sys.setrecursionlimit(2000)
 
-PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind('python3')
+LOOK_FOR_PYTHON3 = 'python3.3'
+PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind(LOOK_FOR_PYTHON3)
 if PYTHON3 is not None:
     PYTHON3 = str(PYTHON3)
 
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -7,7 +7,7 @@
 from rpython.tool.udir import udir
 from contextlib import contextmanager
 from pypy import pypydir
-from pypy.conftest import PYTHON3
+from pypy.conftest import PYTHON3, LOOK_FOR_PYTHON3
 from pypy.interpreter.test.conftest import banner
 from lib_pypy._pypy_interact import irc_header
 
@@ -18,8 +18,8 @@
     if PYTHON3:
         return PYTHON3
     import py.test
-    py.test.fail("Test requires 'python3' (not found in PATH) or a PYTHON3 "
-                 "environment variable set")
+    py.test.fail("Test requires %r (not found in PATH) or a PYTHON3 "
+                 "environment variable set" % (LOOK_FOR_PYTHON3,))
 
 _counter = 0
 def _get_next_path(ext='.py'):

From pypy.commits at gmail.com  Thu Sep 22 09:30:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 06:30:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57e3dce1.96a61c0a.2d526.e32a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87324:b15a35fba9da
Date: 2016-09-22 15:29 +0200
http://bitbucket.org/pypy/pypy/changeset/b15a35fba9da/

Log:	hg merge py3k

diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -23,7 +23,7 @@
     i = 0
     members = []
     while res.gr_mem[i]:
-        members.append(ffi.string(res.gr_mem[i]))
+        members.append(os.fsdecode(ffi.string(res.gr_mem[i])))
         i += 1
     return struct_group([
         os.fsdecode(ffi.string(res.gr_name)),
diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -5,7 +5,8 @@
     # some tests fail otherwise
     sys.setrecursionlimit(2000)
 
-PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind('python3')
+LOOK_FOR_PYTHON3 = 'python3.5'
+PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind(LOOK_FOR_PYTHON3)
 if PYTHON3 is not None:
     PYTHON3 = str(PYTHON3)
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,12 @@
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
 
+.. branch: mappingproxy
+.. branch: py3k-finish_time
+.. branch: py3k-kwonly-builtin
+.. branch: py3k_add_terminal_size
+.. branch: testing-cleanup-py3k
+
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
 
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -7,7 +7,7 @@
 from rpython.tool.udir import udir
 from contextlib import contextmanager
 from pypy import pypydir
-from pypy.conftest import PYTHON3
+from pypy.conftest import PYTHON3, LOOK_FOR_PYTHON3
 from pypy.interpreter.test.conftest import banner
 from lib_pypy._pypy_interact import irc_header
 
@@ -18,8 +18,8 @@
     if PYTHON3:
         return PYTHON3
     import py.test
-    py.test.fail("Test requires 'python3' (not found in PATH) or a PYTHON3 "
-                 "environment variable set")
+    py.test.fail("Test requires %r (not found in PATH) or a PYTHON3 "
+                 "environment variable set" % (LOOK_FOR_PYTHON3,))
 
 _counter = 0
 def _get_next_path(ext='.py'):

From pypy.commits at gmail.com  Thu Sep 22 09:52:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 06:52:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Test and fix
Message-ID: <57e3e21a.657bc20a.1a3a4.ca82@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87325:6208459577ec
Date: 2016-09-22 15:51 +0200
http://bitbucket.org/pypy/pypy/changeset/6208459577ec/

Log:	Test and fix

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -198,17 +198,17 @@
 
 
     def _start_from_tuple(self, space, w_tuple):
-        from pypy.objspace.std.tupleobject import W_TupleObject
+        from pypy.objspace.std.tupleobject import W_AbstractTupleObject
         start = 0
 
         view = self.buf
         length = space.len_w(w_tuple)
         dim = view.getndim()
         dim = 0
-        assert isinstance(w_tuple, W_TupleObject)
+        assert isinstance(w_tuple, W_AbstractTupleObject)
         while dim < length:
             w_obj = w_tuple.getitem(space, dim)
-            index = w_obj.int_w(space)
+            index = space.getindex_w(w_obj, space.w_IndexError)
             start = self.lookup_dimension(space, start, dim, index)
             dim += 1
         return start
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -323,7 +323,8 @@
         assert view[-1,-1] == 11
         assert view[-3,-4] == 0
 
-        raises(IndexError, "view.__getitem__((2**63-1,0))")
+        raises(IndexError, "view.__getitem__((2**31-1, 0))")
+        raises(IndexError, "view.__getitem__((2**63+1, 0))")
         raises(TypeError, "view.__getitem__((0, 0, 0))")
 
     def test_tuple_indexing_int(self):

From pypy.commits at gmail.com  Thu Sep 22 09:59:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 06:59:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix for test_hash
Message-ID: <57e3e3da.411d1c0a.aea37.ecb1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87326:c15a7ecf51e6
Date: 2016-09-22 15:59 +0200
http://bitbucket.org/pypy/pypy/changeset/c15a7ecf51e6/

Log:	Fix for test_hash

diff --git a/pypy/objspace/std/test/test_smalllongobject.py b/pypy/objspace/std/test/test_smalllongobject.py
--- a/pypy/objspace/std/test/test_smalllongobject.py
+++ b/pypy/objspace/std/test/test_smalllongobject.py
@@ -49,9 +49,10 @@
 
     def setup_class(cls):
         from pypy.interpreter import gateway
-        from pypy.objspace.std.smalllongobject import W_SmallLongObject
         def w__long(space, w_obj):
-            return W_SmallLongObject.frombigint(space.bigint_w(w_obj))
+            assert space.config.objspace.std.withsmalllong
+            b = space.bigint_w(w_obj)
+            return space.wraplong(b.tolong())
         cls.w__long = cls.space.wrap(gateway.interp2app(w__long))
 
     def test_sl_simple(self):

From pypy.commits at gmail.com  Thu Sep 22 10:43:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 07:43:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Test and fix
Message-ID: <57e3edf8.a699c20a.501d5.e136@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87327:4b761da6ef73
Date: 2016-09-22 16:42 +0200
http://bitbucket.org/pypy/pypy/changeset/4b761da6ef73/

Log:	Test and fix

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -292,7 +292,7 @@
         itemsize = self.getitemsize()
         dim = 0
         self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size*step)
-        shape[dim] = size
+        shape[dim] = size//itemsize
         strides[dim] = strides[dim] * step
         self.strides = strides
         self.shape = shape
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -165,6 +165,11 @@
         slice = m[2:8]
         assert slice.format == 'i'
         assert slice.itemsize == 4
+        assert slice.ndim == 1
+        assert slice.readonly is False
+        assert slice.shape == (6,)
+        assert slice.strides == (4,)
+        assert slice.suboffsets == ()
         assert len(slice) == 6
         assert len(slice.tobytes()) == 24
         assert slice[0] == 2

From pypy.commits at gmail.com  Thu Sep 22 11:23:57 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 22 Sep 2016 08:23:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Remove obsolete code
Message-ID: <57e3f78d.c50e1c0a.422ce.21a5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87328:651c97ef2d46
Date: 2016-09-22 16:15 +0100
http://bitbucket.org/pypy/pypy/changeset/651c97ef2d46/

Log:	Remove obsolete code

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -295,14 +295,6 @@
     """Truncate a file (by file descriptor) to a specified length."""
     try:
         os.ftruncate(fd, length)
-    except IOError as e:
-        if not objectmodel.we_are_translated():
-            # Python 2.6 raises an IOError here. Let's not repeat that mistake.
-            w_error = space.call_function(space.w_OSError, space.wrap(e.errno),
-                                          space.wrap(e.strerror),
-                                          space.wrap(e.filename))
-            raise OperationError(space.w_OSError, w_error)
-        raise AssertionError
     except OSError as e:
         raise wrap_oserror(space, e)
 
@@ -2142,7 +2134,7 @@
         have_functions.append("HAVE_%s" % name)
 if _WIN32:
     have_functions.append("HAVE_MS_WINDOWS")
-   
+
 def get_terminal_size(space, w_fd=None):
     if w_fd is None:
         fd = rfile.RFile(rfile.c_stdout(), close2=(None, None)).fileno()
@@ -2169,14 +2161,14 @@
             raise oefmt(space.w_OSError, "handle cannot be retrieved")
         elif handle == rwin32.INVALID_HANDLE_VALUE:
             raise rwin32.lastSavedWindowsError()
-        with lltype.scoped_alloc(rwin32.CONSOLE_SCREEN_BUFFER_INFO) as buffer_info: 
+        with lltype.scoped_alloc(rwin32.CONSOLE_SCREEN_BUFFER_INFO) as buffer_info:
             success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
             if not success:
                 raise rwin32.lastSavedWindowsError()
             w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
             w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
     else:
-        with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
+        with lltype.scoped_alloc(rposix.WINSIZE) as winsize:
             failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
             if failed:
                 raise exception_from_saved_errno(space, space.w_OSError)

From pypy.commits at gmail.com  Thu Sep 22 11:49:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 08:49:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Test and fix: correctly count the number
 of kwonly argument given, for
Message-ID: <57e3fd70.4398c20a.53134.fe8e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87329:37ab7781ce67
Date: 2016-09-22 17:48 +0200
http://bitbucket.org/pypy/pypy/changeset/37ab7781ce67/

Log:	Test and fix: correctly count the number of kwonly argument given,
	for the error message

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -198,7 +198,6 @@
             input_argcount += take
 
         # collect extra positional arguments into the *vararg
-        kwonly_given = 0
         if signature.has_vararg():
             args_left = co_argcount - upfront
             if args_left < 0:  # check required by rpython
@@ -212,9 +211,6 @@
             loc = co_argcount + co_kwonlyargcount
             scope_w[loc] = self.space.newtuple(starargs_w)
         elif avail > co_argcount:
-            for i in range(co_argcount, co_argcount + co_kwonlyargcount):
-                if scope_w[i] is None:
-                    kwonly_given += 1
             too_many_args = True
 
         # if a **kwargs argument is needed, create the dict
@@ -250,16 +246,13 @@
                 else:
                     raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
                                             kwds_mapping, self.keyword_names_w)
-        if too_many_args:
-            raise ArgErrTooMany(signature.num_argnames(),
-                                0 if defaults_w is None else len(defaults_w),
-                                avail, kwonly_given)
 
         # check for missing arguments and fill them from the kwds,
         # or with defaults, if available
         missing_positional = []
         missing_kwonly = []
-        if input_argcount < co_argcount + co_kwonlyargcount:
+        more_filling = (input_argcount < co_argcount + co_kwonlyargcount)
+        if more_filling:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
             j = 0
             kwds_index = -1
@@ -271,6 +264,16 @@
                     if kwds_index >= 0:
                         scope_w[i] = keywords_w[kwds_index]
 
+        if too_many_args:
+            kwonly_given = 0
+            for i in range(co_argcount, co_argcount + co_kwonlyargcount):
+                if scope_w[i] is not None:
+                    kwonly_given += 1
+            raise ArgErrTooMany(signature.num_argnames(),
+                                0 if defaults_w is None else len(defaults_w),
+                                avail, kwonly_given)
+
+        if more_filling:
             # then, fill the normal arguments with defaults_w (if needed)
             for i in range(input_argcount, co_argcount):
                 if scope_w[i] is not None:
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -634,6 +634,62 @@
         else:
             assert 0, "did not raise"
 
+    def test_dont_count_default_arguments(self):
+        space = self.space
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c): pass
+            try:
+                f1(4)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 1 was given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c=8): pass
+            try:
+                f1(4)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 1 was given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(a, b, *, c): pass
+            try:
+                f1(4, 5, 6)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 2 positional arguments but 3 were given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c): pass
+            try:
+                f1(6, c=7)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 1 positional argument (and 1 keyword-only argument) were given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c, d=8, e=9): pass
+            try:
+                f1(6, 2, c=7, d=8)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 2 positional arguments (and 2 keyword-only arguments) were given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c, d=8, e=9, **kwds): pass
+            try:
+                f1(6, 2, c=7, d=8, morestuff=9)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 2 positional arguments (and 2 keyword-only arguments) were given'
+
     def test_unknown_keywords(self):
         space = DummySpace()
         err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [0], None)

From pypy.commits at gmail.com  Thu Sep 22 11:50:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 08:50:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57e3fdb4.0558c20a.a5182.f959@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87330:a48793436b6d
Date: 2016-09-22 17:49 +0200
http://bitbucket.org/pypy/pypy/changeset/a48793436b6d/

Log:	hg merge py3k

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -198,7 +198,6 @@
             input_argcount += take
 
         # collect extra positional arguments into the *vararg
-        kwonly_given = 0
         if signature.has_vararg():
             args_left = co_argcount - upfront
             if args_left < 0:  # check required by rpython
@@ -212,9 +211,6 @@
             loc = co_argcount + co_kwonlyargcount
             scope_w[loc] = self.space.newtuple(starargs_w)
         elif avail > co_argcount:
-            for i in range(co_argcount, co_argcount + co_kwonlyargcount):
-                if scope_w[i] is None:
-                    kwonly_given += 1
             too_many_args = True
 
         # if a **kwargs argument is needed, create the dict
@@ -250,16 +246,13 @@
                 else:
                     raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
                                             kwds_mapping, self.keyword_names_w)
-        if too_many_args:
-            raise ArgErrTooMany(signature.num_argnames(),
-                                0 if defaults_w is None else len(defaults_w),
-                                avail, kwonly_given)
 
         # check for missing arguments and fill them from the kwds,
         # or with defaults, if available
         missing_positional = []
         missing_kwonly = []
-        if input_argcount < co_argcount + co_kwonlyargcount:
+        more_filling = (input_argcount < co_argcount + co_kwonlyargcount)
+        if more_filling:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
             j = 0
             kwds_index = -1
@@ -271,6 +264,16 @@
                     if kwds_index >= 0:
                         scope_w[i] = keywords_w[kwds_index]
 
+        if too_many_args:
+            kwonly_given = 0
+            for i in range(co_argcount, co_argcount + co_kwonlyargcount):
+                if scope_w[i] is not None:
+                    kwonly_given += 1
+            raise ArgErrTooMany(signature.num_argnames(),
+                                0 if defaults_w is None else len(defaults_w),
+                                avail, kwonly_given)
+
+        if more_filling:
             # then, fill the normal arguments with defaults_w (if needed)
             for i in range(input_argcount, co_argcount):
                 if scope_w[i] is not None:
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -634,6 +634,62 @@
         else:
             assert 0, "did not raise"
 
+    def test_dont_count_default_arguments(self):
+        space = self.space
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c): pass
+            try:
+                f1(4)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 1 was given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c=8): pass
+            try:
+                f1(4)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 1 was given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(a, b, *, c): pass
+            try:
+                f1(4, 5, 6)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 2 positional arguments but 3 were given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c): pass
+            try:
+                f1(6, c=7)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 1 positional argument (and 1 keyword-only argument) were given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c, d=8, e=9): pass
+            try:
+                f1(6, 2, c=7, d=8)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 2 positional arguments (and 2 keyword-only arguments) were given'
+        #
+        msg = space.unwrap(space.appexec([], """():
+            def f1(*, c, d=8, e=9, **kwds): pass
+            try:
+                f1(6, 2, c=7, d=8, morestuff=9)
+            except TypeError as e:
+                return str(e)
+        """))
+        assert msg == 'f1() takes 0 positional arguments but 2 positional arguments (and 2 keyword-only arguments) were given'
+
     def test_unknown_keywords(self):
         space = DummySpace()
         err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [0], None)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -300,14 +300,6 @@
     """Truncate a file (by file descriptor) to a specified length."""
     try:
         os.ftruncate(fd, length)
-    except IOError as e:
-        if not objectmodel.we_are_translated():
-            # Python 2.6 raises an IOError here. Let's not repeat that mistake.
-            w_error = space.call_function(space.w_OSError, space.wrap(e.errno),
-                                          space.wrap(e.strerror),
-                                          space.wrap(e.filename))
-            raise OperationError(space.w_OSError, w_error)
-        raise AssertionError
     except OSError as e:
         raise wrap_oserror(space, e)
 
@@ -2172,7 +2164,7 @@
         have_functions.append("HAVE_%s" % name)
 if _WIN32:
     have_functions.append("HAVE_MS_WINDOWS")
-   
+
 def get_terminal_size(space, w_fd=None):
     if w_fd is None:
         fd = rfile.RFile(rfile.c_stdout(), close2=(None, None)).fileno()
@@ -2199,14 +2191,14 @@
             raise oefmt(space.w_OSError, "handle cannot be retrieved")
         elif handle == rwin32.INVALID_HANDLE_VALUE:
             raise rwin32.lastSavedWindowsError()
-        with lltype.scoped_alloc(rwin32.CONSOLE_SCREEN_BUFFER_INFO) as buffer_info: 
+        with lltype.scoped_alloc(rwin32.CONSOLE_SCREEN_BUFFER_INFO) as buffer_info:
             success = rwin32.GetConsoleScreenBufferInfo(handle, buffer_info)
             if not success:
                 raise rwin32.lastSavedWindowsError()
             w_columns = space.wrap(r_int(buffer_info.c_srWindow.c_Right) - r_int(buffer_info.c_srWindow.c_Left) + 1)
             w_lines = space.wrap(r_int(buffer_info.c_srWindow.c_Bottom) - r_int(buffer_info.c_srWindow.c_Top) + 1)
     else:
-        with lltype.scoped_alloc(rposix.WINSIZE) as winsize: 
+        with lltype.scoped_alloc(rposix.WINSIZE) as winsize:
             failed = rposix.c_ioctl_voidp(fd, rposix.TIOCGWINSZ, winsize)
             if failed:
                 raise exception_from_saved_errno(space, space.w_OSError)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -198,17 +198,17 @@
 
 
     def _start_from_tuple(self, space, w_tuple):
-        from pypy.objspace.std.tupleobject import W_TupleObject
+        from pypy.objspace.std.tupleobject import W_AbstractTupleObject
         start = 0
 
         view = self.buf
         length = space.len_w(w_tuple)
         dim = view.getndim()
         dim = 0
-        assert isinstance(w_tuple, W_TupleObject)
+        assert isinstance(w_tuple, W_AbstractTupleObject)
         while dim < length:
             w_obj = w_tuple.getitem(space, dim)
-            index = w_obj.int_w(space)
+            index = space.getindex_w(w_obj, space.w_IndexError)
             start = self.lookup_dimension(space, start, dim, index)
             dim += 1
         return start
@@ -292,7 +292,7 @@
         itemsize = self.getitemsize()
         dim = 0
         self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size*step)
-        shape[dim] = size
+        shape[dim] = size//itemsize
         strides[dim] = strides[dim] * step
         self.strides = strides
         self.shape = shape
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -168,6 +168,11 @@
         slice = m[2:8]
         assert slice.format == 'i'
         assert slice.itemsize == 4
+        assert slice.ndim == 1
+        assert slice.readonly is False
+        assert slice.shape == (6,)
+        assert slice.strides == (4,)
+        assert slice.suboffsets == ()
         assert len(slice) == 6
         assert len(slice.tobytes()) == 24
         assert slice.nbytes == 24
@@ -330,7 +335,8 @@
         assert view[-1,-1] == 11
         assert view[-3,-4] == 0
 
-        raises(IndexError, "view.__getitem__((2**63-1,0))")
+        raises(IndexError, "view.__getitem__((2**31-1, 0))")
+        raises(IndexError, "view.__getitem__((2**63+1, 0))")
         raises(TypeError, "view.__getitem__((0, 0, 0))")
 
     def test_tuple_indexing_int(self):
diff --git a/pypy/objspace/std/test/test_smalllongobject.py b/pypy/objspace/std/test/test_smalllongobject.py
--- a/pypy/objspace/std/test/test_smalllongobject.py
+++ b/pypy/objspace/std/test/test_smalllongobject.py
@@ -49,9 +49,10 @@
 
     def setup_class(cls):
         from pypy.interpreter import gateway
-        from pypy.objspace.std.smalllongobject import W_SmallLongObject
         def w__long(space, w_obj):
-            return W_SmallLongObject.frombigint(space.bigint_w(w_obj))
+            assert space.config.objspace.std.withsmalllong
+            b = space.bigint_w(w_obj)
+            return space.wraplong(b.tolong())
         cls.w__long = cls.space.wrap(gateway.interp2app(w__long))
 
     def test_sl_simple(self):

From pypy.commits at gmail.com  Thu Sep 22 14:40:54 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 22 Sep 2016 11:40:54 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Clean up for virtualstate.py
Message-ID: <57e425b6.ca11c30a.d4df7.3d77@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87331:770b5b1dda6b
Date: 2016-09-22 14:40 -0400
http://bitbucket.org/pypy/pypy/changeset/770b5b1dda6b/

Log:	Clean up for virtualstate.py

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -29,6 +29,14 @@
         self.optearlyforce = None
         self.optimizer = Optimizer
 
+
+class FakeDescr(object):
+    def __init__(self, vtable):
+        self.vtable = vtable
+    def get_vtable(self):
+        return self.vtable
+FakeDescr.value = FakeDescr(42)
+
 class BaseTestGenerateGuards(BaseTest):
     def setup_class(self):
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
@@ -158,7 +166,8 @@
         info1.fieldstate = info2.fieldstate = []
         postest(info1, info2)
 
-        info1, info2 = VirtualStateInfo(ConstInt(42), []), VirtualStateInfo(ConstInt(42), [])
+        fakedescr = FakeDescr.value
+        info1, info2 = VirtualStateInfo(fakedescr, []), VirtualStateInfo(fakedescr, [])
         info1.fieldstate = info2.fieldstate = []
         postest(info1, info2)
 
@@ -222,10 +231,10 @@
             def get_index(self):
                 return 0
         fielddescr = FieldDescr()
-        fakedescr = object()
+        fakedescr = FakeDescr.value
         fldtst(VArrayStateInfo(fakedescr), VArrayStateInfo(fakedescr))
         fldtst(VStructStateInfo(fakedescr, [fielddescr]), VStructStateInfo(fakedescr, [fielddescr]))
-        fldtst(VirtualStateInfo(ConstInt(42), [fielddescr]), VirtualStateInfo(ConstInt(42), [fielddescr]))
+        fldtst(VirtualStateInfo(fakedescr, [fielddescr]), VirtualStateInfo(fakedescr, [fielddescr]))
         fldtst(VArrayStructStateInfo(fakedescr, [fielddescr], 1), VArrayStructStateInfo(fakedescr, [fielddescr], 1))
 
     def test_known_class_generalization(self):
@@ -249,7 +258,7 @@
 
     def test_circular_generalization(self):
         for info in (VArrayStateInfo(42), VStructStateInfo(42, [7]),
-                     VirtualStateInfo(ConstInt(42), [7])):
+                     VirtualStateInfo(FakeDescr.value, [7])):
             info.position = 0
             info.fieldstate = [info]
             self.check_no_guards(info, info)
@@ -568,10 +577,10 @@
         innerinfo2 = not_virtual(self.cpu, 'r', None)
         innerinfo2.position = 1
 
-        info1 = VirtualStateInfo(ConstInt(42), [self.nextdescr])
+        info1 = VirtualStateInfo(FakeDescr.value, [self.nextdescr])
         info1.fieldstate = [innerinfo1]
 
-        info2 = VirtualStateInfo(ConstInt(42), [self.nextdescr])
+        info2 = VirtualStateInfo(FakeDescr.value, [self.nextdescr])
         info2.fieldstate = [innerinfo2]
 
         value1 = info.InstancePtrInfo(self.nodesize, classbox, is_virtual=True)
@@ -660,12 +669,12 @@
     # the below tests don't really have anything to do with guard generation
 
     def test_virtuals_with_equal_fields(self):
-        info1 = VirtualStateInfo(ConstInt(42), [1, 2])
+        info1 = VirtualStateInfo(FakeDescr.value, [1, 2])
         info1.fieldstate = [self.knownclass_info, self.knownclass_info]
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        info2 = VirtualStateInfo(ConstInt(42), [1, 2])
+        info2 = VirtualStateInfo(FakeDescr.value, [1, 2])
         unknown_info1 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
         info2.fieldstate = [unknown_info1, unknown_info1]
@@ -674,7 +683,7 @@
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
         assert vstate2.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        info3 = VirtualStateInfo(ConstInt(42), [1, 2])
+        info3 = VirtualStateInfo(FakeDescr.value, [1, 2])
         unknown_info1 = not_virtual(self.cpu, 'r',
                                             info.InstancePtrInfo())
         unknown_info2 = not_virtual(self.cpu, 'r',
@@ -687,7 +696,7 @@
         assert not vstate1.generalization_of(vstate3, FakeOptimizer(self.cpu))
 
     def test_virtuals_with_nonmatching_fields(self):
-        info1 = VirtualStateInfo(ConstInt(42), [1, 2])
+        info1 = VirtualStateInfo(FakeDescr.value, [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
         knownclass_info = not_virtual(self.cpu, 'r', value)
@@ -695,7 +704,7 @@
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        info2 = VirtualStateInfo(ConstInt(42), [1, 2])
+        info2 = VirtualStateInfo(FakeDescr.value, [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
         knownclass_info = not_virtual(self.cpu, 'r', value)
@@ -707,7 +716,7 @@
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
 
     def test_virtuals_with_nonmatching_descrs(self):
-        info1 = VirtualStateInfo(ConstInt(42), [10, 20])
+        info1 = VirtualStateInfo(FakeDescr.value, [10, 20])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
         knownclass_info = not_virtual(self.cpu, 'r', value)
@@ -715,7 +724,7 @@
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        info2 = VirtualStateInfo(ConstInt(42), [1, 2])
+        info2 = VirtualStateInfo(FakeDescr.value, [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
         knownclass_info = not_virtual(self.cpu, 'r', value)
@@ -727,7 +736,7 @@
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
         
     def test_virtuals_with_nonmatching_classes(self):
-        info1 = VirtualStateInfo(ConstInt(42), [1, 2])
+        info1 = VirtualStateInfo(FakeDescr.value, [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
         knownclass_info = not_virtual(self.cpu, 'r', value)
@@ -735,7 +744,7 @@
         vstate1 = VirtualState([info1])
         assert vstate1.generalization_of(vstate1, FakeOptimizer(self.cpu))
 
-        info2 = VirtualStateInfo(ConstInt(7), [1, 2])
+        info2 = VirtualStateInfo(FakeDescr(7), [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.node2addr))
         value = info.InstancePtrInfo(None, classbox)
         knownclass_info = not_virtual(self.cpu, 'r', value)
@@ -747,7 +756,7 @@
         assert not vstate1.generalization_of(vstate2, FakeOptimizer(self.cpu))
 
     def test_nonvirtual_is_not_virtual(self):
-        info1 = VirtualStateInfo(ConstInt(42), [1, 2])
+        info1 = VirtualStateInfo(FakeDescr.value, [1, 2])
         classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
         value = info.InstancePtrInfo(None, classbox)
         knownclass_info = not_virtual(self.cpu, 'r', value)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -65,6 +65,7 @@
                                                                  descr))
 
 class AbstractVirtualStateInfo(object):
+    _attrs_ = ('position',)
     position = -1
 
     def generate_guards(self, other, op, runtime_op, state):
@@ -133,7 +134,10 @@
 
 
 class AbstractVirtualStructStateInfo(AbstractVirtualStateInfo):
-    def __init__(self, fielddescrs):
+    _attrs_ = ('typedescr', 'fielddescrs')
+
+    def __init__(self, typedescr, fielddescrs):
+        self.typedescr = typedescr
         self.fielddescrs = fielddescrs
 
     def _generate_guards(self, other, box, runtime_box, state):
@@ -173,9 +177,8 @@
                                                    fieldbox,
                                                    fieldbox_runtime, state)
 
-
     def _generalization_of_structpart(self, other):
-        raise NotImplementedError
+        return type(self) is type(other) and self.typedescr is other.typedescr
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
@@ -202,30 +205,17 @@
 
 
 class VirtualStateInfo(AbstractVirtualStructStateInfo):
-    def is_virtual(self):
-        return True
+    _attrs_ = ('known_class',)
 
-    def __init__(self, known_class, fielddescrs):
-        AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
-        self.known_class = known_class
-
-    def _generalization_of_structpart(self, other):
-        return (isinstance(other, VirtualStateInfo) and
-                self.known_class.same_constant(other.known_class))
-
+    def __init__(self, typedescr, fielddescrs):
+        AbstractVirtualStructStateInfo.__init__(self, typedescr, fielddescrs)
+        self.known_class = ConstInt(typedescr.get_vtable())
 
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
 
 
 class VStructStateInfo(AbstractVirtualStructStateInfo):
-    def __init__(self, typedescr, fielddescrs):
-        AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
-        self.typedescr = typedescr
-
-    def _generalization_of_structpart(self, other):
-        return (isinstance(other, VStructStateInfo) and
-                self.typedescr is other.typedescr)
 
     def debug_header(self, indent):
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
@@ -372,12 +362,6 @@
             self.level = LEVEL_CONSTANT
             self.constbox = info.getconst()
 
-    def is_const(self):
-        return self.constbox is not None
-
-    def is_virtual(self):
-        return False
-
     def _generate_guards(self, other, box, runtime_box, state):
         # XXX This will always retrace instead of forcing anything which
         # might be what we want sometimes?
@@ -745,11 +729,12 @@
         return not_virtual(self.optimizer.cpu, box.type,
                            self.optimizer.getinfo(box))
 
-    def visit_virtual(self, descr, fielddescrs):
-        known_class = ConstInt(descr.get_vtable())
-        return VirtualStateInfo(known_class, fielddescrs)
+    def visit_virtual(self, typedescr, fielddescrs):
+        assert typedescr.is_object()
+        return VirtualStateInfo(typedescr, fielddescrs)
 
     def visit_vstruct(self, typedescr, fielddescrs):
+        assert not typedescr.is_object()
         return VStructStateInfo(typedescr, fielddescrs)
 
     def visit_varray(self, arraydescr, clear):

From pypy.commits at gmail.com  Thu Sep 22 16:57:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 22 Sep 2016 13:57:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Functions and methods that are actually
 "built-in" on CPython might end
Message-ID: <57e445aa.e17cc20a.dafbc.647c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87332:e00dbe634410
Date: 2016-09-22 18:11 +0200
http://bitbucket.org/pypy/pypy/changeset/e00dbe634410/

Log:	Functions and methods that are actually "built-in" on CPython might
	end up with FunctionType or MethodType. We need to work harder to be
	sure to exclude them here.

diff --git a/lib-python/3/idlelib/CallTips.py b/lib-python/3/idlelib/CallTips.py
--- a/lib-python/3/idlelib/CallTips.py
+++ b/lib-python/3/idlelib/CallTips.py
@@ -144,7 +144,8 @@
         fob = ob_call
     else:
         fob = ob
-    if isinstance(fob, (types.FunctionType, types.MethodType)):
+    if (isinstance(fob, (types.FunctionType, types.MethodType)) and
+            hasattr(fob.__code__, 'co_code')):  # PyPy: not on <builtin-code>
         argspec = inspect.formatargspec(*inspect.getfullargspec(fob))
         if (isinstance(ob, (type, types.MethodType)) or
                 isinstance(ob_call, types.MethodType)):

From pypy.commits at gmail.com  Thu Sep 22 16:58:09 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 22 Sep 2016 13:58:09 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Fix some translation errors
Message-ID: <57e445e1.c4371c0a.7c6db.877a@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87333:36ea15125469
Date: 2016-09-22 16:54 -0400
http://bitbucket.org/pypy/pypy/changeset/36ea15125469/

Log:	Fix some translation errors

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -65,7 +65,7 @@
                                                                  descr))
 
 class AbstractVirtualStateInfo(object):
-    _attrs_ = ('position',)
+    _attrs_ = ('position', 'fieldstate')
     position = -1
 
     def generate_guards(self, other, op, runtime_op, state):
@@ -177,8 +177,8 @@
                                                    fieldbox,
                                                    fieldbox_runtime, state)
 
-    def _generalization_of_structpart(self, other):
-        return type(self) is type(other) and self.typedescr is other.typedescr
+    def _generate_guards_non_virtual(self, other, box, runtime_box, state):
+        pass
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
@@ -203,7 +203,6 @@
             if s:
                 s.enum(virtual_state)
 
-
 class VirtualStateInfo(AbstractVirtualStructStateInfo):
     _attrs_ = ('known_class',)
 
@@ -211,16 +210,22 @@
         AbstractVirtualStructStateInfo.__init__(self, typedescr, fielddescrs)
         self.known_class = ConstInt(typedescr.get_vtable())
 
+    def _generalization_of_structpart(self, other):
+        return (isinstance(other, VirtualStateInfo) and
+                self.typedescr is other.typedescr)
+
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
 
+class VStructStateInfo(AbstractVirtualStructStateInfo):
 
-class VStructStateInfo(AbstractVirtualStructStateInfo):
+    def _generalization_of_structpart(self, other):
+        return (isinstance(other, VStructStateInfo) and
+                self.typedescr is other.typedescr)
 
     def debug_header(self, indent):
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
 
-
 class VArrayStateInfo(AbstractVirtualStateInfo):
 
     def __init__(self, arraydescr):

From pypy.commits at gmail.com  Thu Sep 22 17:34:28 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 22 Sep 2016 14:34:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: sys.hash_info.algorithm = 'rpython'
Message-ID: <57e44e64.43921c0a.ceddd.a325@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r87334:4d4125dd1c84
Date: 2016-09-22 23:31 +0200
http://bitbucket.org/pypy/pypy/changeset/4d4125dd1c84/

Log:	sys.hash_info.algorithm = 'rpython' The algorithm looks like
	CPython's 'fnv' function, but does not use 8 bytes blocks.

	This unblocks running tests with the "-v" option.

diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -5,6 +5,7 @@
 from pypy.objspace.std.complexobject import HASH_IMAG
 from pypy.objspace.std.floatobject import HASH_INF, HASH_NAN
 from pypy.objspace.std.intobject import HASH_MODULUS
+from pypy.objspace.std.bytesobject import HASH_ALGORITHM
 from pypy.interpreter import gateway
 from rpython.rlib import rbigint, rfloat
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -38,6 +39,7 @@
     inf = structseqfield(2)
     nan = structseqfield(3)
     imag = structseqfield(4)
+    algorithm = structseqfield(5)
 
 class thread_info(metaclass=structseqtype):
     name = structseqfield(0)
@@ -80,6 +82,7 @@
         space.wrap(HASH_INF),
         space.wrap(HASH_NAN),
         space.wrap(HASH_IMAG),
+        space.wrap(HASH_ALGORITHM),
     ]
     w_hash_info = app.wget(space, "hash_info")
     return space.call_function(w_hash_info, space.newtuple(info_w))
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -201,6 +201,7 @@
         assert isinstance(li.inf, int)
         assert isinstance(li.nan, int)
         assert isinstance(li.imag, int)
+        assert isinstance(li.algorithm, str)
 
     def test_sys_exit(self):
         import sys
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -3,7 +3,7 @@
 from rpython.rlib.jit import we_are_jitted
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin, newlist_hint,
-    resizelist_hint)
+    resizelist_hint, HASH_ALGORITHM)
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder
 
@@ -16,7 +16,6 @@
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 from pypy.objspace.std.formatting import mod_format, FORMAT_BYTES
 
-
 class W_AbstractBytesObject(W_Root):
     __slots__ = ()
 
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -499,6 +499,8 @@
 
 # ----------
 
+HASH_ALGORITHM = "rpython"  # XXX Is there a better name?
+
 def _hash_string(s):
     """The algorithm behind compute_hash() for a string or a unicode."""
     from rpython.rlib.rarithmetic import intmask

From pypy.commits at gmail.com  Thu Sep 22 17:41:06 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 22 Sep 2016 14:41:06 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-null-slots: add a test that shows
 incompatibility in method lookup() resolution
Message-ID: <57e44ff2.a7bec20a.dde92.7899@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-null-slots
Changeset: r87335:e42723277e23
Date: 2016-09-23 00:39 +0300
http://bitbucket.org/pypy/pypy/changeset/e42723277e23/

Log:	add a test that shows incompatibility in method lookup() resolution

diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -99,13 +99,13 @@
              """),
             ('alloc_rw', "METH_NOARGS",
              '''
-                PyObject *obj = _PyObject_NewVar(&PyBytes_Type, 10);
+                PyVarObject *obj = _PyObject_NewVar(&PyBytes_Type, 10);
                 memcpy(PyBytes_AS_STRING(obj), "works", 6);
                 return (PyObject*)obj;
              '''),
             ])
         s = module.alloc_rw()
-        assert s == b'works' + b'\x00' * 5
+        assert s[:6] == b'works\x00'
         s = module.tpalloc()
         assert s == b'\x00' * 10
 
@@ -343,14 +343,16 @@
         assert module.test_sstate()
 
     def test_subclass(self):
-        # taken from PyStringArrType_Type in numpy's scalartypes.c.src
+        # part 1 - taken from PyStringArrType_Type in numpy's scalartypes.c.src
+        # part 2 - make sure int(np.string_('abc')) does not reach nb_int,
+        #          i.e. it fails space.lookup(ob, "__int__") in 
+        #          _newint() from std/intobject.py
         module = self.import_extension('bar', [
             ("newsubstr", "METH_O",
              """
                 PyObject * obj;
                 char * data;
                 int len;
-                PyType_Ready(&PyStringArrType_Type);
 
                 data = PyString_AS_STRING(args);
                 len = PyString_GET_SIZE(args);
@@ -369,63 +371,98 @@
                     0                             /* tp_itemsize */
                     };
 
-                    static PyObject *
-                    stringtype_repr(PyObject *self)
-                    {
-                        const char *dptr, *ip;
-                        int len;
-                        PyObject *new;
+                static PyObject *
+                stringtype_repr(PyObject *self)
+                {
+                    const char *dptr, *ip;
+                    int len;
+                    PyObject *new;
 
-                        ip = dptr = PyString_AS_STRING(self);
-                        len = PyString_GET_SIZE(self);
-                        dptr += len-1;
-                        while(len > 0 && *dptr-- == 0) {
-                            len--;
-                        }
-                        new = PyString_FromStringAndSize(ip, len);
-                        if (new == NULL) {
-                            return PyString_FromString("");
-                        }
-                        return new;
+                    ip = dptr = PyString_AS_STRING(self);
+                    len = PyString_GET_SIZE(self);
+                    dptr += len-1;
+                    while(len > 0 && *dptr-- == 0) {
+                        len--;
                     }
+                    new = PyString_FromStringAndSize(ip, len);
+                    if (new == NULL) {
+                        return PyString_FromString("");
+                    }
+                    return new;
+                }
 
-                    static PyObject *
-                    stringtype_str(PyObject *self)
-                    {
-                        const char *dptr, *ip;
-                        int len;
-                        PyObject *new;
+                static PyObject *
+                stringtype_str(PyObject *self)
+                {
+                    const char *dptr, *ip;
+                    int len;
+                    PyObject *new;
 
-                        ip = dptr = PyString_AS_STRING(self);
-                        len = PyString_GET_SIZE(self);
-                        dptr += len-1;
-                        while(len > 0 && *dptr-- == 0) {
-                            len--;
-                        }
-                        new = PyString_FromStringAndSize(ip, len);
-                        if (new == NULL) {
-                            return PyString_FromString("");
-                        }
-                        return new;
+                    ip = dptr = PyString_AS_STRING(self);
+                    len = PyString_GET_SIZE(self);
+                    dptr += len-1;
+                    while(len > 0 && *dptr-- == 0) {
+                        len--;
                     }
+                    new = PyString_FromStringAndSize(ip, len);
+                    if (new == NULL) {
+                        return PyString_FromString("");
+                    }
+                    return new;
+                }
 
-                    PyObject *
-                    PyArray_Scalar(char *data, int n)
-                    {
-                        PyTypeObject *type = &PyStringArrType_Type;
-                        PyObject *obj;
-                        void *destptr;
-                        int itemsize = n;
-                        obj = type->tp_alloc(type, itemsize);
-                        if (obj == NULL) {
-                            return NULL;
-                        }
-                        destptr = PyString_AS_STRING(obj);
-                        ((PyBytesObject *)obj)->ob_shash = -1;
-                        memcpy(destptr, data, itemsize);
-                        return obj;
+                PyObject *
+                PyArray_Scalar(char *data, int n)
+                {
+                    PyTypeObject *type = &PyStringArrType_Type;
+                    PyObject *obj;
+                    void *destptr;
+                    int itemsize = n;
+                    obj = type->tp_alloc(type, itemsize);
+                    if (obj == NULL) {
+                        return NULL;
                     }
+                    destptr = PyString_AS_STRING(obj);
+                    ((PyBytesObject *)obj)->ob_shash = -1;
+                    memcpy(destptr, data, itemsize);
+                    return obj;
+                }
+
+                static PyTypeObject Intermediate_Type = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,
+                    "bar.Intermediate",
+                };
+
+                static PyTypeObject Base_Type = {
+                    PyObject_HEAD_INIT(NULL)
+                    0,
+                    "bar.Base",
+                };
+
+                PyObject *
+                nb_int(PyObject * self)
+                {
+                    return PyLong_FromLong(42);
+                }
+
+                PyNumberMethods base_as_number;
             """, more_init = '''
+                if (PyType_Ready(&PyString_Type)<0)
+                    return;
+
+                base_as_number.nb_int = nb_int;
+                /*Base_Type.tp_base = &PyType_Type;*/
+                Base_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                Base_Type.tp_as_number = &base_as_number;
+                if (PyType_Ready(&Base_Type) < 0)
+                    return;
+
+                Intermediate_Type.tp_base = &Base_Type;
+                Intermediate_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                if (PyType_Ready(&Intermediate_Type) < 0)
+                    return;
+
                 PyStringArrType_Type.tp_alloc = NULL;
                 PyStringArrType_Type.tp_free = NULL;
 
@@ -434,11 +471,18 @@
                 PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
                 PyStringArrType_Type.tp_itemsize = sizeof(char);
                 PyStringArrType_Type.tp_base = &PyString_Type;
+                PyStringArrType_Type.tp_bases = Py_BuildValue("(OO)",
+                        &PyString_Type, &Intermediate_Type);
+                PyType_Ready(&PyStringArrType_Type);
+                Py_INCREF(&PyStringArrType_Type);    
             ''')
 
         a = module.newsubstr('abc')
         assert type(a).__name__ == 'string_'
         assert a == 'abc'
+        #print type(a).mro(type(a))
+        raises(ValueError, int, a)
+
 
 class TestBytes(BaseApiTest):
     def test_bytes_resize(self, space, api):

From pypy.commits at gmail.com  Thu Sep 22 23:03:07 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Thu, 22 Sep 2016 20:03:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: refactor for py3k
Message-ID: <57e49b6b.0558c20a.a5182.abc1@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r87336:9f43cc50b128
Date: 2016-09-22 19:57 -0700
http://bitbucket.org/pypy/pypy/changeset/9f43cc50b128/

Log:	refactor for py3k

diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -22,7 +22,10 @@
     code, message = _ffi.getwinerror()
     raise WindowsError(code, message)
 
-_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+def _int2handle(val):
+    return _ffi.cast("HANDLE", val)
+
+_INVALID_HANDLE_VALUE = _int2handle(-1)
 
 class _handle(object):
     def __init__(self, c_handle):
@@ -70,9 +73,9 @@
     target = _ffi.new("HANDLE[1]")
 
     res = _kernel32.DuplicateHandle(
-        _ffi.cast("HANDLE", source_process),
-        _ffi.cast("HANDLE", source),
-        _ffi.cast("HANDLE", target_process),
+        _int2handle(source_process),
+        _int2handle(source),
+        _int2handle(target_process),
         target, access, inherit, options)
 
     if not res:
@@ -119,12 +122,14 @@
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
+    return (_handle(pi.hProcess),
+            _handle(pi.hThread),
+            pi.dwProcessId,
+            pi.dwThreadId)
 
 def WaitForSingleObject(handle, milliseconds):
     # CPython: the first argument is expected to be an integer.
-    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
-                                        milliseconds)
+    res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds)
     if res < 0:
         raise _WinError()
 
@@ -134,7 +139,7 @@
     # CPython: the first argument is expected to be an integer.
     code = _ffi.new("DWORD[1]")
 
-    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
+    res = _kernel32.GetExitCodeProcess(_int2handle(handle), code)
 
     if not res:
         raise _WinError()
@@ -144,7 +149,7 @@
 def TerminateProcess(handle, exitcode):
     # CPython: the first argument is expected to be an integer.
     # The second argument is silently wrapped in a UINT.
-    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+    res = _kernel32.TerminateProcess(_int2handle(handle),
                                      _ffi.cast("UINT", exitcode))
 
     if not res:

From pypy.commits at gmail.com  Thu Sep 22 23:03:09 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Thu, 22 Sep 2016 20:03:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <57e49b6d.89d11c0a.5238f.2fd9@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r87337:2bac03f27a04
Date: 2016-09-22 19:57 -0700
http://bitbucket.org/pypy/pypy/changeset/2bac03f27a04/

Log:	merge default

diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -22,7 +22,10 @@
     code, message = _ffi.getwinerror()
     raise WindowsError(code, message)
 
-_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+def _int2handle(val):
+    return _ffi.cast("HANDLE", val)
+
+_INVALID_HANDLE_VALUE = _int2handle(-1)
 
 class _handle(object):
     def __init__(self, c_handle):
@@ -70,9 +73,9 @@
     target = _ffi.new("HANDLE[1]")
 
     res = _kernel32.DuplicateHandle(
-        _ffi.cast("HANDLE", source_process),
-        _ffi.cast("HANDLE", source),
-        _ffi.cast("HANDLE", target_process),
+        _int2handle(source_process),
+        _int2handle(source),
+        _int2handle(target_process),
         target, access, inherit, options)
 
     if not res:
@@ -120,12 +123,14 @@
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
+    return (_handle(pi.hProcess),
+            _handle(pi.hThread),
+            pi.dwProcessId,
+            pi.dwThreadId)
 
 def WaitForSingleObject(handle, milliseconds):
     # CPython: the first argument is expected to be an integer.
-    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
-                                        milliseconds)
+    res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds)
     if res < 0:
         raise _WinError()
 
@@ -135,7 +140,7 @@
     # CPython: the first argument is expected to be an integer.
     code = _ffi.new("DWORD[1]")
 
-    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
+    res = _kernel32.GetExitCodeProcess(_int2handle(handle), code)
 
     if not res:
         raise _WinError()
@@ -145,7 +150,7 @@
 def TerminateProcess(handle, exitcode):
     # CPython: the first argument is expected to be an integer.
     # The second argument is silently wrapped in a UINT.
-    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+    res = _kernel32.TerminateProcess(_int2handle(handle),
                                      _ffi.cast("UINT", exitcode))
 
     if not res:

From pypy.commits at gmail.com  Thu Sep 22 23:03:11 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Thu, 22 Sep 2016 20:03:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: kill _handle,
 subprocess.Handle basically replaces it
Message-ID: <57e49b6f.94071c0a.35716.32ce@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r87338:df634d483569
Date: 2016-09-22 20:02 -0700
http://bitbucket.org/pypy/pypy/changeset/df634d483569/

Log:	kill _handle, subprocess.Handle basically replaces it

diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -25,36 +25,11 @@
 def _int2handle(val):
     return _ffi.cast("HANDLE", val)
 
+def _handle2int(handle):
+    return int(_ffi.cast("intptr_t", handle))
+
 _INVALID_HANDLE_VALUE = _int2handle(-1)
 
-class _handle(object):
-    def __init__(self, c_handle):
-        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
-        self.c_handle = c_handle
-        if int(self) != -1:
-            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
-
-    def __int__(self):
-        return int(_ffi.cast("intptr_t", self.c_handle))
-
-    def __repr__(self):
-        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
-
-    def Detach(self):
-        h = int(self)
-        if h != -1:
-            c_handle = self.c_handle
-            self.c_handle = _INVALID_HANDLE_VALUE
-            _ffi.gc(c_handle, None)
-        return h
-
-    def Close(self):
-        if int(self) != -1:
-            c_handle = self.c_handle
-            self.c_handle = _INVALID_HANDLE_VALUE
-            _ffi.gc(c_handle, None)
-            _kernel32.CloseHandle(c_handle)
-
 def CreatePipe(attributes, size):
     handles = _ffi.new("HANDLE[2]")
 
@@ -63,10 +38,10 @@
     if not res:
         raise _WinError()
 
-    return _handle(handles[0]), _handle(handles[1])
+    return _handle2int(handles[0]), _handle2int(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_kernel32.GetCurrentProcess())
+    return _handle2int(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
     # CPython: the first three arguments are expected to be integers
@@ -81,7 +56,7 @@
     if not res:
         raise _WinError()
 
-    return _handle(target[0])
+    return _handle2int(target[0])
 
 def _z(input):
     if input is None:
@@ -97,13 +72,14 @@
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
-        # CPython: these three handles are expected to be _handle objects
+        # CPython: these three handles are expected to be
+        # subprocess.Handle (int) objects
         if startup_info.hStdInput:
-            si.hStdInput = startup_info.hStdInput.c_handle
+            si.hStdInput = _int2handle(startup_info.hStdInput)
         if startup_info.hStdOutput:
-            si.hStdOutput = startup_info.hStdOutput.c_handle
+            si.hStdOutput = _int2handle(startup_info.hStdOutput)
         if startup_info.hStdError:
-            si.hStdError = startup_info.hStdError.c_handle
+            si.hStdError = _int2handle(startup_info.hStdError)
 
     pi = _ffi.new("PROCESS_INFORMATION *")
     flags |= CREATE_UNICODE_ENVIRONMENT
@@ -123,8 +99,8 @@
     if not res:
         raise _WinError()
 
-    return (_handle(pi.hProcess),
-            _handle(pi.hThread),
+    return (_handle2int(pi.hProcess),
+            _handle2int(pi.hThread),
             pi.dwProcessId,
             pi.dwThreadId)
 
@@ -164,18 +140,17 @@
         return None
     else:
         # note: returns integer, not handle object
-        return int(_ffi.cast("intptr_t", res))
+        return _handle2int(res)
 
 def CloseHandle(handle):
-    res = _kernel32.CloseHandle(_ffi.cast("HANDLE", handle))
+    res = _kernel32.CloseHandle(_int2handle(handle))
 
     if not res:
         raise _WinError()
 
 def GetModuleFileName(module):
     buf = _ffi.new("wchar_t[]", _MAX_PATH)
-    res = _kernel32.GetModuleFileNameW(_ffi.cast("HANDLE", module),
-                                       buf, _MAX_PATH)
+    res = _kernel32.GetModuleFileNameW(_int2handle(module), buf, _MAX_PATH)
 
     if not res:
         raise _WinError()

From pypy.commits at gmail.com  Thu Sep 22 23:04:18 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Thu, 22 Sep 2016 20:04:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: outdated comment
Message-ID: <57e49bb2.0d8d1c0a.56f6d.3322@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r87339:5d8577e6413d
Date: 2016-09-22 20:03 -0700
http://bitbucket.org/pypy/pypy/changeset/5d8577e6413d/

Log:	outdated comment

diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -139,7 +139,6 @@
     if not res:
         return None
     else:
-        # note: returns integer, not handle object
         return _handle2int(res)
 
 def CloseHandle(handle):

From pypy.commits at gmail.com  Fri Sep 23 04:05:46 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 23 Sep 2016 01:05:46 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: do not force alloc variable of
 trace op the bridge is attached to. use a scratch reg for x86
Message-ID: <57e4e25a.0f1f1c0a.eb063.830c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87340:d04b9daae851
Date: 2016-09-23 09:34 +0200
http://bitbucket.org/pypy/pypy/changeset/d04b9daae851/

Log:	do not force alloc variable of trace op the bridge is attached to.
	use a scratch reg for x86

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -349,6 +349,8 @@
         assert len(self.temp_boxes) == 0
         if self.longevity:
             for v in self.reg_bindings:
+                if v not in self.longevity:
+                    llop.debug_print(lltype.Void, "variable %s not in longevity\n" % v.repr({}))
                 assert self.longevity[v][1] > self.position
 
     def try_allocate_reg(self, v, selected_reg=None, need_lower_byte=False):
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -121,9 +121,15 @@
             # the upper elements will be lost if saved to the stack!
             scalar_arg = accum_info.getoriginal()
             assert isinstance(vector_loc, RegLoc)
+            assert scalar_arg is not None
+            orig_scalar_loc = scalar_loc
             if not isinstance(scalar_loc, RegLoc):
-                scalar_loc = regalloc.force_allocate_reg(scalar_arg)
-            assert scalar_arg is not None
+                # scalar loc might live in memory, use scratch register and save it back later
+                if scalar_arg.type == FLOAT:
+                    scalar_loc = X86_64_XMM_SCRATCH_REG
+                else:
+                    scalar_loc = X86_64_SCRATCH_REG
+                self.mov(orig_scalar_loc, scalar_loc)
             if accum_info.accum_operation == '+':
                 self._accum_reduce_sum(scalar_arg, vector_loc, scalar_loc)
             elif accum_info.accum_operation == '*':
@@ -131,16 +137,16 @@
             else:
                 not_implemented("accum operator %s not implemented" %
                                             (accum_info.accum_operation)) 
+            if scalar_loc is not orig_scalar_loc:
+                self.mov(scalar_loc, orig_scalar_loc)
             accum_info = accum_info.next()
 
     def _accum_reduce_mul(self, arg, accumloc, targetloc):
         scratchloc = X86_64_XMM_SCRATCH_REG
-        self.mov(accumloc, scratchloc)
+        self.mov(accumloc, targetloc)
         # swap the two elements
-        self.mc.SHUFPD_xxi(scratchloc.value, scratchloc.value, 0x01)
-        self.mc.MULSD(accumloc, scratchloc)
-        if accumloc is not targetloc:
-            self.mov(accumloc, targetloc)
+        self.mc.SHUFPD_xxi(targetloc.value, targetloc.value, 0x01)
+        self.mc.MULSD(targetloc, accumloc)
 
     def _accum_reduce_sum(self, arg, accumloc, targetloc):
         # Currently the accumulator can ONLY be the biggest
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -237,7 +237,7 @@
         self.unroll_count = self.get_unroll_count(vsize)
         align_unroll = self.unroll_count==1 and \
                        self.vector_ext.should_align_unroll
-        self.unroll_loop_iterations(info, loop, self.unroll_count,
+        self.unroll_loop_iterations(loop, self.unroll_count,
                                     align_unroll_once=align_unroll)
 
         # vectorize
@@ -262,7 +262,7 @@
 
         return loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
 
-    def unroll_loop_iterations(self, info, loop, unroll_count, align_unroll_once=False):
+    def unroll_loop_iterations(self, loop, unroll_count, align_unroll_once=False):
         """ Unroll the loop `unroll_count` times. There can be an additional unroll step
             if alignment might benefit """
         numops = len(loop.operations)

From pypy.commits at gmail.com  Fri Sep 23 04:05:48 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 23 Sep 2016 01:05:48 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: fix ppc accum issue
Message-ID: <57e4e25c.ca0e1c0a.3f6cf.8607@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87341:dfdd69e68380
Date: 2016-09-23 10:05 +0200
http://bitbucket.org/pypy/pypy/changeset/dfdd69e68380/

Log:	fix ppc accum issue

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -261,11 +261,18 @@
             vector_loc = accum_info.location
             # the upper elements will be lost if saved to the stack!
             scalar_arg = accum_info.getoriginal()
+            orig_scalar_loc = scalar_loc
             if not scalar_loc.is_reg():
-                scalar_loc = regalloc.force_allocate_reg(scalar_arg)
+                if scalar_arg.type == FLOAT:
+                    scalar_loc = r.FP_SCRATCH
+                else:
+                    scalar_loc = r.SCRATCH
+                self.regalloc_mov(orig_scalar_loc, scalar_loc)
             assert scalar_arg is not None
             op = accum_info.accum_operation
             self._accum_reduce(op, scalar_arg, vector_loc, scalar_loc)
+            if orig_scalar_loc is not scalar_loc:
+                self.regalloc_mov(scalar_loc, orig_scalar_loc)
             accum_info = accum_info.next()
 
     def _accum_reduce(self, op, arg, accumloc, targetloc):
@@ -291,11 +298,11 @@
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
             self.mc.stvx(acc, r.SCRATCH2.value, r.SP.value)
             self.mc.load(tgt, r.SP.value, PARAM_SAVE_AREA_OFFSET)
-            self.mc.load(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
+            self.mc.load(r.SCRATCH2.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
             if op == '+':
-                self.mc.add(tgt, tgt, acc)
+                self.mc.add(tgt, tgt, r.SCRATCH2.value)
             elif op == '*':
-                self.mc.mulld(tgt, tgt, acc)
+                self.mc.mulld(tgt, tgt, r.SCRATCH2.value)
             else:
                 not_implemented("sum not implemented")
             return

From pypy.commits at gmail.com  Fri Sep 23 04:07:33 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 23 Sep 2016 01:07:33 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <57e4e2c5.081dc20a.2674b.ff39@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r87342:efea0d994471
Date: 2016-09-23 10:06 +0200
http://bitbucket.org/pypy/pypy/changeset/efea0d994471/

Log:	merge default

diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -22,7 +22,10 @@
     code, message = _ffi.getwinerror()
     raise WindowsError(code, message)
 
-_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+def _int2handle(val):
+    return _ffi.cast("HANDLE", val)
+
+_INVALID_HANDLE_VALUE = _int2handle(-1)
 
 class _handle(object):
     def __init__(self, c_handle):
@@ -70,9 +73,9 @@
     target = _ffi.new("HANDLE[1]")
 
     res = _kernel32.DuplicateHandle(
-        _ffi.cast("HANDLE", source_process),
-        _ffi.cast("HANDLE", source),
-        _ffi.cast("HANDLE", target_process),
+        _int2handle(source_process),
+        _int2handle(source),
+        _int2handle(target_process),
         target, access, inherit, options)
 
     if not res:
@@ -119,12 +122,14 @@
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
+    return (_handle(pi.hProcess),
+            _handle(pi.hThread),
+            pi.dwProcessId,
+            pi.dwThreadId)
 
 def WaitForSingleObject(handle, milliseconds):
     # CPython: the first argument is expected to be an integer.
-    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
-                                        milliseconds)
+    res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds)
     if res < 0:
         raise _WinError()
 
@@ -134,7 +139,7 @@
     # CPython: the first argument is expected to be an integer.
     code = _ffi.new("DWORD[1]")
 
-    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
+    res = _kernel32.GetExitCodeProcess(_int2handle(handle), code)
 
     if not res:
         raise _WinError()
@@ -144,7 +149,7 @@
 def TerminateProcess(handle, exitcode):
     # CPython: the first argument is expected to be an integer.
     # The second argument is silently wrapped in a UINT.
-    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+    res = _kernel32.TerminateProcess(_int2handle(handle),
                                      _ffi.cast("UINT", exitcode))
 
     if not res:
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -147,13 +147,20 @@
         # (we ignore the extra values here for simplicity and portability)
         def stat_result_reduce(st):
             return (st[0], st[1], st[2], st[3], st[4],
-                    st[5], st[6], st[7], st[8], st[9],
-                    st[-3], st[-2], st[-1])
+                    st[5], st[6], st.st_atime, st.st_mtime, st.st_ctime)
 
         def stat_result_recreate(tup):
-            return make_stat_result(tup[:10] + extra_zeroes + tup[-3:])
+            atime, mtime, ctime = tup[7:]
+            result = tup[:7]
+            result += (int(atime), int(mtime), int(ctime))
+            result += extra_zeroes
+            result += (int((atime - result[7]) * 1e9),
+                       int((mtime - result[8]) * 1e9),
+                       int((ctime - result[9]) * 1e9))
+            return make_stat_result(result)
         s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE)
-                                       for name, TYPE in PORTABLE_STAT_FIELDS])
+                                    for name, TYPE in PORTABLE_STAT_FIELDS[:7]]
+                                 + 3 * [lltype_to_annotation(lltype.Float)])
         extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS) - 3)
         return s_reduced, stat_result_reduce, stat_result_recreate
 

From pypy.commits at gmail.com  Fri Sep 23 11:43:07 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 23 Sep 2016 08:43:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Kill unused MODULES_THAT_ALWAYS_SHADOW and
 associated tests (importlib handles that logic now)
Message-ID: <57e54d8b.e2cdc20a.fe841.b479@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87343:7a8153e3f911
Date: 2016-09-23 16:42 +0100
http://bitbucket.org/pypy/pypy/changeset/7a8153e3f911/

Log:	Kill unused MODULES_THAT_ALWAYS_SHADOW and associated tests
	(importlib handles that logic now)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -558,16 +558,6 @@
         'parser', 'fcntl', '_codecs', 'binascii'
     ]
 
-    # These modules are treated like CPython treats built-in modules,
-    # i.e. they always shadow any xx.py.  The other modules are treated
-    # like CPython treats extension modules, and are loaded in sys.path
-    # order by the fake entry '.../lib_pypy/__extensions__'.
-    MODULES_THAT_ALWAYS_SHADOW = dict.fromkeys([
-        '__builtin__', '__pypy__', '_ast', '_codecs', '_sre', '_warnings',
-        '_weakref', 'errno', '__exceptions__', 'gc', 'imp', 'marshal',
-        'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport',
-    ], None)
-
     def make_builtins(self):
         "NOT_RPYTHON: only for initializing the space."
 
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -63,8 +63,6 @@
                     test_reload = "def test():\n    raise ValueError\n",
                     infinite_reload = "import infinite_reload, imp; imp.reload(infinite_reload)",
                     del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n",
-                    _md5 = "hello_world = 42\n",
-                    _pypyjson = "hello_world = 42\n",
                     gc = "should_never_be_seen = 42\n",
                     )
     root.ensure("packagenamespace", dir=1)    # empty, no __init__.py
@@ -158,7 +156,7 @@
 
 def _setup_path(space, path):
     return space.appexec([space.wrap(path)], """
-        (dn): 
+        (dn):
             import sys
             path = list(sys.path)
             sys.path.insert(0, dn)
@@ -660,52 +658,6 @@
                                      ) == 'a/b/c.py'
         raises(ValueError, imp.source_from_cache, 'a/b/c.py')
 
-    def test_shadow_builtin(self):
-        if self.runappdirect: skip("hard to test: module is already imported")
-        # 'import gc' is supposed to always find the built-in module;
-        # like CPython, it is a built-in module, so it shadows everything,
-        # even though there is a gc.py.
-        import sys
-        assert 'gc' not in sys.modules
-        import gc
-        assert not hasattr(gc, 'should_never_be_seen')
-        assert '(built-in)' in repr(gc)
-        del sys.modules['gc']
-
-    def test_shadow_extension_1(self):
-        if not self.runappdirect:
-            skip("I don't understand why it fails, but it works in -A "
-                 "on top of a translated PyPy.  Good enough...")
-        # 'import _pypyjson' is supposed to find _pypyjson.py if there is
-        # one in sys.path.
-        import sys
-        assert '_pypyjson' not in sys.modules
-        try:
-            import _pypyjson
-            assert hasattr(_pypyjson, 'hello_world')
-            assert '(built-in)' not in repr(_pypyjson)
-        finally:
-            sys.modules.pop('_pypyjson', None)
-
-    def test_shadow_extension_2(self):
-        if self.runappdirect: skip("hard to test: module is already imported")
-        # 'import _md5' is supposed to find the built-in module even
-        # if there is also one in sys.path as long as it is *after* the
-        # special entry '.../lib_pypy/__extensions__'.  (Note that for now
-        # there is one in lib_pypy/_md5.py, which should not be seen
-        # either; hence the (built-in) test below.)
-        import sys
-        assert '_md5' not in sys.modules
-        sys.path.append(sys.path.pop(0))
-        try:
-            import _md5
-            assert not hasattr(_md5, 'hello_world')
-            assert hasattr(_md5, 'md5')
-            assert '(built-in)' in repr(_md5)
-        finally:
-            sys.path.insert(0, sys.path.pop())
-            sys.modules.pop('_md5', None)
-
     def test_invalid_pathname(self):
         skip("This test fails on CPython 3.3, but passes on CPython 3.4+")
         import imp
@@ -993,11 +945,11 @@
         assert s.no_nul
 
 
-def test_PYTHONPATH_takes_precedence(space): 
+def test_PYTHONPATH_takes_precedence(space):
     if sys.platform == "win32":
         py.test.skip("unresolved issues with win32 shell quoting rules")
-    from pypy.interpreter.test.test_zpy import pypypath 
-    extrapath = udir.ensure("pythonpath", dir=1) 
+    from pypy.interpreter.test.test_zpy import pypypath
+    extrapath = udir.ensure("pythonpath", dir=1)
     extrapath.join("sched.py").write("print(42)\n")
     old = os.environ.get('PYTHONPATH', None)
     oldlang = os.environ.pop('LANG', None)

From pypy.commits at gmail.com  Fri Sep 23 12:20:25 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 23 Sep 2016 09:20:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix test_load_dynamic() to match CPython
 behaviour
Message-ID: <57e55649.a2ddc20a.4209c.cb56@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87344:6ff2d69f043f
Date: 2016-09-23 17:19 +0100
http://bitbucket.org/pypy/pypy/changeset/6ff2d69f043f/

Log:	Fix test_load_dynamic() to match CPython behaviour

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -54,11 +54,14 @@
         finally:
             del sys.path[0]
 
-    def test_load_dynamic(self):
-        import imp
-        raises(ImportError, imp.load_dynamic, 'foo', 'bar')
-        raises(ImportError, imp.load_dynamic, 'foo', 'bar',
-               open(self.file_module))
+    def test_load_dynamic_error(self):
+        import _imp
+        excinfo = raises(ImportError, _imp.load_dynamic, 'foo', 'bar')
+        assert excinfo.value.name == 'foo'
+        assert 'bar' in excinfo.value.path
+        # Note: On CPython, the behavior changes slightly if a 3rd argument is
+        # passed in, whose value is ignored. We don't implement that.
+        #raises(IOError, _imp.load_dynamic, 'foo', 'bar', 42)
 
     def test_suffixes(self):
         import imp

From pypy.commits at gmail.com  Fri Sep 23 12:40:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 09:40:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: utf-8 encoding before translation accepts
 lone surrogates, because it is
Message-ID: <57e55af7.c22e1c0a.1c008.5177@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87345:dd7bb8b61422
Date: 2016-09-23 11:59 +0200
http://bitbucket.org/pypy/pypy/changeset/dd7bb8b61422/

Log:	utf-8 encoding before translation accepts lone surrogates, because
	it is Python 2.7, but after translation it does not. Moreover,
	CPython 3.x accepts such unicode attributes anyway. This makes this
	test half-wrong for now.

diff --git a/pypy/objspace/test/test_descriptor.py b/pypy/objspace/test/test_descriptor.py
--- a/pypy/objspace/test/test_descriptor.py
+++ b/pypy/objspace/test/test_descriptor.py
@@ -73,6 +73,10 @@
         raises(AttributeError, X.v.__delete__, x)
 
     def test_invalid_unicode_identifier(self):
+        skip("utf-8 encoding before translation accepts lone surrogates, "
+             "because it is Python 2.7, but after translation it does not. "
+             "Moreover, CPython 3.x accepts such unicode attributes anyway. "
+             "This makes this test half-wrong for now.")
         class X(object):
             pass
         x = X()

From pypy.commits at gmail.com  Fri Sep 23 12:40:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 09:40:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: A branch to really implement
 faulthandler
Message-ID: <57e55af9.4398c20a.53134.d620@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87346:582203d71d95
Date: 2016-09-23 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/582203d71d95/

Log:	A branch to really implement faulthandler


From pypy.commits at gmail.com  Fri Sep 23 12:40:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 09:40:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: merge heads
Message-ID: <57e55afb.8c9a1c0a.bfc9a.5829@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87347:cb1bd427cb74
Date: 2016-09-23 18:39 +0200
http://bitbucket.org/pypy/pypy/changeset/cb1bd427cb74/

Log:	merge heads

diff --git a/pypy/objspace/test/test_descriptor.py b/pypy/objspace/test/test_descriptor.py
--- a/pypy/objspace/test/test_descriptor.py
+++ b/pypy/objspace/test/test_descriptor.py
@@ -73,6 +73,10 @@
         raises(AttributeError, X.v.__delete__, x)
 
     def test_invalid_unicode_identifier(self):
+        skip("utf-8 encoding before translation accepts lone surrogates, "
+             "because it is Python 2.7, but after translation it does not. "
+             "Moreover, CPython 3.x accepts such unicode attributes anyway. "
+             "This makes this test half-wrong for now.")
         class X(object):
             pass
         x = X()

From pypy.commits at gmail.com  Fri Sep 23 12:53:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 09:53:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: in-progress
Message-ID: <57e55df8.ca11c30a.d4df7.d51a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87348:0c67f2b605a3
Date: 2016-09-23 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/0c67f2b605a3/

Log:	in-progress

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -422,6 +422,11 @@
         # To be called before using the space
         self.threadlocals.enter_thread(self)
 
+        # Set up faulthandler even if not imported explicitly
+        if self.config.objspace.usemodule.faulthandler:
+            from pypy.module.faulthandler import handler
+            handler.startup(self)
+
         # Initialize already imported builtin modules
         from pypy.interpreter.module import Module
         w_modules = self.sys.get('modules')
@@ -445,6 +450,11 @@
         for w_mod in self.builtin_modules.values():
             if isinstance(w_mod, Module) and w_mod.startup_called:
                 w_mod.shutdown(self)
+        #
+        # Shut down faulthandler
+        if self.config.objspace.usemodule.faulthandler:
+            from pypy.module.faulthandler import handler
+            handler.finish(self)
 
     def wait_for_thread_shutdown(self):
         """Wait until threading._shutdown() completes, provided the threading
diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py
--- a/pypy/module/faulthandler/__init__.py
+++ b/pypy/module/faulthandler/__init__.py
@@ -5,18 +5,18 @@
     }
 
     interpleveldefs = {
-        'enable': 'interp_faulthandler.enable',
-        'disable': 'interp_faulthandler.disable',
-        'is_enabled': 'interp_faulthandler.is_enabled',
-        'register': 'interp_faulthandler.register',
-
-        'dump_traceback': 'interp_faulthandler.dump_traceback',
-
-        '_read_null': 'interp_faulthandler.read_null',
-        '_sigsegv': 'interp_faulthandler.sigsegv',
-        '_sigfpe': 'interp_faulthandler.sigfpe',
-        '_sigabrt': 'interp_faulthandler.sigabrt',
-        #'_sigbus': 'interp_faulthandler.sigbus',
-        #'_sigill': 'interp_faulthandler.sigill',
-        '_fatal_error': 'interp_faulthandler.fatal_error',
+        'enable': 'handler.enable',
+        'disable': 'handler.disable',
+        'is_enabled': 'handler.is_enabled',
+#        'register': 'interp_faulthandler.register',
+#
+#        'dump_traceback': 'interp_faulthandler.dump_traceback',
+#
+#        '_read_null': 'interp_faulthandler.read_null',
+#        '_sigsegv': 'interp_faulthandler.sigsegv',
+#        '_sigfpe': 'interp_faulthandler.sigfpe',
+#        '_sigabrt': 'interp_faulthandler.sigabrt',
+#        #'_sigbus': 'interp_faulthandler.sigbus',
+#        #'_sigill': 'interp_faulthandler.sigill',
+#        '_fatal_error': 'interp_faulthandler.fatal_error',
     }
diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/faulthandler/cintf.py
@@ -0,0 +1,25 @@
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator import cdir
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+
+cwd = py.path.local(__file__).dirpath()
+eci = ExternalCompilationInfo(
+    includes=[cwd.join('faulthandler.h')],
+    include_dirs=[str(cwd), cdir],
+    separate_module_files=[cwd.join('faulthandler.c')])
+
+def llexternal(*args, **kwargs):
+    kwargs.setdefault('releasegil', False)
+    kwargs.setdefault('compilation_info', eci)
+    return rffi.llexternal(*args, **kwargs)
+
+pypy_faulthandler_setup = llexternal(
+    'pypy_faulthandler_setup', [], lltype.Void)
+pypy_faulthandler_teardown = llexternal(
+    'pypy_faulthandler_teardown', [], lltype.Void)
+pypy_faulthandler_enable = llexternal(
+    'pypy_faulthandler_enable', [], lltype.Void,
+    save_err=rffi.RFFI_SAVE_ERRNO)
+pypy_faulthandler_disable = llexternal(
+    'pypy_faulthandler_disable', [], lltype.Void)
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -1,6 +1,135 @@
+#include "faulthandler.h"
 #include <stdlib.h>
-#include "faulthandler.h"
+#include <signal.h>
+#include <assert.h>
+#include <errno.h>
 
+
+typedef struct sigaction _Py_sighandler_t;
+
+typedef struct {
+    int signum;
+    int enabled;
+    const char* name;
+    _Py_sighandler_t previous;
+    int all_threads;
+} fault_handler_t;
+
+static struct {
+    int enabled;
+    int fd, all_threads;
+} fatal_error;
+
+static stack_t stack;
+
+
+static fault_handler_t faulthandler_handlers[] = {
+#ifdef SIGBUS
+    {SIGBUS, 0, "Bus error", },
+#endif
+#ifdef SIGILL
+    {SIGILL, 0, "Illegal instruction", },
+#endif
+    {SIGFPE, 0, "Floating point exception", },
+    {SIGABRT, 0, "Aborted", },
+    /* define SIGSEGV at the end to make it the default choice if searching the
+       handler fails in faulthandler_fatal_error() */
+    {SIGSEGV, 0, "Segmentation fault", }
+};
+static const int faulthandler_nsignals =
+    sizeof(faulthandler_handlers) / sizeof(fault_handler_t);
+
+
+RPY_EXTERN
+char *pypy_faulthandler_setup(void)
+{
+    assert(!fatal_error.enabled);
+
+    /* Try to allocate an alternate stack for faulthandler() signal handler to
+     * be able to allocate memory on the stack, even on a stack overflow. If it
+     * fails, ignore the error. */
+    stack.ss_flags = 0;
+    stack.ss_size = SIGSTKSZ;
+    stack.ss_sp = malloc(stack.ss_size);
+    if (stack.ss_sp != NULL) {
+        int err = sigaltstack(&stack, NULL);
+        if (err) {
+            free(stack.ss_sp);
+            stack.ss_sp = NULL;
+        }
+    }
+    return NULL;
+}
+
+RPY_EXTERN
+void pypy_faulthandler_teardown(void)
+{
+    pypy_faulthandler_disable();
+    free(stack.ss_sp);
+    stack.ss_sp = NULL;
+}
+
+RPY_EXTERN
+int pypy_faulthandler_enable(int fd, int all_threads)
+{
+    fatal_error.fd = fd;
+    fatal_error.all_threads = all_threads;
+
+    if (!fatal_error.enabled) {
+        int i;
+
+        fatal_error.enabled = 1;
+
+        for (i = 0; i < faulthandler_nsignals; i++) {
+            int err;
+            struct sigaction action;
+            fault_handler_t *handler = &faulthandler_handlers[i];
+
+            action.sa_handler = faulthandler_fatal_error;
+            sigemptyset(&action.sa_mask);
+            /* Do not prevent the signal from being received from within
+               its own signal handler */
+            action.sa_flags = SA_NODEFER;
+            if (stack.ss_sp != NULL) {
+                /* Call the signal handler on an alternate signal stack
+                   provided by sigaltstack() */
+                action.sa_flags |= SA_ONSTACK;
+            }
+            err = sigaction(handler->signum, &action, &handler->previous);
+            if (err) {
+                return -1;
+            }
+            handler->enabled = 1;
+        }
+    }
+    return 0;
+}
+
+RPY_EXTERN
+void pypy_faulthandler_disable(void)
+{
+    if (fatal_error.enabled) {
+        int i;
+
+        fatal_error.enabled = 0;
+        for (i = 0; i < faulthandler_nsignals; i++) {
+            fault_handler_t *handler = &faulthandler_handlers[i];
+            if (!handler->enabled)
+                continue;
+            (void)sigaction(handler->signum, &handler->previous, NULL);
+            handler->enabled = 0;
+        }
+    }
+}
+
+RPY_EXTERN
+int pypy_faulthandler_is_enabled(void)
+{
+    return fatal_error.enabled;
+}
+
+
+#if 0
 int
 pypy_faulthandler_read_null(void)
 {
@@ -75,3 +204,4 @@
     raise(SIGILL);
 }
 #endif
+#endif
diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -1,9 +1,16 @@
 #ifndef PYPY_FAULTHANDLER_H
 #define PYPY_FAULTHANDLER_H
 
-#include <signal.h>
 #include "src/precommondefs.h"
 
+RPY_EXTERN char *pypy_faulthandler_setup(void);
+RPY_EXTERN void pypy_faulthandler_teardown(void);
+
+RPY_EXTERN int pypy_faulthandler_enable(int fd, int all_threads);
+RPY_EXTERN void pypy_faulthandler_disable(void);
+RPY_EXTERN int pypy_faulthandler_is_enabled(void);
+
+/*
 RPY_EXTERN int pypy_faulthandler_read_null(void);
 RPY_EXTERN void pypy_faulthandler_sigsegv(void);
 RPY_EXTERN int pypy_faulthandler_sigfpe(void);
@@ -15,5 +22,6 @@
 #ifdef SIGILL
 RPY_EXTERN void pypy_faulthandler_sigill(void);
 #endif
+*/
 
 #endif  /* PYPY_FAULTHANDLER_H */
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/faulthandler/handler.py
@@ -0,0 +1,108 @@
+import os
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rposix import is_valid_fd
+
+from pypy.interpreter.error import oefmt, exception_from_saved_errno
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.module.faulthandler import cintf
+
+
+class Handler(object):
+    def __init__(self, space):
+        self.space = space
+        self._cleanup_()
+
+    def _cleanup_(self):
+        self.is_initialized = False
+        self.fatal_error_w_file = None
+
+    def check_err(self, p_err):
+        if p_err:
+            raise oefmt(self.space.w_RuntimeError, 'faulthandler: %8',
+                        rffi.charp2str(p_err))
+
+    def get_fileno_and_file(self, w_file):
+        space = self.space
+        if space.is_none(w_file):
+            w_file = space.sys.get('stderr')
+            if space.is_none(w_file):
+                raise oefmt(space.w_RuntimeError, "sys.stderr is None")
+        elif space.isinstance_w(w_file, space.w_int):
+            fd = space.int_w(w_file)
+            if fd < 0 or not is_valid_fd(fd):
+                raise oefmt(space.w_ValueError,
+                            "file is not a valid file descriptor")
+            return fd, None
+
+        fd = space.int_w(space.call_method(w_file, 'fileno'))
+        try:
+            space.call_method(w_file, 'flush')
+        except OperationError as e:
+            if e.async(space):
+                raise
+            pass   # ignore flush() error
+        return fd, w_file
+
+    def enable(self, w_file, all_threads):
+        fileno, w_file = self.get_fileno_and_file(w_file)
+        if not self.is_initialized:
+            self.check_err(cintf.pypy_faulthandler_setup())
+            self.is_initialized = True
+
+        self.fatal_error_w_file = w_file
+        err = cintf.pypy_faulthandler_enable(fileno, all_threads)
+        if err:
+            space = self.space
+            raise exception_from_saved_errno(space, space.w_RuntimeError)
+
+    def disable(self):
+        cintf.pypy_faulthandler_disable()
+        self.fatal_error_w_file = None
+
+    def is_enabled(self):
+        return (self.is_initialized and
+                bool(cintf.pypy_faulthandler_is_enabled()))
+
+    def finish(self):
+        if self.is_initialized:
+            cintf.pypy_faulthandler_teardown()
+        self._cleanup_()
+
+
+def startup(space):
+    """Initialize the faulthandler logic when the space is starting
+    (this is called from baseobjspace.py)"""
+    #
+    # Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable
+    # is defined, or if sys._xoptions has a 'faulthandler' key.
+    if not os.environ.get('PYTHONFAULTHANDLER'):
+        w_options = space.sys.get('_xoptions')
+        if not space.contains(w_options, space.wrap('faulthandler')):
+            return
+    #
+    # Like CPython.  Why not just call enable(space)?  Maybe the goal is
+    # to let the user override the 'faulthandler' module.  Maybe someone
+    # mis-uses ``"faulthandler" in sys.modules'' as a way to check if it
+    # was started by checking if it was imported at all.
+    space.appexec([], """
+        import faulthandler
+        faulthandler.enable()
+    """)
+
+def finish(space):
+    """Finalize the faulthandler logic (called from baseobjspace.py)"""
+    space.fromcache(Handler).finish()
+
+
+ at unwrap_spec(all_threads=int)
+def enable(space, w_file=None, all_threads=0):
+    "enable(file=sys.stderr, all_threads=True): enable the fault handler"
+    space.fromcache(Handler).enable(w_file, all_threads)
+
+def disable(space):
+    "disable(): disable the fault handler"
+    space.fromcache(Handler).disable()
+
+def is_enabled(space):
+    "is_enabled()->bool: check if the handler is enabled"
+    return space.wrap(space.fromcache(Handler).is_enabled())
diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py
--- a/pypy/module/faulthandler/test/test_faulthander.py
+++ b/pypy/module/faulthandler/test/test_faulthander.py
@@ -6,7 +6,7 @@
 
 class AppTestFaultHandler:
     spaceconfig = {
-        "usemodules": ["faulthandler"]
+        "usemodules": ["faulthandler", "_vmprof"]
     }
 
     def test_enable(self):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -136,6 +136,7 @@
             space.setitem(self.w_dict, space.wrap('thread_info'), thread_info)
 
     def setup_after_space_initialization(self):
+        "NOT_RPYTHON"
         space = self.space
 
         if not space.config.translating:
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -37,3 +37,8 @@
 
 def disable():
     _get_vmprof().disable()
+
+ at specialize.arg(0)
+def enum_all_code_objs(CodeClass, callback, arg):
+    assert _was_registered(CodeClass)
+    CodeClass._vmprof_enum_all_code_objs(callback, arg)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,6 +1,6 @@
 import sys, os
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib import jit, rposix
+from rpython.rlib import jit, rposix, rgc
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
@@ -116,6 +116,22 @@
         # the types of code objects
         prev = self._gather_all_code_objs
         self._gather_all_code_objs = gather_all_code_objs
+        #
+        # For special usages: the faulthandler pypy module uses this.
+        # It must not allocate anything.
+        @staticmethod
+        @rgc.no_collect
+        def enum_all_code_objs(callback, arg):
+            all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles()
+            i = len(all_code_wrefs) - 1
+            while i >= 0:
+                code = all_code_wrefs[i]()
+                i -= 1
+                if code is not None:
+                    uid = code._vmprof_unique_id
+                    if uid != 0:
+                        callback(code, uid, arg)
+        CodeClass._vmprof_enum_all_code_objs = enum_all_code_objs
 
     @jit.dont_look_inside
     def enable(self, fileno, interval):

From pypy.commits at gmail.com  Fri Sep 23 13:20:38 2016
From: pypy.commits at gmail.com (sbauman)
Date: Fri, 23 Sep 2016 10:20:38 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Prospective test
Message-ID: <57e56466.854bc20a.a4390.e053@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87349:2a49f9a70838
Date: 2016-09-23 13:16 -0400
http://bitbucket.org/pypy/pypy/changeset/2a49f9a70838/

Log:	Prospective test

diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -965,17 +965,19 @@
         self.check_aborted_count(0)
         self.check_target_token_count(4)
 
-    def test_avoid_preamble(self):
+    def test_avoid_preamble_1(self):
         driver = JitDriver(greens=[], reds=['i', 'val'])
         class X(object):
+            _value_class_ = True
             def __init__(self, v):
                 self.v = v
 
+        # A mutable cell to prevent the JIT from inferring that the non-virtual
+        # object is a constant.
         class Box(object):
             def __init__(self, v):
                 self.unbox = v
 
-        mask = -2
         const = Box(X(5))
         def f():
             # Prevent all retracing of side exits. Ensures that the unroll
@@ -1006,6 +1008,51 @@
         self.check_target_token_count(2)
         self.check_trace_count(3)
 
+    def test_avoid_preamble_2(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+        class X(object):
+            _value_class_ = True
+            def __init__(self, v):
+                self.v = v
+
+        # A mutable cell to prevent the JIT from inferring that the non-virtual
+        # object is a constant.
+        class Box(object):
+            def __init__(self, v):
+                self.unbox = v
+
+        const = Box(X(5))
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            val   = X(0)
+            i     = 0
+            const.unbox = X(5)
+            while i < 17:
+                driver.can_enter_jit(i=i, val=val)
+                driver.jit_merge_point(i=i, val=val)
+                # Logical & rather than comparison to confuse range analysis.
+                # Test only succeeds on the first 2 iterations
+                if i & -2 == 0:
+                    val = X(i)
+                else:
+                    val = const.unbox
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
+
+        # With retracing disable, there will be one optimized loop expecting a
+        # virtual X object. The side exit in this case will contain a non-virtual
+        # value class, which should be unpacked into the
+
+        # self.check_resops(jump=3, label=2, new_with_vtable=2)
+        # self.check_target_token_count(2)
+        # self.check_trace_count(3)
+
+
     def test_conflated_virtual_states(self):
         # All cases are covered when forcing one component of the virtual state
         # also forces an as yet unseen component.

From pypy.commits at gmail.com  Fri Sep 23 14:05:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 11:05:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: in-progress
Message-ID: <57e56efa.e2cdc20a.fe841.e9b0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87350:d7433f65fd0c
Date: 2016-09-23 20:05 +0200
http://bitbucket.org/pypy/pypy/changeset/d7433f65fd0c/

Log:	in-progress

diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
@@ -9,17 +9,30 @@
     include_dirs=[str(cwd), cdir],
     separate_module_files=[cwd.join('faulthandler.c')])
 
-def llexternal(*args, **kwargs):
-    kwargs.setdefault('releasegil', False)
+def direct_llexternal(*args, **kwargs):
+    kwargs.setdefault('_nowrapper', True)
     kwargs.setdefault('compilation_info', eci)
     return rffi.llexternal(*args, **kwargs)
 
-pypy_faulthandler_setup = llexternal(
-    'pypy_faulthandler_setup', [], lltype.Void)
-pypy_faulthandler_teardown = llexternal(
+DUMP_CALLBACK = lltype.Ptr(lltype.FuncType([], lltype.Void))
+
+pypy_faulthandler_setup = direct_llexternal(
+    'pypy_faulthandler_setup', [DUMP_CALLBACK], rffi.CCHARP)
+
+pypy_faulthandler_teardown = direct_llexternal(
     'pypy_faulthandler_teardown', [], lltype.Void)
-pypy_faulthandler_enable = llexternal(
-    'pypy_faulthandler_enable', [], lltype.Void,
-    save_err=rffi.RFFI_SAVE_ERRNO)
-pypy_faulthandler_disable = llexternal(
+
+pypy_faulthandler_enable = direct_llexternal(
+    'pypy_faulthandler_enable', [rffi.INT, rffi.INT], rffi.CCHARP)
+
+pypy_faulthandler_disable = direct_llexternal(
     'pypy_faulthandler_disable', [], lltype.Void)
+
+pypy_faulthandler_is_enabled = direct_llexternal(
+    'pypy_faulthandler_is_enabled', [], rffi.INT)
+
+pypy_faulthandler_write = direct_llexternal(
+    'pypy_faulthandler_write', [lltype.Ptr(rstr.STR)])
+
+pypy_faulthandler_write_int = direct_llexternal(
+    'pypy_faulthandler_write_int', [lltype.Signed])
diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/faulthandler/dumper.py
@@ -0,0 +1,58 @@
+from rpython.rtyper.annlowlevel import llstr
+from rpython.rlib import rgc
+from rpython.rlib.rvmprof import enum_all_code_objs
+from rpython.rlib.rvmprof import cintf as rvmprof_cintf
+
+from pypy.interpreter.pycode import PyCode
+from pypy.module.faulthandler.cintf import pypy_faulthandler_write
+from pypy.module.faulthandler.cintf import pypy_faulthandler_write_int
+
+#
+# xxx The dumper is tied to the internals of rvmprof. xxx
+#
+
+
+MAX_FRAME_DEPTH = 100
+
+
+def _dump(s):
+    pypy_faulthandler_write(llstr(s))
+
+def _dump_int(i):
+    pypy_faulthandler_write_int(i)
+
+
+def dump_code(pycode, this_code_id, search_code_id):
+    if this_code_id != search_code_id:
+        return 0
+    _dump('"')
+    _dump(pycode.co_filename)
+    _dump('" in ')
+    _dump(pycode.co_name)
+    _dump(" (starting at line ")
+    _dump_int(pycode.co_firstlineno)
+    _dump(")\n")
+    return 1
+
+
+ at rgc.no_collect
+def _dump_callback():
+    """We are as careful as we can reasonably be here (i.e. not 100%,
+    but hopefully close enough).  In particular, this is written as
+    RPython but shouldn't allocate anything.
+    """
+    _dump("Stack (most recent call first):\n")
+
+    s = rvmprof_cintf.get_rvmprof_stack()
+    depth = 0
+    while s:
+        if depth >= MAX_FRAME_DEPTH:
+            _dump("  ...\n")
+            break
+        if s.c_kind == rvmprof_cintf.VMPROF_CODE_TAG:
+            code_id = s.c_value
+            _dump("  File ")
+            if enum_all_code_objs(PyCode, dump_code, code_id) == 0:
+                _dump("???\n")
+        s = s.c_next
+        depth += 1
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -3,21 +3,24 @@
 #include <signal.h>
 #include <assert.h>
 #include <errno.h>
+#include <string.h>
 
 
 typedef struct sigaction _Py_sighandler_t;
 
 typedef struct {
-    int signum;
-    int enabled;
+    const int signum;
+    volatile int enabled;
     const char* name;
     _Py_sighandler_t previous;
-    int all_threads;
 } fault_handler_t;
 
 static struct {
+    int initialized;
     int enabled;
-    int fd, all_threads;
+    volatile int fd, all_threads;
+    volatile void (*dump_traceback)(void);
+    int _current_fd;
 } fatal_error;
 
 static stack_t stack;
@@ -39,11 +42,106 @@
 static const int faulthandler_nsignals =
     sizeof(faulthandler_handlers) / sizeof(fault_handler_t);
 
+static void
+fh_write(int fd, char *str)
+{
+    (void)write(fd, str, strlen(str));
+}
 
 RPY_EXTERN
-char *pypy_faulthandler_setup(void)
+void pypy_faulthandler_write(RPyString *s)
 {
+    (void)write(fatal_error._current_fd,
+                _RPyString_AsString(s), RPyString_Size(s));
+}
+
+RPY_EXTERN
+void pypy_faulthandler_write_int(long x)
+{
+    char buf[32];
+    int count = sprintf(buf, "%ld", x);
+    (void)write(fatal_error._current_fd, buf, count);
+}
+
+
+static void
+faulthandler_dump_traceback(int fd, int all_threads)
+{
+    static volatile int reentrant = 0;
+
+    if (reentrant)
+        return;
+    reentrant = 1;
+    fatal_error._current_fd = fd;
+
+    /* XXX 'all_threads' ignored */
+    if (fatal_error.dump_traceback)
+        fatal_error.dump_traceback();
+
+    reentrant = 0;
+}
+
+
+/* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals.
+
+   Display the current Python traceback, restore the previous handler and call
+   the previous handler.
+
+   On Windows, don't explicitly call the previous handler, because the Windows
+   signal handler would not be called (for an unknown reason). The execution of
+   the program continues at faulthandler_fatal_error() exit, but the same
+   instruction will raise the same fault (signal), and so the previous handler
+   will be called.
+
+   This function is signal-safe and should only call signal-safe functions. */
+
+static void
+faulthandler_fatal_error(int signum)
+{
+    int fd = fatal_error.fd;
+    int i;
+    fault_handler_t *handler = NULL;
+    int save_errno = errno;
+
+    for (i = 0; i < faulthandler_nsignals; i++) {
+        handler = &faulthandler_handlers[i];
+        if (handler->signum == signum)
+            break;
+    }
+
+    /* restore the previous handler */
+    if (handler->enabled) {
+        (void)sigaction(signum, &handler->previous, NULL);
+        handler->enabled = 0;
+    }
+
+    fh_write(fd, "Fatal Python error: ");
+    fh_write(fd, handler->name);
+    fh_write(fd, "\n\n");
+
+    faulthandler_dump_traceback(fd, fatal_error.all_threads);
+
+    errno = save_errno;
+#ifdef MS_WINDOWS
+    if (signum == SIGSEGV) {
+        /* don't explicitly call the previous handler for SIGSEGV in this signal
+           handler, because the Windows signal handler would not be called */
+        return;
+    }
+#endif
+    /* call the previous signal handler: it is called immediately if we use
+       sigaction() thanks to SA_NODEFER flag, otherwise it is deferred */
+    raise(signum);
+}
+
+
+RPY_EXTERN
+char *pypy_faulthandler_setup(void dump_callback(void))
+{
+    if (fatal_error.initialized)
+        return;
     assert(!fatal_error.enabled);
+    fatal_error.dump_callback = dump_callback;
 
     /* Try to allocate an alternate stack for faulthandler() signal handler to
      * be able to allocate memory on the stack, even on a stack overflow. If it
@@ -58,26 +156,32 @@
             stack.ss_sp = NULL;
         }
     }
+
+    fatal_error.fd = -1;
+    fatal_error.initialized = 1;
     return NULL;
 }
 
 RPY_EXTERN
 void pypy_faulthandler_teardown(void)
 {
-    pypy_faulthandler_disable();
-    free(stack.ss_sp);
-    stack.ss_sp = NULL;
+    if (fatal_error.initialized) {
+        pypy_faulthandler_disable();
+        fatal_error.initialized = 0;
+        free(stack.ss_sp);
+        stack.ss_sp = NULL;
+    }
 }
 
 RPY_EXTERN
-int pypy_faulthandler_enable(int fd, int all_threads)
+char *pypy_faulthandler_enable(int fd, int all_threads)
 {
+    /* Install the handler for fatal signals, faulthandler_fatal_error(). */
+    int i;
     fatal_error.fd = fd;
     fatal_error.all_threads = all_threads;
 
     if (!fatal_error.enabled) {
-        int i;
-
         fatal_error.enabled = 1;
 
         for (i = 0; i < faulthandler_nsignals; i++) {
@@ -97,20 +201,19 @@
             }
             err = sigaction(handler->signum, &action, &handler->previous);
             if (err) {
-                return -1;
+                return strerror(errno);
             }
             handler->enabled = 1;
         }
     }
-    return 0;
+    return NULL;
 }
 
 RPY_EXTERN
 void pypy_faulthandler_disable(void)
 {
+    int i;
     if (fatal_error.enabled) {
-        int i;
-
         fatal_error.enabled = 0;
         for (i = 0; i < faulthandler_nsignals; i++) {
             fault_handler_t *handler = &faulthandler_handlers[i];
@@ -120,6 +223,7 @@
             handler->enabled = 0;
         }
     }
+    fatal_error.fd = -1;
 }
 
 RPY_EXTERN
diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -1,15 +1,18 @@
 #ifndef PYPY_FAULTHANDLER_H
 #define PYPY_FAULTHANDLER_H
 
-#include "src/precommondefs.h"
+#include "common_header.h"
 
-RPY_EXTERN char *pypy_faulthandler_setup(void);
+RPY_EXTERN char *pypy_faulthandler_setup(void dump_callback(void));
 RPY_EXTERN void pypy_faulthandler_teardown(void);
 
-RPY_EXTERN int pypy_faulthandler_enable(int fd, int all_threads);
+RPY_EXTERN char *pypy_faulthandler_enable(int fd, int all_threads);
 RPY_EXTERN void pypy_faulthandler_disable(void);
 RPY_EXTERN int pypy_faulthandler_is_enabled(void);
 
+RPY_EXTERN void pypy_faulthandler_write(RPyString *);
+RPY_EXTERN void pypy_faulthandler_write_int(long);
+
 /*
 RPY_EXTERN int pypy_faulthandler_read_null(void);
 RPY_EXTERN void pypy_faulthandler_sigsegv(void);
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -1,19 +1,22 @@
 import os
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rposix import is_valid_fd
+from rpython.rlib.rarithmetic import widen
+from rpython.rtyper.annlowlevel import llhelper
 
 from pypy.interpreter.error import oefmt, exception_from_saved_errno
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.faulthandler import cintf
+from pypy.module.faulthandler import cintf, dumper
 
 
 class Handler(object):
     def __init__(self, space):
+        "NOT_RPYTHON"
         self.space = space
+        dumper.glob.space = space
         self._cleanup_()
 
     def _cleanup_(self):
-        self.is_initialized = False
         self.fatal_error_w_file = None
 
     def check_err(self, p_err):
@@ -28,13 +31,13 @@
             if space.is_none(w_file):
                 raise oefmt(space.w_RuntimeError, "sys.stderr is None")
         elif space.isinstance_w(w_file, space.w_int):
-            fd = space.int_w(w_file)
+            fd = space.c_int_w(w_file)
             if fd < 0 or not is_valid_fd(fd):
                 raise oefmt(space.w_ValueError,
                             "file is not a valid file descriptor")
             return fd, None
 
-        fd = space.int_w(space.call_method(w_file, 'fileno'))
+        fd = space.c_int_w(space.call_method(w_file, 'fileno'))
         try:
             space.call_method(w_file, 'flush')
         except OperationError as e:
@@ -45,27 +48,24 @@
 
     def enable(self, w_file, all_threads):
         fileno, w_file = self.get_fileno_and_file(w_file)
-        if not self.is_initialized:
-            self.check_err(cintf.pypy_faulthandler_setup())
-            self.is_initialized = True
-
+        #
+        dump_callback = llhelper(cintf.DUMP_CALLBACK, dumper._dump_callback)
+        self.check_err(cintf.pypy_faulthandler_setup(dump_callback))
+        #
         self.fatal_error_w_file = w_file
-        err = cintf.pypy_faulthandler_enable(fileno, all_threads)
-        if err:
-            space = self.space
-            raise exception_from_saved_errno(space, space.w_RuntimeError)
+        self.check_err(cintf.pypy_faulthandler_enable(
+            rffi.cast(rffi.INT, fileno),
+            rffi.cast(rffi.INT, all_threads)))
 
     def disable(self):
         cintf.pypy_faulthandler_disable()
         self.fatal_error_w_file = None
 
     def is_enabled(self):
-        return (self.is_initialized and
-                bool(cintf.pypy_faulthandler_is_enabled()))
+        return bool(widen(cintf.pypy_faulthandler_is_enabled()))
 
     def finish(self):
-        if self.is_initialized:
-            cintf.pypy_faulthandler_teardown()
+        cintf.pypy_faulthandler_teardown()
         self._cleanup_()
 
 
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -154,3 +154,9 @@
 
 def restore_rvmprof_stack(x):
     vmprof_tl_stack.setraw(x)
+
+#
+# faulthandler support
+
+def get_rvmprof_stack():
+    return vmprof_tl_stack.getraw()
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -130,7 +130,10 @@
                 if code is not None:
                     uid = code._vmprof_unique_id
                     if uid != 0:
-                        callback(code, uid, arg)
+                        res = callback(code, uid, arg)
+                        if res != 0:
+                            return res
+            return 0
         CodeClass._vmprof_enum_all_code_objs = enum_all_code_objs
 
     @jit.dont_look_inside

From pypy.commits at gmail.com  Fri Sep 23 14:05:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 11:05:54 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57e56f02.64f9c20a.f5f48.f3f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r801:13dfa333f721
Date: 2016-09-23 20:05 +0200
http://bitbucket.org/pypy/pypy.org/changeset/13dfa333f721/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $65142 of $105000 (62.0%)
+   $65152 of $105000 (62.0%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5591 left
+    we have $5600 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Fri Sep 23 16:29:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 13:29:37 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: updates
Message-ID: <57e590b1.0558c20a.a5182.20cb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5710:acd514577bc1
Date: 2016-09-23 22:29 +0200
http://bitbucket.org/pypy/extradoc/changeset/acd514577bc1/

Log:	updates

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -74,3 +74,10 @@
 * ceval.c: GET_AITER: calls _PyCoro_GetAwaitableIter(), which might
   get an exception from calling the user-defined __await__() or checking
   what it returns; such an exception is completely eaten.
+
+* while I'm at it, this is an old issue that was forgotten twice on the
+  issue tracker: ``class C: __new__=int.__new__`` and ``class C(int):
+  __new__=object.__new__`` can each be instantiated, even though they
+  shouldn't.  This is because ``__new__`` is completely ignored if it is
+  set to a built-in function that uses ``tp_new_wrapper`` as its C code.
+  This is true for most or all the built-in types' ``__new__``.
diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -14,6 +14,8 @@
 * richard: extended slicing for memory view
 * richard: bytes % args, bytearray % args: PEP 461
 
+* arigo: faulthandler module
+
 
 
 Misc stuff not formally in any milestone

From pypy.commits at gmail.com  Fri Sep 23 16:43:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 13:43:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: fix
Message-ID: <57e593e7.a6ecc20a.7daa.2850@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87351:25ae97b49316
Date: 2016-09-23 22:42 +0200
http://bitbucket.org/pypy/pypy/changeset/25ae97b49316/

Log:	fix

diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -80,11 +80,10 @@
         if not space.contains(w_options, space.wrap('faulthandler')):
             return
     #
-    # Like CPython.  Why not just call enable(space)?  Maybe the goal is
-    # to let the user override the 'faulthandler' module.  Maybe someone
+    # Like CPython.  Why not just call enable(space)?  Maybe someone
     # mis-uses ``"faulthandler" in sys.modules'' as a way to check if it
     # was started by checking if it was imported at all.
-    space.appexec([], """
+    space.appexec([], """():
         import faulthandler
         faulthandler.enable()
     """)

From pypy.commits at gmail.com  Fri Sep 23 17:07:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 14:07:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: Better override _compute_value() than
 get_w_value(): the latter is
Message-ID: <57e5997a.d4301c0a.fd857.ae08@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87352:0d9dacbd997f
Date: 2016-09-23 23:05 +0200
http://bitbucket.org/pypy/pypy/changeset/0d9dacbd997f/

Log:	Better override _compute_value() than get_w_value(): the latter is
	otherwise not overridden, and the former is already highly
	polymorphic

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -374,11 +374,8 @@
         self._value = value
         self.setup(w_type)
 
-    def get_w_value(self, space):
-        w_value = self._w_value
-        if w_value is None:
-            self._w_value = w_value = space.wrap(self._value)
-        return w_value
+    def _compute_value(self, space):
+        return self._value
 
 @specialize.memo()
 def get_operr_class(valuefmt):

From pypy.commits at gmail.com  Fri Sep 23 17:08:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 14:08:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix
Message-ID: <57e599e7.4a301c0a.949cc.aed8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87353:77f0c958997b
Date: 2016-09-23 23:08 +0200
http://bitbucket.org/pypy/pypy/changeset/77f0c958997b/

Log:	fix

diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -714,7 +714,7 @@
     def filter(self, builder):
         if not builder.cpu.supports_cond_call_value and \
            self.opnum == rop.COND_CALL_VALUE_I:
-            raise CannotProduceOperation
+            raise test_random.CannotProduceOperation
 
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)

From pypy.commits at gmail.com  Sat Sep 24 02:15:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 23:15:49 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Add references
Message-ID: <57e61a15.04e21c0a.c464e.1f6c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5711:723bd28a3ffd
Date: 2016-09-24 08:15 +0200
http://bitbucket.org/pypy/extradoc/changeset/723bd28a3ffd/

Log:	Add references

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -81,3 +81,5 @@
   shouldn't.  This is because ``__new__`` is completely ignored if it is
   set to a built-in function that uses ``tp_new_wrapper`` as its C code.
   This is true for most or all the built-in types' ``__new__``.
+  http://bugs.python.org/issue1694663#msg75957
+  http://bugs.python.org/issue5322#msg84112

From pypy.commits at gmail.com  Sat Sep 24 02:45:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Sep 2016 23:45:15 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Tied this issue to a 3.5 issue
Message-ID: <57e620fb.0558c20a.a5182.9f60@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5712:2d8db6ebad85
Date: 2016-09-24 08:45 +0200
http://bitbucket.org/pypy/extradoc/changeset/2d8db6ebad85/

Log:	Tied this issue to a 3.5 issue

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -75,11 +75,15 @@
   get an exception from calling the user-defined __await__() or checking
   what it returns; such an exception is completely eaten.
 
-* while I'm at it, this is an old issue that was forgotten twice on the
+* this is an old issue that was forgotten twice on the
   issue tracker: ``class C: __new__=int.__new__`` and ``class C(int):
   __new__=object.__new__`` can each be instantiated, even though they
   shouldn't.  This is because ``__new__`` is completely ignored if it is
   set to a built-in function that uses ``tp_new_wrapper`` as its C code.
   This is true for most or all the built-in types' ``__new__``.
-  http://bugs.python.org/issue1694663#msg75957
-  http://bugs.python.org/issue5322#msg84112
+  http://bugs.python.org/issue1694663#msg75957,
+  http://bugs.python.org/issue5322#msg84112.  In (at least) CPython 3.5,
+  a few classes work only thanks to abuse of this bug: for example,
+  ``io.UnsupportedOperation.__new__(io.UnsupportedOperation)`` doesn't
+  work, but that was not noticed because ``io.UnsupportedOperation()``
+  mistakenly works.

From pypy.commits at gmail.com  Sat Sep 24 03:35:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 00:35:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Test and fix: don't give a unicode
 co_filename from interactive.py
Message-ID: <57e62cdf.670dc20a.e1181.ac01@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87354:1706df7088bd
Date: 2016-09-24 09:35 +0200
http://bitbucket.org/pypy/pypy/changeset/1706df7088bd/

Log:	Test and fix: don't give a unicode co_filename from interactive.py

diff --git a/pypy/interpreter/interactive.py b/pypy/interpreter/interactive.py
--- a/pypy/interpreter/interactive.py
+++ b/pypy/interpreter/interactive.py
@@ -169,7 +169,8 @@
 
     def runsource(self, source, ignored_filename="<input>", symbol="single"):
         # the following hacked file name is recognized specially by error.py
-        hacked_filename = '<inline>\n' + source
+        hacked_filename = '<inline>\n' + source.encode(
+                                            'ascii', 'backslashreplace')
         compiler = self.space.getexecutioncontext().compiler
 
         # CPython 2.6 turns console input into unicode
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -106,6 +106,7 @@
         self.co_varnames = varnames
         self.co_freevars = freevars
         self.co_cellvars = cellvars
+        assert isinstance(filename, str)
         rstring.check_str0(filename)
         self.co_filename = filename
         self.co_name = name

From pypy.commits at gmail.com  Sat Sep 24 03:37:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 00:37:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: fixes fixes
Message-ID: <57e62d3e.43921c0a.46835.318b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87355:95095bc757b8
Date: 2016-09-24 09:37 +0200
http://bitbucket.org/pypy/pypy/changeset/95095bc757b8/

Log:	fixes fixes

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -423,7 +423,7 @@
         self.threadlocals.enter_thread(self)
 
         # Set up faulthandler even if not imported explicitly
-        if self.config.objspace.usemodule.faulthandler:
+        if self.config.objspace.usemodules.faulthandler:
             from pypy.module.faulthandler import handler
             handler.startup(self)
 
@@ -452,7 +452,7 @@
                 w_mod.shutdown(self)
         #
         # Shut down faulthandler
-        if self.config.objspace.usemodule.faulthandler:
+        if self.config.objspace.usemodules.faulthandler:
             from pypy.module.faulthandler import handler
             handler.finish(self)
 
diff --git a/pypy/interpreter/interactive.py b/pypy/interpreter/interactive.py
--- a/pypy/interpreter/interactive.py
+++ b/pypy/interpreter/interactive.py
@@ -169,7 +169,8 @@
 
     def runsource(self, source, ignored_filename="<input>", symbol="single"):
         # the following hacked file name is recognized specially by error.py
-        hacked_filename = '<inline>\n' + source
+        hacked_filename = '<inline>\n' + source.encode(
+                                            'ascii', 'backslashreplace')
         compiler = self.space.getexecutioncontext().compiler
 
         # CPython 2.6 turns console input into unicode
diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py
--- a/pypy/module/faulthandler/__init__.py
+++ b/pypy/module/faulthandler/__init__.py
@@ -10,7 +10,7 @@
         'is_enabled': 'handler.is_enabled',
 #        'register': 'interp_faulthandler.register',
 #
-#        'dump_traceback': 'interp_faulthandler.dump_traceback',
+        'dump_traceback': 'handler.dump_traceback',
 #
 #        '_read_null': 'interp_faulthandler.read_null',
 #        '_sigsegv': 'interp_faulthandler.sigsegv',
diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -1,3 +1,4 @@
+import py
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -32,7 +33,10 @@
     'pypy_faulthandler_is_enabled', [], rffi.INT)
 
 pypy_faulthandler_write = direct_llexternal(
-    'pypy_faulthandler_write', [lltype.Ptr(rstr.STR)])
+    'pypy_faulthandler_write', [rffi.CCHARP], lltype.Void)
 
 pypy_faulthandler_write_int = direct_llexternal(
-    'pypy_faulthandler_write_int', [lltype.Signed])
+    'pypy_faulthandler_write_int', [lltype.Signed], lltype.Void)
+
+pypy_faulthandler_dump_traceback = direct_llexternal(
+    'pypy_faulthandler_dump_traceback', [rffi.INT, rffi.INT], lltype.Void)
diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py
--- a/pypy/module/faulthandler/dumper.py
+++ b/pypy/module/faulthandler/dumper.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.annlowlevel import llstr
+from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import rgc
 from rpython.rlib.rvmprof import enum_all_code_objs
 from rpython.rlib.rvmprof import cintf as rvmprof_cintf
@@ -13,10 +13,22 @@
 
 
 MAX_FRAME_DEPTH = 100
+MAX_STRING_LENGTH = 500
 
+global_buf = lltype.malloc(rffi.CCHARP.TO, MAX_STRING_LENGTH, flavor='raw',
+                           immortal=True, zero=True)
 
 def _dump(s):
-    pypy_faulthandler_write(llstr(s))
+    assert isinstance(s, str)
+    l = len(s)
+    if l >= MAX_STRING_LENGTH:
+        l = MAX_STRING_LENGTH - 1
+    i = 0
+    while i < l:
+        global_buf[i] = s[i]
+        i += 1
+    global_buf[l] = '\x00'
+    pypy_faulthandler_write(global_buf)
 
 def _dump_int(i):
     pypy_faulthandler_write_int(i)
@@ -29,9 +41,9 @@
     _dump(pycode.co_filename)
     _dump('" in ')
     _dump(pycode.co_name)
-    _dump(" (starting at line ")
+    _dump(" starting at line ")
     _dump_int(pycode.co_firstlineno)
-    _dump(")\n")
+    _dump("\n")
     return 1
 
 
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -1,9 +1,11 @@
 #include "faulthandler.h"
 #include <stdlib.h>
+#include <stdio.h>
 #include <signal.h>
 #include <assert.h>
 #include <errno.h>
 #include <string.h>
+#include <unistd.h>
 
 
 typedef struct sigaction _Py_sighandler_t;
@@ -19,7 +21,7 @@
     int initialized;
     int enabled;
     volatile int fd, all_threads;
-    volatile void (*dump_traceback)(void);
+    void (*volatile dump_traceback)(void);
     int _current_fd;
 } fatal_error;
 
@@ -43,41 +45,44 @@
     sizeof(faulthandler_handlers) / sizeof(fault_handler_t);
 
 static void
-fh_write(int fd, char *str)
+fh_write(int fd, const char *str)
 {
     (void)write(fd, str, strlen(str));
 }
 
 RPY_EXTERN
-void pypy_faulthandler_write(RPyString *s)
+void pypy_faulthandler_write(char *str)
 {
-    (void)write(fatal_error._current_fd,
-                _RPyString_AsString(s), RPyString_Size(s));
+    fh_write(fatal_error._current_fd, str);
 }
 
 RPY_EXTERN
 void pypy_faulthandler_write_int(long x)
 {
     char buf[32];
-    int count = sprintf(buf, "%ld", x);
-    (void)write(fatal_error._current_fd, buf, count);
+    sprintf(buf, "%ld", x);
+    fh_write(fatal_error._current_fd, buf);
 }
 
 
-static void
-faulthandler_dump_traceback(int fd, int all_threads)
+RPY_EXTERN
+void pypy_faulthandler_dump_traceback(int fd, int all_threads)
+{
+    fatal_error._current_fd = fd;
+
+    /* XXX 'all_threads' ignored */
+    if (fatal_error.dump_traceback)
+        fatal_error.dump_traceback();
+}
+
+void faulthandler_dump_traceback(int fd, int all_threads)
 {
     static volatile int reentrant = 0;
 
     if (reentrant)
         return;
     reentrant = 1;
-    fatal_error._current_fd = fd;
-
-    /* XXX 'all_threads' ignored */
-    if (fatal_error.dump_traceback)
-        fatal_error.dump_traceback();
-
+    pypy_faulthandler_dump_traceback(fd, all_threads);
     reentrant = 0;
 }
 
@@ -139,9 +144,9 @@
 char *pypy_faulthandler_setup(void dump_callback(void))
 {
     if (fatal_error.initialized)
-        return;
+        return NULL;
     assert(!fatal_error.enabled);
-    fatal_error.dump_callback = dump_callback;
+    fatal_error.dump_traceback = dump_callback;
 
     /* Try to allocate an alternate stack for faulthandler() signal handler to
      * be able to allocate memory on the stack, even on a stack overflow. If it
diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -1,7 +1,7 @@
 #ifndef PYPY_FAULTHANDLER_H
 #define PYPY_FAULTHANDLER_H
 
-#include "common_header.h"
+#include "src/precommondefs.h"
 
 RPY_EXTERN char *pypy_faulthandler_setup(void dump_callback(void));
 RPY_EXTERN void pypy_faulthandler_teardown(void);
@@ -10,9 +10,11 @@
 RPY_EXTERN void pypy_faulthandler_disable(void);
 RPY_EXTERN int pypy_faulthandler_is_enabled(void);
 
-RPY_EXTERN void pypy_faulthandler_write(RPyString *);
+RPY_EXTERN void pypy_faulthandler_write(char *);
 RPY_EXTERN void pypy_faulthandler_write_int(long);
 
+RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads);
+
 /*
 RPY_EXTERN int pypy_faulthandler_read_null(void);
 RPY_EXTERN void pypy_faulthandler_sigsegv(void);
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rposix import is_valid_fd
 from rpython.rlib.rarithmetic import widen
+from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 
 from pypy.interpreter.error import oefmt, exception_from_saved_errno
@@ -13,7 +14,6 @@
     def __init__(self, space):
         "NOT_RPYTHON"
         self.space = space
-        dumper.glob.space = space
         self._cleanup_()
 
     def _cleanup_(self):
@@ -64,6 +64,17 @@
     def is_enabled(self):
         return bool(widen(cintf.pypy_faulthandler_is_enabled()))
 
+    def dump_traceback(self, w_file, all_threads):
+        fileno, w_file = self.get_fileno_and_file(w_file)
+        #
+        dump_callback = llhelper(cintf.DUMP_CALLBACK, dumper._dump_callback)
+        self.check_err(cintf.pypy_faulthandler_setup(dump_callback))
+        #
+        cintf.pypy_faulthandler_dump_traceback(
+            rffi.cast(rffi.INT, fileno),
+            rffi.cast(rffi.INT, all_threads))
+        keepalive_until_here(w_file)
+
     def finish(self):
         cintf.pypy_faulthandler_teardown()
         self._cleanup_()
@@ -77,7 +88,8 @@
     # is defined, or if sys._xoptions has a 'faulthandler' key.
     if not os.environ.get('PYTHONFAULTHANDLER'):
         w_options = space.sys.get('_xoptions')
-        if not space.contains(w_options, space.wrap('faulthandler')):
+        if not space.is_true(space.contains(w_options,
+                                            space.wrap('faulthandler'))):
             return
     #
     # Like CPython.  Why not just call enable(space)?  Maybe someone
@@ -105,3 +117,9 @@
 def is_enabled(space):
     "is_enabled()->bool: check if the handler is enabled"
     return space.wrap(space.fromcache(Handler).is_enabled())
+
+ at unwrap_spec(all_threads=int)
+def dump_traceback(space, w_file=None, all_threads=0):
+   """dump the traceback of the current thread into file
+   including all threads if all_threads is True"""
+   space.fromcache(Handler).dump_traceback(w_file, all_threads)

From pypy.commits at gmail.com  Sat Sep 24 03:46:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 00:46:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: translation fix
Message-ID: <57e62f38.06d41c0a.221c5.359b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87356:c9d2807306dc
Date: 2016-09-24 09:45 +0200
http://bitbucket.org/pypy/pypy/changeset/c9d2807306dc/

Log:	translation fix

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -252,6 +252,7 @@
         missing_positional = []
         missing_kwonly = []
         more_filling = (input_argcount < co_argcount + co_kwonlyargcount)
+        def_first = 0
         if more_filling:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
             j = 0

From pypy.commits at gmail.com  Sat Sep 24 03:50:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 00:50:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: translation fixes
Message-ID: <57e63060.0205c20a.db644.b4d3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87357:46dc7721ec00
Date: 2016-09-24 09:50 +0200
http://bitbucket.org/pypy/pypy/changeset/46dc7721ec00/

Log:	translation fixes

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -252,6 +252,7 @@
         missing_positional = []
         missing_kwonly = []
         more_filling = (input_argcount < co_argcount + co_kwonlyargcount)
+        def_first = 0
         if more_filling:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
             j = 0
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -5,7 +5,8 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 
-from pypy.interpreter.error import oefmt, exception_from_saved_errno
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import exception_from_saved_errno
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.faulthandler import cintf, dumper
 

From pypy.commits at gmail.com  Sat Sep 24 03:52:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 00:52:21 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <57e630b5.c186c20a.c8b50.bbcb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r802:f484e3acdd66
Date: 2016-09-24 09:52 +0200
http://bitbucket.org/pypy/pypy.org/changeset/f484e3acdd66/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $5600 left
+    we have $5060 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sat Sep 24 04:23:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 01:23:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: re-add _read_null()
Message-ID: <57e637f0.ca0e1c0a.3f6cf.4396@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87358:4dee502cd268
Date: 2016-09-24 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/4dee502cd268/

Log:	re-add _read_null()

diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py
--- a/pypy/module/faulthandler/__init__.py
+++ b/pypy/module/faulthandler/__init__.py
@@ -12,7 +12,7 @@
 #
         'dump_traceback': 'handler.dump_traceback',
 #
-#        '_read_null': 'interp_faulthandler.read_null',
+        '_read_null': 'handler.read_null',
 #        '_sigsegv': 'interp_faulthandler.sigsegv',
 #        '_sigfpe': 'interp_faulthandler.sigfpe',
 #        '_sigabrt': 'interp_faulthandler.sigabrt',
diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -40,3 +40,12 @@
 
 pypy_faulthandler_dump_traceback = direct_llexternal(
     'pypy_faulthandler_dump_traceback', [rffi.INT, rffi.INT], lltype.Void)
+
+# for tests...
+
+pypy_faulthandler_read_null = direct_llexternal(
+    'pypy_faulthandler_read_null', [], lltype.Void)
+
+pypy_faulthandler_read_null_releasegil = direct_llexternal(
+    'pypy_faulthandler_read_null', [], lltype.Void,
+    _nowrapper=False, releasegil=True)
diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py
--- a/pypy/module/faulthandler/dumper.py
+++ b/pypy/module/faulthandler/dumper.py
@@ -41,7 +41,7 @@
     _dump(pycode.co_filename)
     _dump('" in ')
     _dump(pycode.co_name)
-    _dump(" starting at line ")
+    _dump(", from line ")
     _dump_int(pycode.co_firstlineno)
     _dump("\n")
     return 1
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -113,6 +113,7 @@
         if (handler->signum == signum)
             break;
     }
+    /* If not found, we use the SIGSEGV handler (the last one in the list) */
 
     /* restore the previous handler */
     if (handler->enabled) {
@@ -173,8 +174,12 @@
     if (fatal_error.initialized) {
         pypy_faulthandler_disable();
         fatal_error.initialized = 0;
-        free(stack.ss_sp);
-        stack.ss_sp = NULL;
+        if (stack.ss_sp) {
+            stack.ss_flags = SS_DISABLE;
+            sigaltstack(&stack, NULL);
+            free(stack.ss_sp);
+            stack.ss_sp = NULL;
+        }
     }
 }
 
@@ -238,18 +243,18 @@
 }
 
 
-#if 0
-int
-pypy_faulthandler_read_null(void)
+/* for tests... */
+
+RPY_EXTERN
+int pypy_faulthandler_read_null(void)
 {
-    volatile int *x;
-    volatile int y;
+    int *volatile x;
 
     x = NULL;
-    y = *x;
-    return y;
+    return *x;
 }
 
+#if 0
 void
 pypy_faulthandler_sigsegv(void)
 {
diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -15,8 +15,9 @@
 
 RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads);
 
+
+RPY_EXTERN int pypy_faulthandler_read_null(void);
 /*
-RPY_EXTERN int pypy_faulthandler_read_null(void);
 RPY_EXTERN void pypy_faulthandler_sigsegv(void);
 RPY_EXTERN int pypy_faulthandler_sigfpe(void);
 RPY_EXTERN void pypy_faulthandler_sigabrt();
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -124,3 +124,12 @@
    """dump the traceback of the current thread into file
    including all threads if all_threads is True"""
    space.fromcache(Handler).dump_traceback(w_file, all_threads)
+
+# for tests...
+
+ at unwrap_spec(release_gil=bool)
+def read_null(space, release_gil):
+    if release_gil:
+        cintf.pypy_faulthandler_read_null_releasegil()
+    else:
+        cintf.pypy_faulthandler_read_null()

From pypy.commits at gmail.com  Sat Sep 24 04:56:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 01:56:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: Add some more of the test
 functions
Message-ID: <57e63fd9.c50e1c0a.71dc9.044b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87359:f16ac9a03e25
Date: 2016-09-24 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/f16ac9a03e25/

Log:	Add some more of the test functions

diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py
--- a/pypy/module/faulthandler/__init__.py
+++ b/pypy/module/faulthandler/__init__.py
@@ -13,10 +13,7 @@
         'dump_traceback': 'handler.dump_traceback',
 #
         '_read_null': 'handler.read_null',
-#        '_sigsegv': 'interp_faulthandler.sigsegv',
-#        '_sigfpe': 'interp_faulthandler.sigfpe',
-#        '_sigabrt': 'interp_faulthandler.sigabrt',
-#        #'_sigbus': 'interp_faulthandler.sigbus',
-#        #'_sigill': 'interp_faulthandler.sigill',
-#        '_fatal_error': 'interp_faulthandler.fatal_error',
+        '_sigsegv': 'handler.sigsegv',
+        '_sigfpe': 'handler.sigfpe',
+        '_sigabrt': 'handler.sigabrt',
     }
diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -49,3 +49,16 @@
 pypy_faulthandler_read_null_releasegil = direct_llexternal(
     'pypy_faulthandler_read_null', [], lltype.Void,
     _nowrapper=False, releasegil=True)
+
+pypy_faulthandler_sigsegv = direct_llexternal(
+    'pypy_faulthandler_sigsegv', [], lltype.Void)
+
+pypy_faulthandler_sigsegv_releasegil = direct_llexternal(
+    'pypy_faulthandler_sigsegv', [], lltype.Void,
+    _nowrapper=False, releasegil=True)
+
+pypy_faulthandler_sigfpe = direct_llexternal(
+    'pypy_faulthandler_sigfpe', [], lltype.Void)
+
+pypy_faulthandler_sigabrt = direct_llexternal(
+    'pypy_faulthandler_sigabrt', [], lltype.Void)
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -6,6 +6,7 @@
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>
+#include <sys/resource.h>
 
 
 typedef struct sigaction _Py_sighandler_t;
@@ -245,19 +246,48 @@
 
 /* for tests... */
 
+static void
+faulthandler_suppress_crash_report(void)
+{
+#ifdef MS_WINDOWS
+    UINT mode;
+
+    /* Configure Windows to not display the Windows Error Reporting dialog */
+    mode = SetErrorMode(SEM_NOGPFAULTERRORBOX);
+    SetErrorMode(mode | SEM_NOGPFAULTERRORBOX);
+#endif
+
+#ifndef MS_WINDOWS
+    struct rlimit rl;
+
+    /* Disable creation of core dump */
+    if (getrlimit(RLIMIT_CORE, &rl) != 0) {
+        rl.rlim_cur = 0;
+        setrlimit(RLIMIT_CORE, &rl);
+    }
+#endif
+
+#ifdef _MSC_VER
+    /* Visual Studio: configure abort() to not display an error message nor
+       open a popup asking to report the fault. */
+    _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
+#endif
+}
+
 RPY_EXTERN
 int pypy_faulthandler_read_null(void)
 {
     int *volatile x;
 
+    faulthandler_suppress_crash_report();
     x = NULL;
     return *x;
 }
 
-#if 0
-void
-pypy_faulthandler_sigsegv(void)
+RPY_EXTERN
+void pypy_faulthandler_sigsegv(void)
 {
+    faulthandler_suppress_crash_report();
 #if defined(MS_WINDOWS)
     /* For SIGSEGV, faulthandler_fatal_error() restores the previous signal
        handler and then gives back the execution flow to the program (without
@@ -277,12 +307,13 @@
 #endif
 }
 
-int
-pypy_faulthandler_sigfpe(void)
+RPY_EXTERN
+int pypy_faulthandler_sigfpe(void)
 {
     /* Do an integer division by zero: raise a SIGFPE on Intel CPU, but not on
        PowerPC. Use volatile to disable compile-time optimizations. */
     volatile int x = 1, y = 0, z;
+    faulthandler_suppress_crash_report();
     z = x / y;
     /* If the division by zero didn't raise a SIGFPE (e.g. on PowerPC),
        raise it manually. */
@@ -292,30 +323,9 @@
     return z;
 }
 
-void
-pypy_faulthandler_sigabrt()
+RPY_EXTERN
+void pypy_faulthandler_sigabrt(void)
 {
-#ifdef _MSC_VER
-    /* Visual Studio: configure abort() to not display an error message nor
-       open a popup asking to report the fault. */
-    _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT);
-#endif
+    faulthandler_suppress_crash_report();
     abort();
 }
-
-#ifdef SIGBUS
-void
-pypy_faulthandler_sigbus(void)
-{
-    raise(SIGBUS);
-}
-#endif
-
-#ifdef SIGILL
-void
-pypy_faulthandler_sigill(void)
-{
-    raise(SIGILL);
-}
-#endif
-#endif
diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -3,6 +3,7 @@
 
 #include "src/precommondefs.h"
 
+
 RPY_EXTERN char *pypy_faulthandler_setup(void dump_callback(void));
 RPY_EXTERN void pypy_faulthandler_teardown(void);
 
@@ -17,17 +18,9 @@
 
 
 RPY_EXTERN int pypy_faulthandler_read_null(void);
-/*
 RPY_EXTERN void pypy_faulthandler_sigsegv(void);
 RPY_EXTERN int pypy_faulthandler_sigfpe(void);
-RPY_EXTERN void pypy_faulthandler_sigabrt();
-#ifdef SIGBUS
-RPY_EXTERN void pypy_faulthandler_sigbus(void);
-#endif
+RPY_EXTERN void pypy_faulthandler_sigabrt(void);
 
-#ifdef SIGILL
-RPY_EXTERN void pypy_faulthandler_sigill(void);
-#endif
-*/
 
 #endif  /* PYPY_FAULTHANDLER_H */
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -47,12 +47,13 @@
             pass   # ignore flush() error
         return fd, w_file
 
+    def setup(self):
+        dump_callback = llhelper(cintf.DUMP_CALLBACK, dumper._dump_callback)
+        self.check_err(cintf.pypy_faulthandler_setup(dump_callback))
+
     def enable(self, w_file, all_threads):
         fileno, w_file = self.get_fileno_and_file(w_file)
-        #
-        dump_callback = llhelper(cintf.DUMP_CALLBACK, dumper._dump_callback)
-        self.check_err(cintf.pypy_faulthandler_setup(dump_callback))
-        #
+        self.setup()
         self.fatal_error_w_file = w_file
         self.check_err(cintf.pypy_faulthandler_enable(
             rffi.cast(rffi.INT, fileno),
@@ -67,10 +68,7 @@
 
     def dump_traceback(self, w_file, all_threads):
         fileno, w_file = self.get_fileno_and_file(w_file)
-        #
-        dump_callback = llhelper(cintf.DUMP_CALLBACK, dumper._dump_callback)
-        self.check_err(cintf.pypy_faulthandler_setup(dump_callback))
-        #
+        self.setup()
         cintf.pypy_faulthandler_dump_traceback(
             rffi.cast(rffi.INT, fileno),
             rffi.cast(rffi.INT, all_threads))
@@ -127,9 +125,22 @@
 
 # for tests...
 
- at unwrap_spec(release_gil=bool)
-def read_null(space, release_gil):
+ at unwrap_spec(release_gil=int)
+def read_null(space, release_gil=0):
     if release_gil:
         cintf.pypy_faulthandler_read_null_releasegil()
     else:
         cintf.pypy_faulthandler_read_null()
+
+ at unwrap_spec(release_gil=int)
+def sigsegv(space, release_gil=0):
+    if release_gil:
+        cintf.pypy_faulthandler_sigsegv_releasegil()
+    else:
+        cintf.pypy_faulthandler_sigsegv()
+
+def sigfpe(space):
+    cintf.pypy_faulthandler_sigfpe()
+
+def sigabrt(space):
+    cintf.pypy_faulthandler_sigabrt()
diff --git a/pypy/module/faulthandler/interp_faulthandler.py b/pypy/module/faulthandler/interp_faulthandler.py
deleted file mode 100644
--- a/pypy/module/faulthandler/interp_faulthandler.py
+++ /dev/null
@@ -1,127 +0,0 @@
-import os
-import py
-
-from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.translator import cdir
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.interpreter.error import OperationError, oefmt
-
-MAX_NTHREADS = 100
-
-cwd = py.path.local(__file__).dirpath()
-eci = ExternalCompilationInfo(
-    includes=[cwd.join('faulthandler.h')],
-    include_dirs=[str(cwd), cdir],
-    separate_module_files=[cwd.join('faulthandler.c')])
-
-def llexternal(*args, **kwargs):
-    kwargs.setdefault('releasegil', False)
-    kwargs.setdefault('compilation_info', eci)
-    return rffi.llexternal(*args, **kwargs)
-
-pypy_faulthandler_read_null = llexternal(
-    'pypy_faulthandler_read_null', [], lltype.Void)
-pypy_faulthandler_read_null_nogil = llexternal(
-    'pypy_faulthandler_read_null', [], lltype.Void,
-    releasegil=True)
-pypy_faulthandler_sigsegv = llexternal(
-    'pypy_faulthandler_sigsegv', [], lltype.Void)
-pypy_faulthandler_sigfpe = llexternal(
-    'pypy_faulthandler_sigfpe', [], lltype.Void)
-pypy_faulthandler_sigabrt = llexternal(
-    'pypy_faulthandler_sigabrt', [], lltype.Void)
-pypy_faulthandler_sigbus = llexternal(
-    'pypy_faulthandler_sigbus', [], lltype.Void)
-pypy_faulthandler_sigill = llexternal(
-    'pypy_faulthandler_sigill', [], lltype.Void)
-
-class FatalErrorState(object):
-    def __init__(self, space):
-        self.enabled = False
-        self.all_threads = True
-
- at unwrap_spec(w_file=WrappedDefault(None),
-             w_all_threads=WrappedDefault(True))
-def enable(space, w_file, w_all_threads):
-    state = space.fromcache(FatalErrorState)
-    state.enabled = True
-    state.all_threads = bool(space.int_w(w_all_threads))
-
-def disable(space):
-    state = space.fromcache(FatalErrorState)
-    state.enabled = False
-
-def is_enabled(space):
-    return space.wrap(space.fromcache(FatalErrorState).enabled)
-
-def register(space, __args__):
-    pass
-
-
- at unwrap_spec(w_file=WrappedDefault(None),
-             w_all_threads=WrappedDefault(True))
-def dump_traceback(space, w_file, w_all_threads):
-    current_ec = space.getexecutioncontext()
-    if space.int_w(w_all_threads):
-        ecs = space.threadlocals.getallvalues()
-    else:
-        ecs = {0: current_ec}
-
-    if space.is_none(w_file):
-        w_file = space.sys.get('stderr')
-    fd = space.c_filedescriptor_w(w_file)
-
-    nthreads = 0
-    for thread_ident, ec in ecs.items():
-        if nthreads:
-            os.write(fd, "\n")
-        if nthreads >= MAX_NTHREADS:
-            os.write(fd, "...\n")
-            break
-        if ec is current_ec:
-            os.write(fd, "Current thread 0x%x:\n" % thread_ident)
-        else:
-            os.write(fd, "Thread 0x%x:\n" % thread_ident)
-
-        frame = ec.gettopframe()
-        while frame:
-            code = frame.pycode
-            lineno = frame.get_last_lineno()
-            if code:
-                os.write(fd, "  File \"%s\", line %s in %s\n" % (
-                        code.co_filename, lineno, code.co_name))
-            else:
-                os.write(fd, "  File ???, line %s in ???\n" % (
-                        lineno,))
-
-            frame = frame.f_backref()
- 
-
- at unwrap_spec(w_release_gil=WrappedDefault(False))
-def read_null(space, w_release_gil):
-    if space.is_true(w_release_gil):
-        pypy_faulthandler_read_null_nogil()
-    else:
-        pypy_faulthandler_read_null()
-
-def sigsegv():
-    pypy_faulthandler_sigsegv()
-
-def sigfpe():
-    pypy_faulthandler_sigfpe()
-
-def sigabrt():
-    pypy_faulthandler_sigabrt()
-
-#def sigbus():
-#    pypy_faulthandler_sigbus()
-
-#def sigill():
-#    pypy_faulthandler_sigill()
-
- at unwrap_spec(msg=str)
-def fatal_error(space, msg):
-    os.write(2, "Fatal Python error: %s\n" % msg);
-    dump_traceback(space, space.wrap(None), space.wrap(True))
-    raise RuntimeError(msg)

From pypy.commits at gmail.com  Sat Sep 24 09:15:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 06:15:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: tweaks, in-progress
Message-ID: <57e67c71.8f081c0a.1e2e7.5d17@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87360:48111b3e0633
Date: 2016-09-24 15:06 +0200
http://bitbucket.org/pypy/pypy/changeset/48111b3e0633/

Log:	tweaks, in-progress

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -518,6 +518,11 @@
     sys._xoptions = dict(x.split('=', 1) if '=' in x else (x, True)
                          for x in options['_xoptions'])
 
+    if 'faulthandler' in sys.builtin_module_names:
+        if 'faulthandler' in sys._xoptions or os.getenv('PYTHONFAULTHANDLER'):
+            import faulthandler
+            faulthandler.enable(2)   # manually set to stderr
+
 ##    if not we_are_translated():
 ##        for key in sorted(options):
 ##            print '%40s: %s' % (key, options[key])
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -422,11 +422,6 @@
         # To be called before using the space
         self.threadlocals.enter_thread(self)
 
-        # Set up faulthandler even if not imported explicitly
-        if self.config.objspace.usemodules.faulthandler:
-            from pypy.module.faulthandler import handler
-            handler.startup(self)
-
         # Initialize already imported builtin modules
         from pypy.interpreter.module import Module
         w_modules = self.sys.get('modules')
@@ -450,11 +445,6 @@
         for w_mod in self.builtin_modules.values():
             if isinstance(w_mod, Module) and w_mod.startup_called:
                 w_mod.shutdown(self)
-        #
-        # Shut down faulthandler
-        if self.config.objspace.usemodules.faulthandler:
-            from pypy.module.faulthandler import handler
-            handler.finish(self)
 
     def wait_for_thread_shutdown(self):
         """Wait until threading._shutdown() completes, provided the threading
diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py
--- a/pypy/module/faulthandler/__init__.py
+++ b/pypy/module/faulthandler/__init__.py
@@ -16,4 +16,10 @@
         '_sigsegv': 'handler.sigsegv',
         '_sigfpe': 'handler.sigfpe',
         '_sigabrt': 'handler.sigabrt',
+        '_stack_overflow': 'handler.stack_overflow',
     }
+
+    def shutdown(self, space):
+        from pypy.module.faulthandler import handler
+        handler.finish(space)
+        MixedModule.shutdown(self, space)
diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -62,3 +62,6 @@
 
 pypy_faulthandler_sigabrt = direct_llexternal(
     'pypy_faulthandler_sigabrt', [], lltype.Void)
+
+pypy_faulthandler_stackoverflow = direct_llexternal(
+    'pypy_faulthandler_stackoverflow', [lltype.Float], lltype.Float)
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -7,6 +7,7 @@
 #include <string.h>
 #include <unistd.h>
 #include <sys/resource.h>
+#include <math.h>
 
 
 typedef struct sigaction _Py_sighandler_t;
@@ -329,3 +330,19 @@
     faulthandler_suppress_crash_report();
     abort();
 }
+
+static double fh_stack_overflow(double levels)
+{
+    if (levels > 2.5) {
+        return (sqrt(fh_stack_overflow(levels - 1.0))
+                + fh_stack_overflow(levels * 1e-10));
+    }
+    return 1e100 + levels;
+}
+
+RPY_EXTERN
+double pypy_faulthandler_stackoverflow(double levels)
+{
+    faulthandler_suppress_crash_report();
+    return fh_stack_overflow(levels);
+}
diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -21,6 +21,7 @@
 RPY_EXTERN void pypy_faulthandler_sigsegv(void);
 RPY_EXTERN int pypy_faulthandler_sigfpe(void);
 RPY_EXTERN void pypy_faulthandler_sigabrt(void);
+RPY_EXTERN double pypy_faulthandler_stackoverflow(double);
 
 
 #endif  /* PYPY_FAULTHANDLER_H */
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -79,28 +79,8 @@
         self._cleanup_()
 
 
-def startup(space):
-    """Initialize the faulthandler logic when the space is starting
-    (this is called from baseobjspace.py)"""
-    #
-    # Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable
-    # is defined, or if sys._xoptions has a 'faulthandler' key.
-    if not os.environ.get('PYTHONFAULTHANDLER'):
-        w_options = space.sys.get('_xoptions')
-        if not space.is_true(space.contains(w_options,
-                                            space.wrap('faulthandler'))):
-            return
-    #
-    # Like CPython.  Why not just call enable(space)?  Maybe someone
-    # mis-uses ``"faulthandler" in sys.modules'' as a way to check if it
-    # was started by checking if it was imported at all.
-    space.appexec([], """():
-        import faulthandler
-        faulthandler.enable()
-    """)
-
 def finish(space):
-    """Finalize the faulthandler logic (called from baseobjspace.py)"""
+    "Finalize the faulthandler logic (called from shutdown())"
     space.fromcache(Handler).finish()
 
 
@@ -144,3 +124,8 @@
 
 def sigabrt(space):
     cintf.pypy_faulthandler_sigabrt()
+
+ at unwrap_spec(levels=int)
+def stack_overflow(space, levels=100000000):
+    levels = float(levels)
+    return space.wrap(cintf.pypy_faulthandler_stackoverflow(levels))
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -159,4 +159,4 @@
 # faulthandler support
 
 def get_rvmprof_stack():
-    return vmprof_tl_stack.getraw()
+    return vmprof_tl_stack.get_or_make_raw()

From pypy.commits at gmail.com  Sat Sep 24 11:59:27 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 24 Sep 2016 08:59:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: cpyext is required to actually test
 _imp.load_dynamic().
Message-ID: <57e6a2df.ca11c30a.d4df7.5acc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87361:9625a7c1b613
Date: 2016-09-24 16:58 +0100
http://bitbucket.org/pypy/pypy/changeset/9625a7c1b613/

Log:	cpyext is required to actually test _imp.load_dynamic().

	Disable check for cpyext when running with -A on CPython.

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -3,8 +3,10 @@
 
 
 class AppTestImpModule:
+    # cpyext is required for _imp.load_dynamic()
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
+        'usemodules': [
+            'binascii', 'imp', 'itertools', 'time', 'struct', 'cpyext'],
     }
 
     def setup_class(cls):
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -141,6 +141,9 @@
             # They may be extension modules on CPython
             name = None
             for name in missing.copy():
+                if name == 'cpyext':
+                    missing.remove(name)
+                    continue
                 try:
                     __import__(name)
                 except ImportError:

From pypy.commits at gmail.com  Sat Sep 24 13:45:28 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 24 Sep 2016 10:45:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Set up ImportError attributes properly in
 _imp.load_dynamic()
Message-ID: <57e6bbb8.c70a1c0a.71017.b3e6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87362:bdd0b2244dd3
Date: 2016-09-24 18:44 +0100
http://bitbucket.org/pypy/pypy/changeset/bdd0b2244dd3/

Log:	Set up ImportError attributes properly in _imp.load_dynamic()

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -582,3 +582,16 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def new_import_error(space, w_msg, w_name, w_path):
+    """Create a new instance of ImportError.
+
+    The result corresponds to ImportError(msg, name=name, path=path)
+    """
+    return space.appexec(
+        [w_msg, w_name, w_path], """(msg, name, path):
+            return ImportError(msg, name=name, path=path)""")
+
+def raise_import_error(space, w_msg, w_name, w_path):
+    w_exc = new_import_error(space, w_msg, w_name, w_path)
+    raise OperationError(space.w_ImportError, w_exc)
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -3,7 +3,7 @@
 import py, os, errno
 from pypy.interpreter.error import (
     OperationError, decompose_valuefmt, get_operrcls2, new_exception_class,
-    oefmt, wrap_oserror)
+    oefmt, wrap_oserror, new_import_error)
 
 
 def test_decompose_valuefmt():
@@ -154,3 +154,8 @@
     assert operr.match(space, space.w_ValueError)
     assert operr.match(space, space.w_TypeError)
 
+def test_import_error(space):
+    w_exc = new_import_error(
+        space, space.wrap(u'msg'), space.wrap(u'name'), space.wrap(u'path'))
+    assert space.getattr(w_exc, space.wrap(u'name')).unwrap(space) == u'name'
+    assert space.getattr(w_exc, space.wrap(u'path')).unwrap(space) == u'path'
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -20,7 +20,7 @@
 from rpython.tool.udir import udir
 from rpython.translator import platform
 from pypy.module.cpyext.state import State
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError, oefmt, raise_import_error
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.nestedscope import Cell
@@ -982,7 +982,7 @@
     py_fatalerror = rffi.llexternal('%s_FatalError' % prefix,
                                     [CONST_STRING], lltype.Void,
                                     compilation_info=eci)
-    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [],
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
@@ -1523,9 +1523,10 @@
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
-        raise oefmt(space.w_ImportError,
-                    "unable to load extension module '%s': %s",
-                    path, e.msg)
+        w_name = space.newunicode(name.decode('ascii'))
+        w_path = space.wrap_fsdecoded(path)
+        raise raise_import_error(space,
+            space.wrap_fsdecoded(e.msg), w_name, w_path)
     look_for = None
     #
     if space.config.objspace.usemodules._cffi_backend:
@@ -1555,9 +1556,12 @@
             look_for += ' or ' + also_look_for
         else:
             look_for = also_look_for
-    #
-    raise oefmt(space.w_ImportError,
-                "function %s not found in library %s", look_for, path)
+    w_msg = u"function %s not found in library %s" % (
+        unicode(look_for), space.unicode_w(space.wrap_fsdecoded(path)))
+    w_name = space.newunicode(name.decode('ascii'))
+    w_path = space.wrap_fsdecoded(path)
+    raise_import_error(space, w_msg, w_name, w_path)
+
 
 initfunctype = lltype.Ptr(lltype.FuncType([], PyObject))
 
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -50,15 +50,17 @@
         fd = space.int_w(space.call_method(w_iobase, 'fileno'))
         return streamio.fdopen_as_stream(fd, filemode)
 
- at unwrap_spec(filename='fsencode')
-def load_dynamic(space, w_modulename, filename, w_file=None):
+ at unwrap_spec(modulename=unicode, filename='fsencode')
+def load_dynamic(space, modulename, filename, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
 
     from pypy.module.cpyext.api import load_extension_module
-    load_extension_module(space, filename, space.str_w(w_modulename))
+    # extension names must be valid C identifiers
+    modulename = modulename.encode('ascii')
+    load_extension_module(space, filename, modulename.encode('ascii'))
 
-    return importing.check_sys_modules(space, w_modulename)
+    return importing.check_sys_modules_w(space, modulename)
 
 def init_builtin(space, w_name):
     name = space.str0_w(w_name)
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -60,11 +60,13 @@
         import _imp
         excinfo = raises(ImportError, _imp.load_dynamic, 'foo', 'bar')
         assert excinfo.value.name == 'foo'
-        assert 'bar' in excinfo.value.path
+        assert excinfo.value.path == './bar'
         # Note: On CPython, the behavior changes slightly if a 3rd argument is
         # passed in, whose value is ignored. We don't implement that.
         #raises(IOError, _imp.load_dynamic, 'foo', 'bar', 42)
 
+        raises(TypeError, _imp.load_dynamic, b'foo', 'bar')
+
     def test_suffixes(self):
         import imp
         for suffix, mode, type in imp.get_suffixes():

From pypy.commits at gmail.com  Sat Sep 24 15:06:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 12:06:02 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Add a task
Message-ID: <57e6ce9a.ca0e1c0a.5322d.d40b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5713:7e5686d16033
Date: 2016-09-24 21:05 +0200
http://bitbucket.org/pypy/extradoc/changeset/7e5686d16033/

Log:	Add a task

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -41,6 +41,11 @@
 
 * bytearray: 'del x[:10]' is now amortized constant-time
 
+* check that 'import array', say, finds and loads a file array.py,
+  whereas 'import gc' does not ('gc' is a built-in module in CPython but
+  'array' is typically an extension module; at least that's the case on
+  Linux with default compilation settings).
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Sat Sep 24 14:58:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 11:58:49 -0700 (PDT)
Subject: [pypy-commit] pypy search-path-from-libpypy: Search for the
 lib-python/lib_pypy directores starting from the
Message-ID: <57e6cce9.e2cdc20a.fe841.8f79@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: search-path-from-libpypy
Changeset: r87363:cf88e0443153
Date: 2016-09-24 20:54 +0200
http://bitbucket.org/pypy/pypy/changeset/cf88e0443153/

Log:	Search for the lib-python/lib_pypy directores starting from the
	libpypy-c.so location, instead of the pypy executable location. This
	is more consistent with what occurs in case of embedding.

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -89,17 +89,20 @@
     def pypy_setup_home(ll_home, verbose):
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
-        if ll_home:
+        if ll_home and ord(ll_home[0]):
             home1 = rffi.charp2str(ll_home)
             home = os.path.join(home1, 'x') # <- so that 'll_home' can be
                                             # directly the root directory
+            dynamic = False
         else:
-            home = home1 = pypydir
-        w_path = pypy_find_stdlib(space, home)
+            home1 = "pypy's shared library location"
+            home = pypydir
+            dynamic = True
+        w_path = pypy_find_stdlib(space, home, dynamic)
         if space.is_none(w_path):
             if verbose:
                 debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
-                      " not found in '%s' or in any parent directory" % home1)
+                      " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.appexec([w_path], """(path):
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -112,29 +112,7 @@
 #define _WIN32_WINNT 0x0501
 #include <windows.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  _MAX_PATH
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    HMODULE hModule = 0;
-    DWORD res;
-
-    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | 
-                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-                       (LPCTSTR)&_cffi_init, &hModule);
-
-    if (hModule == 0 ) {
-        _cffi_init_error("GetModuleHandleEx() failed", "");
-        return -1;
-    }
-    res = GetModuleFileName(hModule, output_home_path, CFFI_INIT_HOME_PATH_MAX);
-    if (res >= CFFI_INIT_HOME_PATH_MAX) {
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -155,28 +133,9 @@
 else:
 
     do_includes = r"""
-#include <dlfcn.h>
 #include <pthread.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  PATH_MAX
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    Dl_info info;
-    dlerror();   /* reset */
-    if (dladdr(&_cffi_init, &info) == 0) {
-        _cffi_init_error("dladdr() failed: ", dlerror());
-        return -1;
-    }
-    if (realpath(info.dli_fname, output_home_path) == NULL) {
-        perror("realpath() failed");
-        _cffi_init_error("realpath() failed", "");
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -201,14 +160,10 @@
 
 static void _cffi_init(void)
 {
-    char home[CFFI_INIT_HOME_PATH_MAX + 1];
-
     rpython_startup_code();
     RPyGilAllocate();
 
-    if (_cffi_init_home(home) != 0)
-        return;
-    if (pypy_setup_home(home, 1) != 0) {
+    if (pypy_setup_home(NULL, 1) != 0) {
         _cffi_init_error("pypy_setup_home() failed", "");
         return;
     }
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -9,6 +9,8 @@
 
 from rpython.rlib import rpath
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.sys.state import get as get_state
@@ -155,8 +157,13 @@
     return space.wrap(resolvedirof(filename))
 
 
- at unwrap_spec(executable='str0')
-def pypy_find_stdlib(space, executable):
+ at unwrap_spec(executable='str0', dynamic=int)
+def pypy_find_stdlib(space, executable, dynamic=1):
+    if dynamic and space.config.translation.shared:
+        dynamic_location = pypy_init_home()
+        if dynamic_location:
+            executable = rffi.charp2str(dynamic_location)
+            pypy_init_free(dynamic_location)
     path, prefix = find_stdlib(get_state(space), executable)
     if path is None:
         return space.w_None
@@ -164,3 +171,75 @@
     space.setitem(space.sys.w_dict, space.wrap('prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), w_prefix)
     return space.newlist([space.wrap(p) for p in path])
+
+
+# ____________________________________________________________
+
+
+if os.name == 'nt':
+
+    _source_code = r"""
+#define _WIN32_WINNT 0x0501
+#include <windows.h>
+#include <stdio.h>
+
+RPY_EXPORTED
+char *_pypy_init_home(void)
+{
+    HMODULE hModule = 0;
+    DWORD res;
+    char *p;
+
+    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                       (LPCTSTR)&_pypy_init_home, &hModule);
+
+    if (hModule == 0 ) {
+        fprintf(stderr, "PyPy initialization: GetModuleHandleEx() failed\n");
+        return NULL;
+    }
+    p = malloc(_MAX_PATH);
+    if (p == NULL)
+        return NULL;
+    res = GetModuleFileName(hModule, p, _MAX_PATH);
+    if (res >= _MAX_PATH || res <= 0) {
+        free(p);
+        fprintf(stderr, "PyPy initialization: GetModuleFileName() failed\n");
+        return NULL;
+    }
+    return p;
+}
+"""
+
+else:
+
+    _source_code = r"""
+#include <dlfcn.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+RPY_EXPORTED
+char *_pypy_init_home(void)
+{
+    Dl_info info;
+    dlerror();   /* reset */
+    if (dladdr(&_pypy_init_home, &info) == 0) {
+        fprintf(stderr, "PyPy initialization: dladdr() failed: %s\n",
+                dlerror());
+        return NULL;
+    }
+    char *p = realpath(info.dli_fname, NULL);
+    if (p == NULL) {
+        p = strdup(info.dli_fname);
+    }
+    return p;
+}
+"""
+
+_eci = ExternalCompilationInfo(separate_module_sources=[_source_code])
+
+pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP,
+                                 _nowrapper=True, compilation_info=_eci)
+pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void,
+                                 _nowrapper=True, compilation_info=_eci)
diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py
--- a/pypy/module/sys/test/test_initpath.py
+++ b/pypy/module/sys/test/test_initpath.py
@@ -1,8 +1,10 @@
 import py
 import os.path
-from pypy.module.sys.initpath import (compute_stdlib_path, find_executable, find_stdlib,
-                                      resolvedirof)
+from pypy.module.sys.initpath import (compute_stdlib_path, find_executable,
+                                      find_stdlib, resolvedirof,
+                                      pypy_init_home, pypy_init_free)
 from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
+from rpython.rtyper.lltypesystem import rffi
 
 def build_hierarchy(prefix):
     dirname = '%d.%d' % CPYTHON_VERSION[:2]
@@ -10,7 +12,7 @@
     b = prefix.join('lib-python', dirname).ensure(dir=1)
     return a, b
 
-def test_find_stdlib(tmpdir, monkeypatch):
+def test_find_stdlib(tmpdir):
     bin_dir = tmpdir.join('bin').ensure(dir=True)
     pypy = bin_dir.join('pypy').ensure(file=True)
     build_hierarchy(tmpdir)
@@ -33,6 +35,14 @@
     path, prefix = find_stdlib(None, str(pypy_sym))
     assert prefix == pypydir
 
+def test_pypy_init_home():
+    p = pypy_init_home()
+    assert p
+    s = rffi.charp2str(p)
+    pypy_init_free(p)
+    print s
+    assert os.path.exists(s)
+
 def test_compute_stdlib_path(tmpdir):
     dirs = build_hierarchy(tmpdir)
     path = compute_stdlib_path(None, str(tmpdir))

From pypy.commits at gmail.com  Sat Sep 24 16:34:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 13:34:25 -0700 (PDT)
Subject: [pypy-commit] pypy search-path-from-libpypy: Fix/skip parts of this
 test (could in theory play tricks, but that's
Message-ID: <57e6e351.c50e1c0a.71dc9.ef69@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: search-path-from-libpypy
Changeset: r87364:8e4b90fb2e98
Date: 2016-09-24 22:12 +0200
http://bitbucket.org/pypy/pypy/changeset/8e4b90fb2e98/

Log:	Fix/skip parts of this test (could in theory play tricks, but that's
	harder now that sys.pypy_find_stdlib() automatically looks from the
	path of the libpypy-c.so)

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1019,23 +1019,32 @@
         old_sys_path = sys.path[:]
         old_cwd = os.getcwd()
 
-        sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
         if self.tmp_dir.startswith(self.trunkdir):
             skip('TMPDIR is inside the PyPy source')
-        os.chdir(self.tmp_dir)
+        sys.path.append(self.goal_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
+            os.chdir(self.tmp_dir)
+
+            # If we are running PyPy with a libpypy-c, the following
+            # lines find the stdlib anyway.  Otherwise, it is not found.
+            expected_found = (
+                '__pypy__' in sys.builtin_module_names and
+                sys.pypy_translation_info['translation.shared'])
+
             import app_main
-            app_main.setup_bootstrap_path(tmp_pypy_c)  # stdlib not found
+            app_main.setup_bootstrap_path(tmp_pypy_c)
             assert sys.executable == ''
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not expected_found:
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
             if not sys.platform == 'win32':
                 # an existing file is always 'executable' on windows
                 assert sys.executable == ''      # not executable!
-                assert sys.path == old_sys_path + [self.goal_dir]
+                if not expected_found:
+                    assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0755)
             app_main.setup_bootstrap_path(self.fake_exe)
@@ -1046,7 +1055,8 @@
             if newpath[0].endswith('__extensions__'):
                 newpath = newpath[1:]
             # we get at least 'expected_path', and maybe more (e.g.plat-linux2)
-            assert newpath[:len(self.expected_path)] == self.expected_path
+            if not expected_found:
+                assert newpath[:len(self.expected_path)] == self.expected_path
         finally:
             sys.path[:] = old_sys_path
             os.chdir(old_cwd)

From pypy.commits at gmail.com  Sat Sep 24 16:34:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 13:34:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge search-path-from-libpypy
Message-ID: <57e6e355.d7bb1c0a.91d23.e93c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87366:dc3eb3eaa65c
Date: 2016-09-24 22:14 +0200
http://bitbucket.org/pypy/pypy/changeset/dc3eb3eaa65c/

Log:	hg merge search-path-from-libpypy

	The compiled pypy-c now looks for its lib-python/lib_pypy path
	starting from the location of the *libpypy-c* instead of the
	executable. This is arguably more consistent, and also it is what
	occurs anyway if you're embedding pypy.

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -89,17 +89,20 @@
     def pypy_setup_home(ll_home, verbose):
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
-        if ll_home:
+        if ll_home and ord(ll_home[0]):
             home1 = rffi.charp2str(ll_home)
             home = os.path.join(home1, 'x') # <- so that 'll_home' can be
                                             # directly the root directory
+            dynamic = False
         else:
-            home = home1 = pypydir
-        w_path = pypy_find_stdlib(space, home)
+            home1 = "pypy's shared library location"
+            home = pypydir
+            dynamic = True
+        w_path = pypy_find_stdlib(space, home, dynamic)
         if space.is_none(w_path):
             if verbose:
                 debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
-                      " not found in '%s' or in any parent directory" % home1)
+                      " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.appexec([w_path], """(path):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1019,23 +1019,32 @@
         old_sys_path = sys.path[:]
         old_cwd = os.getcwd()
 
-        sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
         if self.tmp_dir.startswith(self.trunkdir):
             skip('TMPDIR is inside the PyPy source')
-        os.chdir(self.tmp_dir)
+        sys.path.append(self.goal_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
+            os.chdir(self.tmp_dir)
+
+            # If we are running PyPy with a libpypy-c, the following
+            # lines find the stdlib anyway.  Otherwise, it is not found.
+            expected_found = (
+                '__pypy__' in sys.builtin_module_names and
+                sys.pypy_translation_info['translation.shared'])
+
             import app_main
-            app_main.setup_bootstrap_path(tmp_pypy_c)  # stdlib not found
+            app_main.setup_bootstrap_path(tmp_pypy_c)
             assert sys.executable == ''
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not expected_found:
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
             if not sys.platform == 'win32':
                 # an existing file is always 'executable' on windows
                 assert sys.executable == ''      # not executable!
-                assert sys.path == old_sys_path + [self.goal_dir]
+                if not expected_found:
+                    assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0755)
             app_main.setup_bootstrap_path(self.fake_exe)
@@ -1046,7 +1055,8 @@
             if newpath[0].endswith('__extensions__'):
                 newpath = newpath[1:]
             # we get at least 'expected_path', and maybe more (e.g.plat-linux2)
-            assert newpath[:len(self.expected_path)] == self.expected_path
+            if not expected_found:
+                assert newpath[:len(self.expected_path)] == self.expected_path
         finally:
             sys.path[:] = old_sys_path
             os.chdir(old_cwd)
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -112,29 +112,7 @@
 #define _WIN32_WINNT 0x0501
 #include <windows.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  _MAX_PATH
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    HMODULE hModule = 0;
-    DWORD res;
-
-    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | 
-                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-                       (LPCTSTR)&_cffi_init, &hModule);
-
-    if (hModule == 0 ) {
-        _cffi_init_error("GetModuleHandleEx() failed", "");
-        return -1;
-    }
-    res = GetModuleFileName(hModule, output_home_path, CFFI_INIT_HOME_PATH_MAX);
-    if (res >= CFFI_INIT_HOME_PATH_MAX) {
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -155,28 +133,9 @@
 else:
 
     do_includes = r"""
-#include <dlfcn.h>
 #include <pthread.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  PATH_MAX
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    Dl_info info;
-    dlerror();   /* reset */
-    if (dladdr(&_cffi_init, &info) == 0) {
-        _cffi_init_error("dladdr() failed: ", dlerror());
-        return -1;
-    }
-    if (realpath(info.dli_fname, output_home_path) == NULL) {
-        perror("realpath() failed");
-        _cffi_init_error("realpath() failed", "");
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -201,14 +160,10 @@
 
 static void _cffi_init(void)
 {
-    char home[CFFI_INIT_HOME_PATH_MAX + 1];
-
     rpython_startup_code();
     RPyGilAllocate();
 
-    if (_cffi_init_home(home) != 0)
-        return;
-    if (pypy_setup_home(home, 1) != 0) {
+    if (pypy_setup_home(NULL, 1) != 0) {
         _cffi_init_error("pypy_setup_home() failed", "");
         return;
     }
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -9,6 +9,8 @@
 
 from rpython.rlib import rpath
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.sys.state import get as get_state
@@ -155,8 +157,13 @@
     return space.wrap(resolvedirof(filename))
 
 
- at unwrap_spec(executable='str0')
-def pypy_find_stdlib(space, executable):
+ at unwrap_spec(executable='str0', dynamic=int)
+def pypy_find_stdlib(space, executable, dynamic=1):
+    if dynamic and space.config.translation.shared:
+        dynamic_location = pypy_init_home()
+        if dynamic_location:
+            executable = rffi.charp2str(dynamic_location)
+            pypy_init_free(dynamic_location)
     path, prefix = find_stdlib(get_state(space), executable)
     if path is None:
         return space.w_None
@@ -164,3 +171,75 @@
     space.setitem(space.sys.w_dict, space.wrap('prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), w_prefix)
     return space.newlist([space.wrap(p) for p in path])
+
+
+# ____________________________________________________________
+
+
+if os.name == 'nt':
+
+    _source_code = r"""
+#define _WIN32_WINNT 0x0501
+#include <windows.h>
+#include <stdio.h>
+
+RPY_EXPORTED
+char *_pypy_init_home(void)
+{
+    HMODULE hModule = 0;
+    DWORD res;
+    char *p;
+
+    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                       (LPCTSTR)&_pypy_init_home, &hModule);
+
+    if (hModule == 0 ) {
+        fprintf(stderr, "PyPy initialization: GetModuleHandleEx() failed\n");
+        return NULL;
+    }
+    p = malloc(_MAX_PATH);
+    if (p == NULL)
+        return NULL;
+    res = GetModuleFileName(hModule, p, _MAX_PATH);
+    if (res >= _MAX_PATH || res <= 0) {
+        free(p);
+        fprintf(stderr, "PyPy initialization: GetModuleFileName() failed\n");
+        return NULL;
+    }
+    return p;
+}
+"""
+
+else:
+
+    _source_code = r"""
+#include <dlfcn.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+RPY_EXPORTED
+char *_pypy_init_home(void)
+{
+    Dl_info info;
+    dlerror();   /* reset */
+    if (dladdr(&_pypy_init_home, &info) == 0) {
+        fprintf(stderr, "PyPy initialization: dladdr() failed: %s\n",
+                dlerror());
+        return NULL;
+    }
+    char *p = realpath(info.dli_fname, NULL);
+    if (p == NULL) {
+        p = strdup(info.dli_fname);
+    }
+    return p;
+}
+"""
+
+_eci = ExternalCompilationInfo(separate_module_sources=[_source_code])
+
+pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP,
+                                 _nowrapper=True, compilation_info=_eci)
+pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void,
+                                 _nowrapper=True, compilation_info=_eci)
diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py
--- a/pypy/module/sys/test/test_initpath.py
+++ b/pypy/module/sys/test/test_initpath.py
@@ -1,8 +1,10 @@
 import py
 import os.path
-from pypy.module.sys.initpath import (compute_stdlib_path, find_executable, find_stdlib,
-                                      resolvedirof)
+from pypy.module.sys.initpath import (compute_stdlib_path, find_executable,
+                                      find_stdlib, resolvedirof,
+                                      pypy_init_home, pypy_init_free)
 from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
+from rpython.rtyper.lltypesystem import rffi
 
 def build_hierarchy(prefix):
     dirname = '%d.%d' % CPYTHON_VERSION[:2]
@@ -10,7 +12,7 @@
     b = prefix.join('lib-python', dirname).ensure(dir=1)
     return a, b
 
-def test_find_stdlib(tmpdir, monkeypatch):
+def test_find_stdlib(tmpdir):
     bin_dir = tmpdir.join('bin').ensure(dir=True)
     pypy = bin_dir.join('pypy').ensure(file=True)
     build_hierarchy(tmpdir)
@@ -33,6 +35,14 @@
     path, prefix = find_stdlib(None, str(pypy_sym))
     assert prefix == pypydir
 
+def test_pypy_init_home():
+    p = pypy_init_home()
+    assert p
+    s = rffi.charp2str(p)
+    pypy_init_free(p)
+    print s
+    assert os.path.exists(s)
+
 def test_compute_stdlib_path(tmpdir):
     dirs = build_hierarchy(tmpdir)
     path = compute_stdlib_path(None, str(tmpdir))

From pypy.commits at gmail.com  Sat Sep 24 16:34:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 13:34:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document 'search-path-from-libpypy',
 and write down what it simplifies
Message-ID: <57e6e357.0145c20a.a1785.bd6e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87367:edaa11222a1e
Date: 2016-09-24 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/edaa11222a1e/

Log:	Document 'search-path-from-libpypy', and write down what it
	simplifies in embedding.rst (fwiw)

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -34,9 +34,11 @@
    This function searches the PyPy standard library starting from the given
    "PyPy home directory".  The arguments are:
 
-   * ``home``: NULL terminated path to an executable inside the pypy directory
+   * ``home``: path to an executable inside the pypy directory
      (can be a .so name, can be made up).  Used to look up the standard
-     library, and is also set as ``sys.executable``.
+     library, and is also set as ``sys.executable``.  From PyPy 5.5, you can
+     just say NULL here, as long as the ``libpypy-c.so/dylib/dll`` is itself
+     inside this directory.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
@@ -82,18 +84,14 @@
 
 Note that this API is a lot more minimal than say CPython C API, so at first
 it's obvious to think that you can't do much. However, the trick is to do
-all the logic in Python and expose it via `cffi`_ callbacks. Let's assume
-we're on linux and pypy is installed in ``/opt/pypy`` (with
-subdirectories like ``lib-python`` and ``lib_pypy``), and with the
-library in ``/opt/pypy/bin/libpypy-c.so``.  (It doesn't need to be
-installed; you can also replace these paths with a local extract of the
-installation tarballs, or with your local checkout of pypy.) We write a
-little C program:
+all the logic in Python and expose it via `cffi`_ callbacks.
+We write a little C program:
 
 .. code-block:: c
 
     #include "PyPy.h"
     #include <stdio.h>
+    #include <stdlib.h>
 
     static char source[] = "print 'hello from pypy'";
 
@@ -102,9 +100,9 @@
         int res;
 
         rpython_startup_code();
-        /* note: in the path /opt/pypy/x, the final x is ignored and
-           replaced with lib-python and lib_pypy. */
-        res = pypy_setup_home("/opt/pypy/x", 1);
+        /* Before PyPy 5.5, you may need to say e.g. "/opt/pypy/bin" instead
+         * of NULL. */
+        res = pypy_setup_home(NULL, 1);
         if (res) {
             printf("Error setting pypy home!\n");
             return 1;
@@ -123,11 +121,6 @@
     $ LD_LIBRARY_PATH=/opt/pypy/bin ./x
     hello from pypy
 
-.. note:: If the compilation fails because of missing PyPy.h header file,
-          you are running PyPy <= 2.2.1.  Get it here__.
-
-.. __: https://bitbucket.org/pypy/pypy/raw/c4cd6eca9358066571500ac82aaacfdaa3889e8c/include/PyPy.h
-
 On OSX it is necessary to set the rpath of the binary if one wants to link to it,
 with a command like::
 
@@ -181,6 +174,7 @@
     /* C example */
     #include "PyPy.h"
     #include <stdio.h>
+    #include <stdlib.h>
 
     struct API {
         double (*add_numbers)(double x, double y);
@@ -196,7 +190,7 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/x", 1);
+        res = pypy_setup_home(NULL, 1);
         if (res) {
             fprintf(stderr, "Error setting pypy home!\n");
             return -1;
@@ -237,6 +231,8 @@
 Finding pypy_home
 -----------------
 
+**You can usually skip this section if you are running PyPy >= 5.5.**
+
 The function pypy_setup_home() takes as first parameter the path to a
 file from which it can deduce the location of the standard library.
 More precisely, it tries to remove final components until it finds
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -21,3 +21,14 @@
 JIT residual calls: if the called function starts with a fast-path
 like "if x.foo != 0: return x.foo", then inline the check before
 doing the CALL.  For now, string hashing is about the only case.
+
+.. branch: search-path-from-libpypy
+
+The compiled pypy now looks for its lib-python/lib_pypy path starting
+from the location of the *libpypy-c* instead of the executable. This is
+arguably more consistent, and also it is what occurs anyway if you're
+embedding pypy.  Linux distribution packagers, take note!  At a minimum,
+the ``libpypy-c.so`` must really be inside the path containing
+``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
+from somewhere else.  You no longer have to do the same with the
+``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.

From pypy.commits at gmail.com  Sat Sep 24 16:34:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 13:34:28 -0700 (PDT)
Subject: [pypy-commit] pypy search-path-from-libpypy: ready to merge
Message-ID: <57e6e354.079bc20a.b67a4.b240@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: search-path-from-libpypy
Changeset: r87365:8d10f4ee4c02
Date: 2016-09-24 22:12 +0200
http://bitbucket.org/pypy/pypy/changeset/8d10f4ee4c02/

Log:	ready to merge


From pypy.commits at gmail.com  Sat Sep 24 16:50:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 13:50:25 -0700 (PDT)
Subject: [pypy-commit] cffi default: Document inline the various options to
 compile with embedding
Message-ID: <57e6e711.87adc20a.2e741.b573@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2777:f0d20750d3d6
Date: 2016-09-24 22:50 +0200
http://bitbucket.org/cffi/cffi/changeset/f0d20750d3d6/

Log:	Document inline the various options to compile with embedding

diff --git a/demo/embedding_test.c b/demo/embedding_test.c
--- a/demo/embedding_test.c
+++ b/demo/embedding_test.c
@@ -1,7 +1,30 @@
-/* Link this program with libembedding_test.so.
+/* There are two options:
+
+   =====1=====
+
+   Link this program with _embedding_test.so.
    E.g. with gcc:
 
       gcc -o embedding_test embedding_test.c _embedding_cffi*.so
+
+   You must then run the executable with the right command
+   (LD_LIBRARY_PATH on Linux), otherwise it won't find the
+   _embedding_cffi*.so:
+
+      LD_LIBRARY_PATH=. ./embedding_test
+
+   There are platform-specific options to gcc to avoid needing
+   that, too.  Linux:
+
+      gcc -o embedding_test embedding_test.c _embedding_cffi*.so -Wl,-rpath=.
+
+   =====2=====
+
+   Compile and link the _embedding_test.c source code together with
+   this example (e.g. with PyPy):
+
+      gcc -o embedding_test embedding_test.c _embedding_cffi.c  \
+          -I/opt/pypy/include -pthread -lpypy-c
 */
 
 #include <stdio.h>

From pypy.commits at gmail.com  Sat Sep 24 16:52:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Sep 2016 13:52:51 -0700 (PDT)
Subject: [pypy-commit] cffi default: $ORIGIN is better than .
Message-ID: <57e6e7a3.c50e1c0a.71dc9.f442@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2778:1e4e8ede7303
Date: 2016-09-24 22:52 +0200
http://bitbucket.org/cffi/cffi/changeset/1e4e8ede7303/

Log:	$ORIGIN is better than .

diff --git a/demo/embedding_test.c b/demo/embedding_test.c
--- a/demo/embedding_test.c
+++ b/demo/embedding_test.c
@@ -16,7 +16,8 @@
    There are platform-specific options to gcc to avoid needing
    that, too.  Linux:
 
-      gcc -o embedding_test embedding_test.c _embedding_cffi*.so -Wl,-rpath=.
+      gcc -o embedding_test embedding_test.c _embedding_cffi*.so  \
+          -Wl,-rpath=\$ORIGIN/
 
    =====2=====
 

From pypy.commits at gmail.com  Sat Sep 24 17:00:54 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Sat, 24 Sep 2016 14:00:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Make error messages for dict and other
 non-numeric types consistent with CPython 3.5
Message-ID: <57e6e986.a3adc20a.e8481.b939@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87368:0bf6794a3933
Date: 2016-09-24 21:20 +0100
http://bitbucket.org/pypy/pypy/changeset/0bf6794a3933/

Log:	Make error messages for dict and other non-numeric types consistent
	with CPython 3.5

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -226,7 +226,8 @@
                 if e.match(space, space.w_TypeError):
                     raise oefmt(
                         space.w_TypeError,
-                        "float() argument must be a string or a number")
+                        "float() argument must be a string or a number,\
+                                not '{}'".format(type(space.unwrap(w_value)).__name__))
                 raise
             value = _string_to_float(space, w_value, value)
         w_obj = space.allocate_instance(W_FloatObject, w_floattype)
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -479,6 +479,45 @@
         assert hash(-1.0) == -2
         assert (-1.0).__hash__() == -2
 
+    def test_float_from_dict(self):
+        try:
+            float({})
+        except TypeError as e:
+            assert "not 'dict'" in str(e)
+        else:
+            assert False, 'did not raise'
+
+    def test_non_numeric_input_types(self):
+        # Test possible non-numeric types for the argument x, including
+        # subclasses of the explicitly documented accepted types.
+        class CustomStr(str): pass
+        class CustomBytes(bytes): pass
+        class CustomByteArray(bytearray): pass
+
+        factories = [
+            bytes,
+            bytearray,
+            lambda b: CustomStr(b.decode()),
+            CustomBytes,
+            CustomByteArray,
+            memoryview,
+        ]
+        try:
+            from array import array
+        except ImportError:
+            pass
+        else:
+            factories.append(lambda b: array('B', b))
+
+        for f in factories:
+            x = f(b" 3.14  ")
+            assert float(x) == 3.14
+            try:
+                float(f(b'A' * 0x10))
+            except ValueError as e:
+                assert "could not convert" in str(e)
+            else:
+                assert False, 'did not raise'
 
 class AppTestFloatHex:
     spaceconfig = {
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -25,7 +25,7 @@
 
 globals().update(rffi_platform.configure(CConfig))
 
-INVALID_MSG = "invalid literal for float()"
+INVALID_MSG = "could not convert string to float"
 
 def string_to_float(s):
     """

From pypy.commits at gmail.com  Sat Sep 24 19:42:30 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 24 Sep 2016 16:42:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix translation
Message-ID: <57e70f66.c50e1c0a.71dc9.1e14@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87369:51fce56a2bf1
Date: 2016-09-25 00:41 +0100
http://bitbucket.org/pypy/pypy/changeset/51fce56a2bf1/

Log:	Fix translation

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1556,11 +1556,11 @@
             look_for += ' or ' + also_look_for
         else:
             look_for = also_look_for
-    w_msg = u"function %s not found in library %s" % (
+    msg = u"function %s not found in library %s" % (
         unicode(look_for), space.unicode_w(space.wrap_fsdecoded(path)))
     w_name = space.newunicode(name.decode('ascii'))
     w_path = space.wrap_fsdecoded(path)
-    raise_import_error(space, w_msg, w_name, w_path)
+    raise_import_error(space, space.newunicode(msg), w_name, w_path)
 
 
 initfunctype = lltype.Ptr(lltype.FuncType([], PyObject))
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -58,7 +58,7 @@
     from pypy.module.cpyext.api import load_extension_module
     # extension names must be valid C identifiers
     modulename = modulename.encode('ascii')
-    load_extension_module(space, filename, modulename.encode('ascii'))
+    load_extension_module(space, filename, modulename)
 
     return importing.check_sys_modules_w(space, modulename)
 

From pypy.commits at gmail.com  Sat Sep 24 23:52:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 24 Sep 2016 20:52:37 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: note need to use upstream NumPy
 HEAD (issue #2403), remove whitepsace at ends of lines
Message-ID: <57e74a05.64f9c20a.f5f48.1655@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r803:baf5f6543eb1
Date: 2016-09-25 06:51 +0300
http://bitbucket.org/pypy/pypy.org/changeset/baf5f6543eb1/

Log:	note need to use upstream NumPy HEAD (issue #2403), remove
	whitepsace at ends of lines

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -106,8 +106,10 @@
 that <strong>Linux binaries are only usable on the distributions written next to
 them</strong> unless you're ready to hack your system by adding symlinks to the
 libraries it tries to open.  There are better solutions:</p>
+<pre class="literal-block">
+* use Squeaky's `portable Linux binaries`_.
+</pre>
 <ul class="simple">
-<li>use Squeaky's <a class="reference external" href="https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux">portable Linux binaries</a>.</li>
 <li>or download PyPy from your release vendor (usually an outdated
 version): <a class="reference external" href="http://packages.ubuntu.com/search?keywords=pypy&amp;searchon=names">Ubuntu</a> (<a class="reference external" href="https://launchpad.net/~pypy/+archive/ppa">PPA</a>), <a class="reference external" href="http://packages.debian.org/sid/pypy">Debian</a>, <a class="reference external" href="https://github.com/mxcl/homebrew/blob/master/Library/Formula/pypy.rb">Homebrew</a>, MacPorts,
 <a class="reference external" href="http://fedoraproject.org/wiki/Features/PyPyStack">Fedora</a>, <a class="reference external" href="http://packages.gentoo.org/package/dev-python/pypy">Gentoo</a> and <a class="reference external" href="https://wiki.archlinux.org/index.php/PyPy">Arch</a> are known to package PyPy, with various
@@ -222,10 +224,12 @@
 the Python side and NumPy objects are mediated through the slower cpyext
 layer (which hurts a few benchmarks that do a lot of element-by-element
 array accesses, for example).</p>
-<p>Installation works as usual.  For example, without using a virtualenv:</p>
+<p>Installation works using the current developement version of NumPy, since
+PyPy specific changes have been merged but not in a release version.
+For example, without using a virtualenv:</p>
 <pre class="literal-block">
 $ ./pypy-xxx/bin/pypy -m ensurepip
-$ ./pypy-xxx/bin/pip install numpy
+$ ./pypy-xxx/bin/pip install cython git+https://github.com/numpy/numpy.git
 </pre>
 <p>(See the general <a class="reference external" href="http://doc.pypy.org/en/latest/install.html">installation documentation</a> for more.)</p>
 </div>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -10,7 +10,7 @@
 
   There are `nightly binary builds`_ available. Those builds are not always
   as stable as the release, but they contain numerous bugfixes and
-  performance improvements. 
+  performance improvements.
 
 We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
 
@@ -56,7 +56,7 @@
 not be usable due to the sad story of linux binary compatibility.  This means
 that **Linux binaries are only usable on the distributions written next to
 them** unless you're ready to hack your system by adding symlinks to the
-libraries it tries to open.  There are better solutions:
+libraries it tries to open.  There are better solutions::
 
 * use Squeaky's `portable Linux binaries`_.
 
@@ -201,7 +201,7 @@
 
 .. _`nightly binary builds`: http://buildbot.pypy.org/nightly/trunk/
 
-Installing 
+Installing
 ----------
 
 All binary versions are packaged in a ``tar.bz2`` or ``zip`` file.  When
@@ -217,7 +217,7 @@
 -------------------------------
 
 The recommended way is to install ``pip``, which is the standard package
-manager of Python.  It works like it does on CPython as explained in the 
+manager of Python.  It works like it does on CPython as explained in the
 `installation documentation`_.
 
 .. _installation documentation: http://doc.pypy.org/en/latest/install.html
@@ -241,10 +241,12 @@
 layer (which hurts a few benchmarks that do a lot of element-by-element
 array accesses, for example).
 
-Installation works as usual.  For example, without using a virtualenv::
-    
+Installation works using the current developement version of NumPy, since
+PyPy specific changes have been merged but not in a release version.
+For example, without using a virtualenv::
+
     $ ./pypy-xxx/bin/pypy -m ensurepip
-    $ ./pypy-xxx/bin/pip install numpy
+    $ ./pypy-xxx/bin/pip install cython git+https://github.com/numpy/numpy.git
 
 (See the general `installation documentation`_ for more.)
 
@@ -285,7 +287,7 @@
 
 .. _translate:
 
-Building from source 
+Building from source
 --------------------
 
 (see more build instructions_)
@@ -413,8 +415,8 @@
 Packaging
 ---------
 
-Once PyPy is translated from source the binary package similar to those 
-provided in the section `Default (with a JIT Compiler)`_ above could be 
+Once PyPy is translated from source the binary package similar to those
+provided in the section `Default (with a JIT Compiler)`_ above could be
 easily created with ``package.py`` script as following::
 
     cd ./pypy/pypy/tool/release/

From pypy.commits at gmail.com  Sun Sep 25 12:28:33 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 25 Sep 2016 09:28:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3k
Message-ID: <57e7fb31.6937c20a.35c5c.f09c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r87370:991b0bdd50ab
Date: 2016-09-25 17:27 +0100
http://bitbucket.org/pypy/pypy/changeset/991b0bdd50ab/

Log:	hg merge py3k

diff --git a/lib-python/3/idlelib/CallTips.py b/lib-python/3/idlelib/CallTips.py
--- a/lib-python/3/idlelib/CallTips.py
+++ b/lib-python/3/idlelib/CallTips.py
@@ -144,7 +144,8 @@
         fob = ob_call
     else:
         fob = ob
-    if isinstance(fob, (types.FunctionType, types.MethodType)):
+    if (isinstance(fob, (types.FunctionType, types.MethodType)) and
+            hasattr(fob.__code__, 'co_code')):  # PyPy: not on <builtin-code>
         argspec = inspect.formatargspec(*inspect.getfullargspec(fob))
         if (isinstance(ob, (type, types.MethodType)) or
                 isinstance(ob_call, types.MethodType)):
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -22,35 +22,13 @@
     code, message = _ffi.getwinerror()
     raise WindowsError(code, message)
 
-_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+def _int2handle(val):
+    return _ffi.cast("HANDLE", val)
 
-class _handle(object):
-    def __init__(self, c_handle):
-        # 'c_handle' is a cffi cdata of type HANDLE, which is basically 'void *'
-        self.c_handle = c_handle
-        if int(self) != -1:
-            self.c_handle = _ffi.gc(self.c_handle, _kernel32.CloseHandle)
+def _handle2int(handle):
+    return int(_ffi.cast("intptr_t", handle))
 
-    def __int__(self):
-        return int(_ffi.cast("intptr_t", self.c_handle))
-
-    def __repr__(self):
-        return '<_subprocess.handle %d at 0x%x>' % (int(self), id(self))
-
-    def Detach(self):
-        h = int(self)
-        if h != -1:
-            c_handle = self.c_handle
-            self.c_handle = _INVALID_HANDLE_VALUE
-            _ffi.gc(c_handle, None)
-        return h
-
-    def Close(self):
-        if int(self) != -1:
-            c_handle = self.c_handle
-            self.c_handle = _INVALID_HANDLE_VALUE
-            _ffi.gc(c_handle, None)
-            _kernel32.CloseHandle(c_handle)
+_INVALID_HANDLE_VALUE = _int2handle(-1)
 
 def CreatePipe(attributes, size):
     handles = _ffi.new("HANDLE[2]")
@@ -60,25 +38,25 @@
     if not res:
         raise _WinError()
 
-    return _handle(handles[0]), _handle(handles[1])
+    return _handle2int(handles[0]), _handle2int(handles[1])
 
 def GetCurrentProcess():
-    return _handle(_kernel32.GetCurrentProcess())
+    return _handle2int(_kernel32.GetCurrentProcess())
 
 def DuplicateHandle(source_process, source, target_process, access, inherit, options=0):
     # CPython: the first three arguments are expected to be integers
     target = _ffi.new("HANDLE[1]")
 
     res = _kernel32.DuplicateHandle(
-        _ffi.cast("HANDLE", source_process),
-        _ffi.cast("HANDLE", source),
-        _ffi.cast("HANDLE", target_process),
+        _int2handle(source_process),
+        _int2handle(source),
+        _int2handle(target_process),
         target, access, inherit, options)
 
     if not res:
         raise _WinError()
 
-    return _handle(target[0])
+    return _handle2int(target[0])
 
 def _z(input):
     if input is None:
@@ -94,13 +72,14 @@
     if startup_info is not None:
         si.dwFlags = startup_info.dwFlags
         si.wShowWindow = startup_info.wShowWindow
-        # CPython: these three handles are expected to be _handle objects
+        # CPython: these three handles are expected to be
+        # subprocess.Handle (int) objects
         if startup_info.hStdInput:
-            si.hStdInput = startup_info.hStdInput.c_handle
+            si.hStdInput = _int2handle(startup_info.hStdInput)
         if startup_info.hStdOutput:
-            si.hStdOutput = startup_info.hStdOutput.c_handle
+            si.hStdOutput = _int2handle(startup_info.hStdOutput)
         if startup_info.hStdError:
-            si.hStdError = startup_info.hStdError.c_handle
+            si.hStdError = _int2handle(startup_info.hStdError)
 
     pi = _ffi.new("PROCESS_INFORMATION *")
     flags |= CREATE_UNICODE_ENVIRONMENT
@@ -120,12 +99,14 @@
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
+    return (_handle2int(pi.hProcess),
+            _handle2int(pi.hThread),
+            pi.dwProcessId,
+            pi.dwThreadId)
 
 def WaitForSingleObject(handle, milliseconds):
     # CPython: the first argument is expected to be an integer.
-    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
-                                        milliseconds)
+    res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds)
     if res < 0:
         raise _WinError()
 
@@ -135,7 +116,7 @@
     # CPython: the first argument is expected to be an integer.
     code = _ffi.new("DWORD[1]")
 
-    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
+    res = _kernel32.GetExitCodeProcess(_int2handle(handle), code)
 
     if not res:
         raise _WinError()
@@ -145,7 +126,7 @@
 def TerminateProcess(handle, exitcode):
     # CPython: the first argument is expected to be an integer.
     # The second argument is silently wrapped in a UINT.
-    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+    res = _kernel32.TerminateProcess(_int2handle(handle),
                                      _ffi.cast("UINT", exitcode))
 
     if not res:
@@ -158,19 +139,17 @@
     if not res:
         return None
     else:
-        # note: returns integer, not handle object
-        return int(_ffi.cast("intptr_t", res))
+        return _handle2int(res)
 
 def CloseHandle(handle):
-    res = _kernel32.CloseHandle(_ffi.cast("HANDLE", handle))
+    res = _kernel32.CloseHandle(_int2handle(handle))
 
     if not res:
         raise _WinError()
 
 def GetModuleFileName(module):
     buf = _ffi.new("wchar_t[]", _MAX_PATH)
-    res = _kernel32.GetModuleFileNameW(_ffi.cast("HANDLE", module),
-                                       buf, _MAX_PATH)
+    res = _kernel32.GetModuleFileNameW(_int2handle(module), buf, _MAX_PATH)
 
     if not res:
         raise _WinError()
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -252,6 +252,7 @@
         missing_positional = []
         missing_kwonly = []
         more_filling = (input_argcount < co_argcount + co_kwonlyargcount)
+        def_first = 0
         if more_filling:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
             j = 0
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -558,16 +558,6 @@
         'parser', 'fcntl', '_codecs', 'binascii'
     ]
 
-    # These modules are treated like CPython treats built-in modules,
-    # i.e. they always shadow any xx.py.  The other modules are treated
-    # like CPython treats extension modules, and are loaded in sys.path
-    # order by the fake entry '.../lib_pypy/__extensions__'.
-    MODULES_THAT_ALWAYS_SHADOW = dict.fromkeys([
-        '__builtin__', '__pypy__', '_ast', '_codecs', '_sre', '_warnings',
-        '_weakref', 'errno', '__exceptions__', 'gc', 'imp', 'marshal',
-        'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport',
-    ], None)
-
     def make_builtins(self):
         "NOT_RPYTHON: only for initializing the space."
 
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -582,3 +582,16 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def new_import_error(space, w_msg, w_name, w_path):
+    """Create a new instance of ImportError.
+
+    The result corresponds to ImportError(msg, name=name, path=path)
+    """
+    return space.appexec(
+        [w_msg, w_name, w_path], """(msg, name, path):
+            return ImportError(msg, name=name, path=path)""")
+
+def raise_import_error(space, w_msg, w_name, w_path):
+    w_exc = new_import_error(space, w_msg, w_name, w_path)
+    raise OperationError(space.w_ImportError, w_exc)
diff --git a/pypy/interpreter/interactive.py b/pypy/interpreter/interactive.py
--- a/pypy/interpreter/interactive.py
+++ b/pypy/interpreter/interactive.py
@@ -169,7 +169,8 @@
 
     def runsource(self, source, ignored_filename="<input>", symbol="single"):
         # the following hacked file name is recognized specially by error.py
-        hacked_filename = '<inline>\n' + source
+        hacked_filename = '<inline>\n' + source.encode(
+                                            'ascii', 'backslashreplace')
         compiler = self.space.getexecutioncontext().compiler
 
         # CPython 2.6 turns console input into unicode
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -107,6 +107,7 @@
         self.co_varnames = varnames
         self.co_freevars = freevars
         self.co_cellvars = cellvars
+        assert isinstance(filename, str)
         rstring.check_str0(filename)
         self.co_filename = filename
         self.co_name = name
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -3,7 +3,7 @@
 import py, os, errno
 from pypy.interpreter.error import (
     OperationError, decompose_valuefmt, get_operrcls2, new_exception_class,
-    oefmt, wrap_oserror)
+    oefmt, wrap_oserror, new_import_error)
 
 
 def test_decompose_valuefmt():
@@ -154,3 +154,8 @@
     assert operr.match(space, space.w_ValueError)
     assert operr.match(space, space.w_TypeError)
 
+def test_import_error(space):
+    w_exc = new_import_error(
+        space, space.wrap(u'msg'), space.wrap(u'name'), space.wrap(u'path'))
+    assert space.getattr(w_exc, space.wrap(u'name')).unwrap(space) == u'name'
+    assert space.getattr(w_exc, space.wrap(u'path')).unwrap(space) == u'path'
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -20,7 +20,7 @@
 from rpython.tool.udir import udir
 from rpython.translator import platform
 from pypy.module.cpyext.state import State
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError, oefmt, raise_import_error
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.nestedscope import Cell
@@ -982,7 +982,7 @@
     py_fatalerror = rffi.llexternal('%s_FatalError' % prefix,
                                     [CONST_STRING], lltype.Void,
                                     compilation_info=eci)
-    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [],
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
@@ -1523,9 +1523,10 @@
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
-        raise oefmt(space.w_ImportError,
-                    "unable to load extension module '%s': %s",
-                    path, e.msg)
+        w_name = space.newunicode(name.decode('ascii'))
+        w_path = space.wrap_fsdecoded(path)
+        raise raise_import_error(space,
+            space.wrap_fsdecoded(e.msg), w_name, w_path)
     look_for = None
     #
     if space.config.objspace.usemodules._cffi_backend:
@@ -1555,9 +1556,12 @@
             look_for += ' or ' + also_look_for
         else:
             look_for = also_look_for
-    #
-    raise oefmt(space.w_ImportError,
-                "function %s not found in library %s", look_for, path)
+    msg = u"function %s not found in library %s" % (
+        unicode(look_for), space.unicode_w(space.wrap_fsdecoded(path)))
+    w_name = space.newunicode(name.decode('ascii'))
+    w_path = space.wrap_fsdecoded(path)
+    raise_import_error(space, space.newunicode(msg), w_name, w_path)
+
 
 initfunctype = lltype.Ptr(lltype.FuncType([], PyObject))
 
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -3,8 +3,10 @@
 
 
 class AppTestImpModule:
+    # cpyext is required for _imp.create_dynamic()
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
+        'usemodules': [
+            'binascii', 'imp', 'itertools', 'time', 'struct', 'cpyext'],
     }
 
     def setup_class(cls):
@@ -55,12 +57,21 @@
             del sys.path[0]
 
     def test_create_dynamic(self):
-        import imp
+        import _imp
+        PATH = 'this/path/does/not/exist'
         class FakeSpec:
-            name = 'foo'
-            origin = 'this/path/does/not/exist'
-        raises(ImportError, imp.create_dynamic, FakeSpec())
-        raises(ImportError, imp.create_dynamic, FakeSpec(), "unused")
+            origin = PATH
+            def __init__(self, name):
+                self.name = name
+
+        excinfo = raises(ImportError, _imp.create_dynamic, FakeSpec('foo'))
+        assert excinfo.value.name == 'foo'
+        assert excinfo.value.path == PATH
+        # Note: On CPython, the behavior changes slightly if a 2nd argument is
+        # passed in, whose value is ignored. We don't implement that.
+        #raises(IOError, _imp.create_dynamic, FakeSpec(), "unused")
+
+        raises(ImportError, _imp.create_dynamic, FakeSpec(b'foo'))
 
     def test_suffixes(self):
         import imp
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -63,8 +63,6 @@
                     test_reload = "def test():\n    raise ValueError\n",
                     infinite_reload = "import infinite_reload, imp; imp.reload(infinite_reload)",
                     del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n",
-                    _md5 = "hello_world = 42\n",
-                    _pypyjson = "hello_world = 42\n",
                     gc = "should_never_be_seen = 42\n",
                     )
     root.ensure("packagenamespace", dir=1)    # empty, no __init__.py
@@ -158,7 +156,7 @@
 
 def _setup_path(space, path):
     return space.appexec([space.wrap(path)], """
-        (dn): 
+        (dn):
             import sys
             path = list(sys.path)
             sys.path.insert(0, dn)
@@ -660,52 +658,6 @@
                                      ) == 'a/b/c.py'
         raises(ValueError, imp.source_from_cache, 'a/b/c.py')
 
-    def test_shadow_builtin(self):
-        if self.runappdirect: skip("hard to test: module is already imported")
-        # 'import gc' is supposed to always find the built-in module;
-        # like CPython, it is a built-in module, so it shadows everything,
-        # even though there is a gc.py.
-        import sys
-        assert 'gc' not in sys.modules
-        import gc
-        assert not hasattr(gc, 'should_never_be_seen')
-        assert '(built-in)' in repr(gc)
-        del sys.modules['gc']
-
-    def test_shadow_extension_1(self):
-        if not self.runappdirect:
-            skip("I don't understand why it fails, but it works in -A "
-                 "on top of a translated PyPy.  Good enough...")
-        # 'import _pypyjson' is supposed to find _pypyjson.py if there is
-        # one in sys.path.
-        import sys
-        assert '_pypyjson' not in sys.modules
-        try:
-            import _pypyjson
-            assert hasattr(_pypyjson, 'hello_world')
-            assert '(built-in)' not in repr(_pypyjson)
-        finally:
-            sys.modules.pop('_pypyjson', None)
-
-    def test_shadow_extension_2(self):
-        if self.runappdirect: skip("hard to test: module is already imported")
-        # 'import _md5' is supposed to find the built-in module even
-        # if there is also one in sys.path as long as it is *after* the
-        # special entry '.../lib_pypy/__extensions__'.  (Note that for now
-        # there is one in lib_pypy/_md5.py, which should not be seen
-        # either; hence the (built-in) test below.)
-        import sys
-        assert '_md5' not in sys.modules
-        sys.path.append(sys.path.pop(0))
-        try:
-            import _md5
-            assert not hasattr(_md5, 'hello_world')
-            assert hasattr(_md5, 'md5')
-            assert '(built-in)' in repr(_md5)
-        finally:
-            sys.path.insert(0, sys.path.pop())
-            sys.modules.pop('_md5', None)
-
     def test_invalid_pathname(self):
         skip("This test fails on CPython 3.3, but passes on CPython 3.4+")
         import imp
@@ -993,11 +945,11 @@
         assert s.no_nul
 
 
-def test_PYTHONPATH_takes_precedence(space): 
+def test_PYTHONPATH_takes_precedence(space):
     if sys.platform == "win32":
         py.test.skip("unresolved issues with win32 shell quoting rules")
-    from pypy.interpreter.test.test_zpy import pypypath 
-    extrapath = udir.ensure("pythonpath", dir=1) 
+    from pypy.interpreter.test.test_zpy import pypypath
+    extrapath = udir.ensure("pythonpath", dir=1)
     extrapath.join("sched.py").write("print(42)\n")
     old = os.environ.get('PYTHONPATH', None)
     oldlang = os.environ.pop('LANG', None)
diff --git a/pypy/objspace/test/test_descriptor.py b/pypy/objspace/test/test_descriptor.py
--- a/pypy/objspace/test/test_descriptor.py
+++ b/pypy/objspace/test/test_descriptor.py
@@ -73,6 +73,10 @@
         raises(AttributeError, X.v.__delete__, x)
 
     def test_invalid_unicode_identifier(self):
+        skip("utf-8 encoding before translation accepts lone surrogates, "
+             "because it is Python 2.7, but after translation it does not. "
+             "Moreover, CPython 3.x accepts such unicode attributes anyway. "
+             "This makes this test half-wrong for now.")
         class X(object):
             pass
         x = X()
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -142,6 +142,9 @@
             # They may be extension modules on CPython
             name = None
             for name in missing.copy():
+                if name == 'cpyext':
+                    missing.remove(name)
+                    continue
                 try:
                     __import__(name)
                 except ImportError:

From pypy.commits at gmail.com  Sun Sep 25 12:46:20 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 25 Sep 2016 09:46:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: reapply 995f8366f830 to cffi: accept py3
 strs via CreateProcessW
Message-ID: <57e7ff5c.d32d1c0a.d0b33.2c3a@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r87371:0490b8a7a564
Date: 2016-09-25 09:45 -0700
http://bitbucket.org/pypy/pypy/changeset/0490b8a7a564/

Log:	reapply 995f8366f830 to cffi: accept py3 strs via CreateProcessW

diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -76,9 +76,9 @@
 HANDLE WINAPI GetCurrentProcess(void);
 BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
                             DWORD, BOOL, DWORD);
-BOOL WINAPI CreateProcessA(char *, char *, void *,
-                           void *, BOOL, DWORD, char *,
-                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+BOOL WINAPI CreateProcessW(wchar_t *, wchar_t *, void *,
+                           void *, BOOL, DWORD, wchar_t *,
+                           wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
 DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
 BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -58,12 +58,12 @@
 
     return _handle2int(target[0])
 
-def _z(input):
+def _Z(input):
     if input is None:
         return _ffi.NULL
-    if isinstance(input, basestring):
-        return str(input)
-    raise TypeError("string/unicode/None expected, got %r" % (
+    if isinstance(input, str):
+        return input
+    raise TypeError("str or None expected, got %r" % (
         type(input).__name__,))
 
 def CreateProcess(name, command_line, process_attr, thread_attr,
@@ -92,9 +92,9 @@
     else:
         envbuf = _ffi.NULL
 
-    res = _kernel32.CreateProcessA(_z(name), _z(command_line), _ffi.NULL,
+    res = _kernel32.CreateProcessW(_Z(name), _Z(command_line), _ffi.NULL,
                                    _ffi.NULL, inherit, flags, envbuf,
-                                   _z(start_dir), si, pi)
+                                   _Z(start_dir), si, pi)
 
     if not res:
         raise _WinError()

From pypy.commits at gmail.com  Sun Sep 25 13:30:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 25 Sep 2016 10:30:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: tweak
Message-ID: <57e809c5.c50e1c0a.71dc9.4630@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87372:07eef320df99
Date: 2016-09-25 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/07eef320df99/

Log:	tweak

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -521,7 +521,10 @@
     if 'faulthandler' in sys.builtin_module_names:
         if 'faulthandler' in sys._xoptions or os.getenv('PYTHONFAULTHANDLER'):
             import faulthandler
-            faulthandler.enable(2)   # manually set to stderr
+            try:
+                faulthandler.enable(2)   # manually set to stderr
+            except ValueError:
+                pass      # ignore "2 is not a valid file descriptor"
 
 ##    if not we_are_translated():
 ##        for key in sorted(options):
diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py
--- a/pypy/module/faulthandler/dumper.py
+++ b/pypy/module/faulthandler/dumper.py
@@ -53,7 +53,7 @@
     but hopefully close enough).  In particular, this is written as
     RPython but shouldn't allocate anything.
     """
-    _dump("Stack (most recent call first):\n")
+    _dump("Current stack (most recent call first):\n")
 
     s = rvmprof_cintf.get_rvmprof_stack()
     depth = 0

From pypy.commits at gmail.com  Sun Sep 25 13:30:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 25 Sep 2016 10:30:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: Change the API in rvmprof to
 support more directly what we need, 
Message-ID: <57e809c6.0aa71c0a.e309a.405e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87373:7ac3dadd629a
Date: 2016-09-25 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/7ac3dadd629a/

Log:	Change the API in rvmprof to support more directly what we need, in
	a way that is also more testable

diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -37,8 +37,3 @@
 
 def disable():
     _get_vmprof().disable()
-
- at specialize.arg(0)
-def enum_all_code_objs(CodeClass, callback, arg):
-    assert _was_registered(CodeClass)
-    CodeClass._vmprof_enum_all_code_objs(callback, arg)
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -156,7 +156,7 @@
     vmprof_tl_stack.setraw(x)
 
 #
-# faulthandler support
+# rvmprof.traceback support
 
 def get_rvmprof_stack():
     return vmprof_tl_stack.get_or_make_raw()
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -116,25 +116,6 @@
         # the types of code objects
         prev = self._gather_all_code_objs
         self._gather_all_code_objs = gather_all_code_objs
-        #
-        # For special usages: the faulthandler pypy module uses this.
-        # It must not allocate anything.
-        @staticmethod
-        @rgc.no_collect
-        def enum_all_code_objs(callback, arg):
-            all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles()
-            i = len(all_code_wrefs) - 1
-            while i >= 0:
-                code = all_code_wrefs[i]()
-                i -= 1
-                if code is not None:
-                    uid = code._vmprof_unique_id
-                    if uid != 0:
-                        res = callback(code, uid, arg)
-                        if res != 0:
-                            return res
-            return 0
-        CodeClass._vmprof_enum_all_code_objs = enum_all_code_objs
 
     @jit.dont_look_inside
     def enable(self, fileno, interval):
diff --git a/rpython/rlib/rvmprof/test/test_traceback.py b/rpython/rlib/rvmprof/test/test_traceback.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/test/test_traceback.py
@@ -0,0 +1,31 @@
+from rpython.rlib import rvmprof
+from rpython.rlib.rvmprof.traceback import traceback
+
+
+def test_direct():
+    class MyCode:
+        pass
+    def get_name(mycode):
+        raise NotImplementedError
+    rvmprof.register_code_object_class(MyCode, get_name)
+    #
+    @rvmprof.vmprof_execute_code("mycode", lambda code, level: code,
+                                 _hack_update_stack_untranslated=True)
+    def mainloop(code, level):
+        if level > 0:
+            mainloop(code, level - 1)
+        else:
+            traceback(MyCode, my_callback, 42)
+    #
+    seen = []
+    def my_callback(depth, code, arg):
+        seen.append((depth, code, arg))
+        return 0
+    #
+    code1 = MyCode()
+    rvmprof.register_code(code1, "foo")
+    mainloop(code1, 2)
+    #
+    assert seen == [(0, code1, 42),
+                    (1, code1, 42),
+                    (2, code1, 42)]
diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/traceback.py
@@ -0,0 +1,38 @@
+"""
+Semi-public interface to gather and print a raw traceback, e.g.
+from the faulthandler module.
+"""
+
+from rpython.rlib.rvmprof import cintf
+from rpython.rlib.objectmodel import specialize
+
+
+ at specialize.arg(0, 1)
+def traceback(CodeClass, callback, arg):
+    """For each frame, invoke 'callback(depth, code_obj, arg)'.
+    'code_obj' is either a CodeClass, or None if we fail to determine
+    what it should be (shouldn't occur, but you never know).  If it
+    returns a non-null integer, stop and return that.  Otherwise,
+    continue.  If all callbacks return 0, this returns 0.
+    """
+    s = cintf.get_rvmprof_stack()
+    depth = 0
+    while s:
+        if s.c_kind == cintf.VMPROF_CODE_TAG:
+            code_id = s.c_value
+            found_code = None
+            if code_id != 0:
+                all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles()
+                i = len(all_code_wrefs) - 1
+                while i >= 0:
+                    code = all_code_wrefs[i]()
+                    if code is not None and code._vmprof_unique_id == code_id:
+                        found_code = code
+                        break
+                    i -= 1
+            result = callback(depth, found_code, arg)
+            if result != 0:
+                return result
+            depth += 1
+        s = s.c_next
+    return 0

From pypy.commits at gmail.com  Sun Sep 25 13:41:40 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 25 Sep 2016 10:41:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: Translation test
Message-ID: <57e80c54.e50fc20a.b7e47.0f07@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87374:a38dd2ec24eb
Date: 2016-09-25 19:41 +0200
http://bitbucket.org/pypy/pypy/changeset/a38dd2ec24eb/

Log:	Translation test

diff --git a/rpython/rlib/rvmprof/test/test_traceback.py b/rpython/rlib/rvmprof/test/test_traceback.py
--- a/rpython/rlib/rvmprof/test/test_traceback.py
+++ b/rpython/rlib/rvmprof/test/test_traceback.py
@@ -1,5 +1,7 @@
+import re
 from rpython.rlib import rvmprof
 from rpython.rlib.rvmprof.traceback import traceback
+from rpython.translator.interactive import Translation
 
 
 def test_direct():
@@ -29,3 +31,35 @@
     assert seen == [(0, code1, 42),
                     (1, code1, 42),
                     (2, code1, 42)]
+
+def test_compiled():
+    class MyCode:
+        pass
+    def get_name(mycode):
+        raise NotImplementedError
+    rvmprof.register_code_object_class(MyCode, get_name)
+
+    @rvmprof.vmprof_execute_code("mycode", lambda code, level: code)
+    def mainloop(code, level):
+        if level > 0:
+            mainloop(code, level - 1)
+        else:
+            traceback(MyCode, my_callback, 42)
+
+    def my_callback(depth, code, arg):
+        print depth, code, arg
+        return 0
+
+    def f(argv):
+        code1 = MyCode()
+        rvmprof.register_code(code1, "foo")
+        mainloop(code1, 2)
+        return 0
+
+    t = Translation(f, None, gc="boehm")
+    t.compile_c()
+    stdout = t.driver.cbuilder.cmdexec('')
+    r = re.compile("(\d+) [<]MyCode object at 0x([0-9a-f]+)[>] 42\n")
+    got = r.findall(stdout)
+    addr = got[0][1]
+    assert got == [("0", addr), ("1", addr), ("2", addr)]

From pypy.commits at gmail.com  Sun Sep 25 13:58:28 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 25 Sep 2016 10:58:28 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-null-slots: cpython compatible slot
 lookup does not traverse mro,
 implement and ensure slots are filled at PyType_Ready
Message-ID: <57e81044.43e61c0a.8b044.6635@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-null-slots
Changeset: r87375:c85aef87f0df
Date: 2016-09-25 15:37 +0300
http://bitbucket.org/pypy/pypy/changeset/c85aef87f0df/

Log:	cpython compatible slot lookup does not traverse mro, implement and
	ensure slots are filled at PyType_Ready

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -40,7 +40,7 @@
 def _PyObject_New(space, type):
     return _PyObject_NewVar(space, type, 0)
 
- at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject, result_is_ll=True)
+ at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyVarObject, result_is_ll=True)
 def _PyObject_NewVar(space, type, itemcount):
     w_type = from_ref(space, rffi.cast(PyObject, type))
     assert isinstance(w_type, W_TypeObject)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -453,6 +453,23 @@
             api_func = slot_func.api_func
             handled = True
 
+    # unary functions returning Py_ssize_t
+    for tp_name, attr in [('tp_as_sequence.c_sq_length', '__len__'),
+                          #('tp_as_mapping.c_mp_length', '__len__'),
+                          ]:
+        if name == tp_name:
+            slot_fn = w_type.getdictvalue(space, attr)
+            if slot_fn is None:
+                return
+
+            @cpython_api([PyObject], Py_ssize_t, header=header, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self):
+                ret = space.call_function(slot_fn, w_self)
+                return space.int_w(ret)
+            api_func = slot_func.api_func
+            handled = True
+
     # binary functions
     for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'),
                           ('tp_as_number.c_nb_subtract', '__sub__'),
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -480,6 +480,7 @@
         a = module.newsubstr('abc')
         assert type(a).__name__ == 'string_'
         assert a == 'abc'
+        assert str(a) == 'abc'
         #print type(a).mro(type(a))
         raises(ValueError, int, a)
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -479,6 +479,10 @@
         if pto.c_tp_doc:
             self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
+    def lookup(self, name):
+        # do not traverse the mro, look only in self
+        return self.getdictvalue(self.space, name)
+
 @bootstrap_function
 def init_typeobject(space):
     make_typedescr(space.w_type.layout.typedef,
@@ -819,15 +823,24 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
+        remap_slots = False
         if not py_type.c_tp_as_number: 
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
+            remap_slots = True
         if not py_type.c_tp_as_sequence:
             py_type.c_tp_as_sequence = base.c_tp_as_sequence
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
-        if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping
-        if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer
+            remap_slots = True
+        if not py_type.c_tp_as_mapping:
+            py_type.c_tp_as_mapping = base.c_tp_as_mapping
+            remap_slots = True
+        if not py_type.c_tp_as_buffer:
+            py_type.c_tp_as_buffer = base.c_tp_as_buffer
+            remap_slots = True
+        if remap_slots:
+            add_operators(space, w_obj.dict_w, py_type)
 
     return w_obj
 

From pypy.commits at gmail.com  Sun Sep 25 15:30:48 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Sun, 25 Sep 2016 12:30:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove excess spaces from error message
 (commit f1684db)
Message-ID: <57e825e8.4398c20a.53134.3ba5@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87377:a049fdfa2c3c
Date: 2016-09-25 20:25 +0100
http://bitbucket.org/pypy/pypy/changeset/a049fdfa2c3c/

Log:	Remove excess spaces from error message (commit f1684db)

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -226,8 +226,8 @@
                 if e.match(space, space.w_TypeError):
                     raise oefmt(
                         space.w_TypeError,
-                        "float() argument must be a string or a number,\
-                                not '%T'", w_value)
+                        ("float() argument must be a string or a number,"
+                                "not '%T'"), w_value)
                 raise
             value = _string_to_float(space, w_value, value)
         w_obj = space.allocate_instance(W_FloatObject, w_floattype)

From pypy.commits at gmail.com  Sun Sep 25 15:30:46 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Sun, 25 Sep 2016 12:30:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Replace invalid RPython in floatobject.py
 (commit 0bf6794, pull request #481)
Message-ID: <57e825e6.c22e1c0a.fa424.65b9@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87376:f1684dba7231
Date: 2016-09-25 19:25 +0100
http://bitbucket.org/pypy/pypy/changeset/f1684dba7231/

Log:	Replace invalid RPython in floatobject.py (commit 0bf6794, pull
	request #481)

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -227,7 +227,7 @@
                     raise oefmt(
                         space.w_TypeError,
                         "float() argument must be a string or a number,\
-                                not '{}'".format(type(space.unwrap(w_value)).__name__))
+                                not '%T'", w_value)
                 raise
             value = _string_to_float(space, w_value, value)
         w_obj = space.allocate_instance(W_FloatObject, w_floattype)

From pypy.commits at gmail.com  Sun Sep 25 16:15:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 25 Sep 2016 13:15:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: formatting
Message-ID: <57e8306a.4756c20a.6a349.3f44@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r87378:aef73af89833
Date: 2016-09-25 22:15 +0200
http://bitbucket.org/pypy/pypy/changeset/aef73af89833/

Log:	formatting

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -226,7 +226,7 @@
                 if e.match(space, space.w_TypeError):
                     raise oefmt(
                         space.w_TypeError,
-                        ("float() argument must be a string or a number,"
+                        ("float() argument must be a string or a number, "
                                 "not '%T'"), w_value)
                 raise
             value = _string_to_float(space, w_value, value)

From pypy.commits at gmail.com  Sun Sep 25 16:41:12 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 25 Sep 2016 13:41:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: regenerate
Message-ID: <57e83668.119d1c0a.6256.8281@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r87379:2a81d27e7c2b
Date: 2016-09-25 13:40 -0700
http://bitbucket.org/pypy/pypy/changeset/2a81d27e7c2b/

Log:	regenerate

diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
-    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
-    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
-    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x13\x11\x00\x00\x59\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x16\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x13\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x4C\x03\x00\x00\x2D\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2D\x11\x00\x00\x2D\x11\x00\x00\x54\x03\x00\x00\x53\x03\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x16\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x15\x11\x00\x00\x2D\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x02\x0F\x00\x00\x47\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x56\x03\x00\x00\x02\x01\x00\x00\x58\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x18\x23CloseHandle',0,b'\x00\x00\x12\x23CreatePipe',0,b'\x00\x00\x2C\x23CreateProcessW',0,b'\x00\x00\x23\x23DuplicateHandle',0,b'\x00\x00\x51\x23GetCurrentProcess',0,b'\x00\x00\x1F\x23GetExitCodeProcess',0,b'\x00\x00\x3F\x23GetModuleFileNameW',0,b'\x00\x00\x4E\x23GetStdHandle',0,b'\x00\x00\x44\x23GetVersion',0,b'\x00\x00\x1B\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x49\x23_getwch',0,b'\x00\x00\x49\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x4B\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x46\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x53\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x16\x11dwProcessId',b'\x00\x00\x16\x11dwThreadId'),(b'\x00\x00\x00\x54\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x16\x11cb',b'\x00\x00\x55\x11lpReserved',b'\x00\x00\x55\x11lpDesktop',b'\x00\x00\x55\x11lpTitle',b'\x00\x00\x16\x11dwX',b'\x00\x00\x16\x11dwY',b'\x00\x00\x16\x11dwXSize',b'\x00\x00\x16\x11dwYSize',b'\x00\x00\x16\x11dwXCountChars',b'\x00\x00\x16\x11dwYCountChars',b'\x00\x00\x16\x11dwFillAttribute',b'\x00\x00\x16\x11dwFlags',b'\x00\x00\x47\x11wShowWindow',b'\x00\x00\x47\x11cbReserved2',b'\x00\x00\x57\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x36LPPROCESS_INFORMATION',b'\x00\x00\x00\x35LPSTARTUPINFO',b'\x00\x00\x00\x53PROCESS_INFORMATION',b'\x00\x00\x00\x54STARTUPINFO',b'\x00\x00\x00\x47wint_t'),
 )

From pypy.commits at gmail.com  Sun Sep 25 22:53:37 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Sun, 25 Sep 2016 19:53:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Revert change to rpython/rlib/rfloat.py
 in commit 0bf6794 (pull request #481);
 amend error message in pypy/objspace/std/floatobject.py instead.
Message-ID: <57e88db1.cbc11c0a.70bcb.c11d@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87380:a388db9afbb6
Date: 2016-09-25 21:28 +0100
http://bitbucket.org/pypy/pypy/changeset/a388db9afbb6/

Log:	Revert change to rpython/rlib/rfloat.py in commit 0bf6794 (pull
	request #481); amend error message in
	pypy/objspace/std/floatobject.py instead.

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -207,7 +207,8 @@
             try:
                 return rfloat.string_to_float(string)
             except ParseStringError as e:
-                raise wrap_parsestringerror(space, e, w_source)
+                raise oefmt(space.w_ValueError,
+                            "could not convert string to float: %R", w_source)
 
         w_value = w_x     # 'x' is the keyword argument name in CPython
         if space.lookup(w_value, "__float__") is not None:
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -25,7 +25,7 @@
 
 globals().update(rffi_platform.configure(CConfig))
 
-INVALID_MSG = "could not convert string to float"
+INVALID_MSG = "invalid literal for float()"
 
 def string_to_float(s):
     """

From pypy.commits at gmail.com  Sun Sep 25 22:53:39 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Sun, 25 Sep 2016 19:53:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Merged pypy/pypy/py3.5 into py3.5
Message-ID: <57e88db3.a970c20a.6630c.8eb7@mx.google.com>

Author: remarkablerocket <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87381:60abb56dd00a
Date: 2016-09-25 21:37 +0100
http://bitbucket.org/pypy/pypy/changeset/60abb56dd00a/

Log:	Merged pypy/pypy/py3.5 into py3.5

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -227,7 +227,7 @@
                 if e.match(space, space.w_TypeError):
                     raise oefmt(
                         space.w_TypeError,
-                        ("float() argument must be a string or a number,"
+                        ("float() argument must be a string or a number, "
                                 "not '%T'"), w_value)
                 raise
             value = _string_to_float(space, w_value, value)

From pypy.commits at gmail.com  Sun Sep 25 22:57:59 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Sun, 25 Sep 2016 19:57:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: simplify
Message-ID: <57e88eb7.898b1c0a.d61c1.c88e@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r87382:f7b4a8aedcfc
Date: 2016-09-25 19:57 -0700
http://bitbucket.org/pypy/pypy/changeset/f7b4a8aedcfc/

Log:	simplify

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -225,10 +225,9 @@
                 value = space.charbuf_w(w_value)
             except OperationError as e:
                 if e.match(space, space.w_TypeError):
-                    raise oefmt(
-                        space.w_TypeError,
-                        ("float() argument must be a string or a number, "
-                                "not '%T'"), w_value)
+                    raise oefmt(space.w_TypeError,
+                                "float() argument must be a string or a "
+                                "number, not '%T'", w_value)
                 raise
             value = _string_to_float(space, w_value, value)
         w_obj = space.allocate_instance(W_FloatObject, w_floattype)

From pypy.commits at gmail.com  Mon Sep 26 02:13:18 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 25 Sep 2016 23:13:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: add try_inline,
 like always_inline and dont_inline
Message-ID: <57e8bc7e.ce471c0a.0895.016b@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r87384:25baf9e0c32e
Date: 2016-09-22 09:18 +0200
http://bitbucket.org/pypy/pypy/changeset/25baf9e0c32e/

Log:	add try_inline, like always_inline and dont_inline

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -217,6 +217,12 @@
     func._dont_inline_ = True
     return func
 
+def try_inline(func):
+    """ tell the RPython inline (not the JIT!), to try to inline this function,
+    no matter its size."""
+    func._always_inline_ = 'try'
+    return func
+
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config)
+    fetch_translated_config, try_inline)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -483,6 +483,13 @@
         return a, b, c
     assert f._always_inline_ is True
 
+def test_try_inline():
+    @try_inline
+    def f(a, b, c):
+        return a, b, c
+    assert f._always_inline_ == "try"
+
+
 def test_enforceargs_defaults():
     @enforceargs(int, int)
     def f(a, b=40):

From pypy.commits at gmail.com  Mon Sep 26 02:13:16 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 25 Sep 2016 23:13:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: replace some _annspecialcase_ with
 @specialize in the interpreter
Message-ID: <57e8bc7c.c336c20a.461f3.c156@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r87383:5960b6ead8d1
Date: 2016-09-22 08:05 +0200
http://bitbucket.org/pypy/pypy/changeset/5960b6ead8d1/

Log:	replace some _annspecialcase_ with @specialize in the interpreter

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -375,12 +375,12 @@
 class BufferInterfaceNotFound(Exception):
     pass
 
+ at specialize.memo()
 def wrappable_class_name(Class):
     try:
         return Class.typedef.name
     except AttributeError:
         return 'internal subclass of %s' % (Class.__name__,)
-wrappable_class_name._annspecialcase_ = 'specialize:memo'
 
 class CannotHaveLock(Exception):
     """Raised by space.allocate_lock() if we're translating."""
@@ -829,12 +829,13 @@
             assert type(s) is str
         return self.interned_strings.get(s) is not None
 
+    @specialize.arg(1)
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
             raise DescrMismatch()
         return w_obj
-    descr_self_interp_w._annspecialcase_ = 'specialize:arg(1)'
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         """
         Unwrap w_obj, checking that it is an instance of the required internal
@@ -849,7 +850,6 @@
                         wrappable_class_name(RequiredClass),
                         w_obj.getclass(self))
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def unpackiterable(self, w_iterable, expected_length=-1):
         """Unpack an iterable into a real (interpreter-level) list.
@@ -1288,6 +1288,7 @@
             self.setitem(w_globals, w_key, self.wrap(self.builtin))
         return statement.exec_code(self, w_globals, w_locals)
 
+    @specialize.arg(2)
     def appexec(self, posargs_w, source):
         """ return value from executing given source at applevel.
             EXPERIMENTAL. The source must look like
@@ -1299,7 +1300,6 @@
         w_func = self.fromcache(AppExecCache).getorbuild(source)
         args = Arguments(self, list(posargs_w))
         return self.call_args(w_func, args)
-    appexec._annspecialcase_ = 'specialize:arg(2)'
 
     def _next_or_none(self, w_it):
         try:
@@ -1309,6 +1309,7 @@
                 raise
             return None
 
+    @specialize.arg(3)
     def compare_by_iteration(self, w_iterable1, w_iterable2, op):
         w_it1 = self.iter(w_iterable1)
         w_it2 = self.iter(w_iterable2)
@@ -1331,7 +1332,6 @@
                 if op == 'gt': return self.gt(w_x1, w_x2)
                 if op == 'ge': return self.ge(w_x1, w_x2)
                 assert False, "bad value for op"
-    compare_by_iteration._annspecialcase_ = 'specialize:arg(3)'
 
     def decode_index(self, w_index_or_slice, seqlength):
         """Helper for custom sequence implementations
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -109,6 +109,7 @@
 # we need two subclasses of the app-level type, one to add mapdict, and then one
 # to add del to not slow down the GC.
 
+ at specialize.memo()
 def get_unique_interplevel_subclass(space, cls):
     "NOT_RPYTHON: initialization-time only"
     assert cls.typedef.acceptable_as_base_class
@@ -119,7 +120,6 @@
         assert cls not in _unique_subclass_cache
         _unique_subclass_cache[cls] = subcls
         return subcls
-get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo"
 _unique_subclass_cache = {}
 
 def _getusercls(cls, reallywantdict=False):
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.debug import check_nonneg
+from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
@@ -316,12 +317,12 @@
             w_currently_in_repr = ec._py_repr = space.newdict()
         return dequerepr(space, w_currently_in_repr, space.wrap(self))
 
+    @specialize.arg(2)
     def compare(self, w_other, op):
         space = self.space
         if not isinstance(w_other, W_Deque):
             return space.w_NotImplemented
         return space.compare_by_iteration(space.wrap(self), w_other, op)
-    compare._annspecialcase_ = 'specialize:arg(2)'
 
     def lt(self, w_other):
         return self.compare(w_other, 'lt')
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -6,6 +6,7 @@
 import time
 from pypy.interpreter.error import OperationError
 from pypy.module._pypyjson.interp_decoder import loads
+from rpython.rlib.objectmodel import specialize
 
 
 ## MSG = open('msg.json').read()
@@ -91,6 +92,7 @@
     def wrapfloat(self, x):
         return W_Float(x)
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if isinstance(x, int):
             return W_Int(x)
@@ -100,7 +102,6 @@
         ##     assert False
         else:
             return W_Unicode(unicode(x))
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
 
 fakespace = FakeSpace()
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -9,6 +9,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 import rpython.rlib.rposix as rposix
 
 _MS_WINDOWS = os.name == "nt"
@@ -255,6 +256,7 @@
 
 _ARM = rffi_platform.getdefined('__arm__', '')
 
+ at specialize.arg(2)
 def read_ptr(ptr, ofs, TP):
     T = lltype.Ptr(rffi.CArray(TP))
     for c in unroll_letters_for_floats:
@@ -274,8 +276,8 @@
                 return ptr_val
     else:
         return rffi.cast(T, ptr)[ofs]
-read_ptr._annspecialcase_ = 'specialize:arg(2)'
 
+ at specialize.argtype(2)
 def write_ptr(ptr, ofs, value):
     TP = lltype.typeOf(value)
     T = lltype.Ptr(rffi.CArray(TP))
@@ -296,7 +298,6 @@
                 return
     else:
         rffi.cast(T, ptr)[ofs] = value
-write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
@@ -374,14 +375,15 @@
     def getrawsize(self):
         raise NotImplementedError("abstract base class")
 
+ at specialize.arg(0)
 def unwrap_truncate_int(TP, space, w_arg):
     if space.isinstance_w(w_arg, space.w_int):
         return rffi.cast(TP, space.int_w(w_arg))
     else:
         return rffi.cast(TP, space.bigint_w(w_arg).ulonglongmask())
-unwrap_truncate_int._annspecialcase_ = 'specialize:arg(0)'
 
 
+ at specialize.arg(1)
 def unwrap_value(space, push_func, add_arg, argdesc, letter, w_arg):
     if letter in TYPEMAP_PTR_LETTERS:
         # check for NULL ptr
@@ -422,10 +424,10 @@
                 return
         else:
             raise oefmt(space.w_TypeError, "cannot directly write value")
-unwrap_value._annspecialcase_ = 'specialize:arg(1)'
 
 ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items())
 
+ at specialize.arg(1)
 def wrap_value(space, func, add_arg, argdesc, letter):
     for c, ll_type in ll_typemap_iter:
         if letter == c:
@@ -437,7 +439,6 @@
             else:
                 return space.wrap(func(add_arg, argdesc, ll_type))
     raise oefmt(space.w_TypeError, "cannot directly read value")
-wrap_value._annspecialcase_ = 'specialize:arg(1)'
 
 NARROW_INTEGER_TYPES = 'cbhiBIH?'
 
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, signedtype, r_uint, \
     r_ulonglong
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import specialize
 import sys
 
 IS_BIG_ENDIAN = sys.byteorder == 'big'
@@ -284,6 +285,7 @@
 def NUM_BITS(x):
     return x >> 16
 
+ at specialize.arg(1)
 def BIT_MASK(x, ll_t):
     if ll_t is lltype.SignedLongLong or ll_t is lltype.UnsignedLongLong:
         one = r_ulonglong(1)
@@ -291,8 +293,8 @@
         one = r_uint(1)
     # to avoid left shift by x == sizeof(ll_t)
     return (((one << (x - 1)) - 1) << 1) + 1
-BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.argtype(2)
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
     TP = lltype.typeOf(value)
@@ -313,8 +315,8 @@
                 value = rffi.cast(TP, current)
             break
     write_ptr(ptr, 0, value)
-push_field._annspecialcase_ = 'specialize:argtype(2)'
 
+ at specialize.arg(2)
 def cast_pos(self, i, ll_t):
     pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
     value = read_ptr(pos, 0, ll_t)
@@ -337,7 +339,6 @@
                 value = rffi.cast(ll_t, value)
             break
     return value
-cast_pos._annspecialcase_ = 'specialize:arg(2)'
 
 class W_StructureInstance(W_DataInstance):
     def __init__(self, space, shape, address):
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -2,6 +2,7 @@
 import sys
 
 from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError, oefmt
 
 class State:
@@ -17,6 +18,7 @@
     else:
         return space.float_w(space.float(w_x))
 
+ at specialize.arg(1)
 def math1(space, f, w_x):
     x = _get_double(space, w_x)
     try:
@@ -26,8 +28,8 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return space.wrap(y)
-math1._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.arg(1)
 def math1_w(space, f, w_x):
     x = _get_double(space, w_x)
     try:
@@ -37,8 +39,8 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return r
-math1_w._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.arg(1)
 def math2(space, f, w_x, w_snd):
     x = _get_double(space, w_x)
     snd = _get_double(space, w_snd)
@@ -49,7 +51,6 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return space.wrap(r)
-math2._annspecialcase_ = 'specialize:arg(1)'
 
 def trunc(space, w_x):
     """Truncate x."""
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -7,6 +7,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
+from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
@@ -571,6 +572,7 @@
             return self.w_character_data_handler or space.w_None
         return self.handlers[index]
 
+    @specialize.arg(2)
     def sethandler(self, space, name, w_handler, index, setter, handler):
         if name == 'CharacterDataHandler':
             self.flush_character_buffer(space)
@@ -582,8 +584,6 @@
         self.handlers[index] = w_handler
         setter(self.itself, handler)
 
-    sethandler._annspecialcase_ = 'specialize:arg(2)'
-
     all_chars = ''.join(chr(i) for i in range(256))
 
     def UnknownEncodingHandler(self, space, name, info):
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -9,68 +9,68 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 
+ at specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
     w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
                                                        '__getattribute__')
     return w_getattribute
-object_getattribute._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
     w_src, w_setattr = space.lookup_in_type_where(space.w_object,
                                                   '__setattr__')
     return w_setattr
-object_setattr._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
     w_src, w_delattr = space.lookup_in_type_where(space.w_object,
                                                   '__delattr__')
     return w_delattr
-object_delattr._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
     w_src, w_hash = space.lookup_in_type_where(space.w_object,
                                                   '__hash__')
     return w_hash
-object_hash._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
     w_src, w_eq = space.lookup_in_type_where(space.w_type,
                                              '__eq__')
     return w_eq
-type_eq._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
     w_src, w_iter = space.lookup_in_type_where(space.w_list,
                                                '__iter__')
     return w_iter
-list_iter._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
     w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
                                                '__iter__')
     return w_iter
-tuple_iter._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
     w_src, w_iter = space.lookup_in_type_where(space.w_str,
                                                '__getitem__')
     return w_iter
-str_getitem._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
     w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
                                                '__getitem__')
     return w_iter
-unicode_getitem._annspecialcase_ = 'specialize:memo'
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -7,6 +7,7 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
+from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 
@@ -344,6 +345,7 @@
                         "unsupported format character '%s' (%s) at index %d",
                         s, hex(ord(c)), self.fmtpos - 1)
 
+        @specialize.argtype(2)
         def std_wp(self, r):
             length = len(r)
             if do_unicode and isinstance(r, str):
@@ -369,7 +371,6 @@
             if padding > 0:
                 result.append_multiple_char(const(' '), padding)
             # add any remaining padding at the right
-        std_wp._annspecialcase_ = 'specialize:argtype(1)'
 
         def std_wp_number(self, r, prefix=''):
             result = self.result
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -351,6 +351,7 @@
         assert isinstance(w_starttype, W_TypeObject)
         return w_type.lookup_starting_at(w_starttype, name)
 
+    @specialize.arg(1)
     def allocate_instance(self, cls, w_subtype):
         """Allocate the memory needed for an instance of an internal or
         user-defined type, without actually __init__ializing the instance."""
@@ -375,7 +376,6 @@
                         "%N.__new__(%N): only for the type %N",
                         w_type, w_subtype, w_type)
         return instance
-    allocate_instance._annspecialcase_ = "specialize:arg(1)"
 
     # two following functions are almost identical, but in fact they
     # have different return type. First one is a resizable list, second

From pypy.commits at gmail.com  Mon Sep 26 02:13:20 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 25 Sep 2016 23:13:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: some inlining decorators (instead of
 function attributes)
Message-ID: <57e8bc80.05d71c0a.a5f86.f784@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r87385:3bdb1ace723f
Date: 2016-09-26 08:12 +0200
http://bitbucket.org/pypy/pypy/changeset/3bdb1ace723f/

Log:	some inlining decorators (instead of function attributes)

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib import jit, rgc
+from rpython.rlib import jit, rgc, objectmodel
 
 TICK_COUNTER_STEP = 100
 
@@ -131,6 +131,7 @@
         if self.gettrace() is not None:
             self._trace(frame, 'return', w_retval)
 
+    @objectmodel.always_inline
     def bytecode_trace(self, frame, decr_by=TICK_COUNTER_STEP):
         "Trace function called before each bytecode."
         # this is split into a fast path and a slower path that is
@@ -139,7 +140,6 @@
         actionflag = self.space.actionflag
         if actionflag.decrement_ticker(decr_by) < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
-    bytecode_trace._always_inline_ = True
 
     def _run_finalizers_now(self):
         # Tests only: run the actions now, to ensure that the
@@ -147,6 +147,7 @@
         # pypy.tool.pytest.apptest.
         self.space.actionflag.action_dispatcher(self, None)
 
+    @objectmodel.always_inline
     def bytecode_only_trace(self, frame):
         """
         Like bytecode_trace() but doesn't invoke any other events besides the
@@ -156,7 +157,6 @@
             self.gettrace() is None):
             return
         self.run_trace_func(frame)
-    bytecode_only_trace._always_inline_ = True
 
     @jit.unroll_safe
     def run_trace_func(self, frame):
@@ -203,13 +203,13 @@
 
         d.instr_prev_plus_one = frame.last_instr + 1
 
+    @objectmodel.try_inline
     def bytecode_trace_after_exception(self, frame):
         "Like bytecode_trace(), but without increasing the ticker."
         actionflag = self.space.actionflag
         self.bytecode_only_trace(frame)
         if actionflag.get_ticker() < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
-    bytecode_trace_after_exception._always_inline_ = 'try'
     # NB. this function is not inlined right now.  backendopt.inline would
     # need some improvements to handle this case, but it's not really an
     # issue
@@ -456,6 +456,7 @@
         periodic_actions = unrolling_iterable(self._periodic_actions)
 
         @jit.unroll_safe
+        @objectmodel.dont_inline
         def action_dispatcher(ec, frame):
             # periodic actions (first reset the bytecode counter)
             self.reset_ticker(self.checkinterval_scaled)
@@ -477,7 +478,6 @@
                     action._fired = False
                     action.perform(ec, frame)
 
-        action_dispatcher._dont_inline_ = True
         self.action_dispatcher = action_dispatcher
 
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -6,7 +6,8 @@
 
 from rpython.rlib import jit, rstackovf
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import (we_are_translated, always_inline,
+        dont_inline)
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -483,20 +484,20 @@
         # of oparg failed to produce an integer which is annotated as non-neg
         check_nonneg(oparg)
 
+    @always_inline
     def LOAD_FAST(self, varindex, next_instr):
         # access a local variable directly
         w_value = self.locals_cells_stack_w[varindex]
         if w_value is None:
             self._load_fast_failed(varindex)
         self.pushvalue(w_value)
-    LOAD_FAST._always_inline_ = True
 
+    @dont_inline
     def _load_fast_failed(self, varindex):
         varname = self.getlocalvarname(varindex)
         raise oefmt(self.space.w_UnboundLocalError,
                     "local variable '%s' referenced before assignment",
                     varname)
-    _load_fast_failed._dont_inline_ = True
 
     def LOAD_CONST(self, constindex, next_instr):
         w_const = self.getconstant_w(constindex)
@@ -888,6 +889,7 @@
                 return
         self.LOAD_GLOBAL(nameindex, next_instr)    # fall-back
 
+    @always_inline
     def _load_global(self, varname):
         w_value = self.space.finditem_str(self.get_w_globals(), varname)
         if w_value is None:
@@ -896,16 +898,15 @@
             if w_value is None:
                 self._load_global_failed(varname)
         return w_value
-    _load_global._always_inline_ = True
 
+    @dont_inline
     def _load_global_failed(self, varname):
         raise oefmt(self.space.w_NameError,
                     "global name '%s' is not defined", varname)
-    _load_global_failed._dont_inline_ = True
 
+    @always_inline
     def LOAD_GLOBAL(self, nameindex, next_instr):
         self.pushvalue(self._load_global(self.getname_u(nameindex)))
-    LOAD_GLOBAL._always_inline_ = True
 
     def DELETE_FAST(self, varindex, next_instr):
         if self.locals_cells_stack_w[varindex] is None:
@@ -939,6 +940,7 @@
         self.pushvalue(space.newlist([], sizehint=length_hint))
         self.pushvalue(last_val)
 
+    @always_inline
     def LOAD_ATTR(self, nameindex, next_instr):
         "obj.attributename"
         w_obj = self.popvalue()
@@ -949,7 +951,6 @@
             w_attributename = self.getname_w(nameindex)
             w_value = self.space.getattr(w_obj, w_attributename)
         self.pushvalue(w_value)
-    LOAD_ATTR._always_inline_ = True
 
     @jit.unroll_safe
     def cmp_exc_match(self, w_1, w_2):
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec
@@ -25,12 +26,12 @@
     def iter_w(self):
         return self.space.wrap(self)
 
+    @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
         w_error = space.getattr(w_module, space.wrap('Error'))
         raise oefmt(w_error, "line %d: %s", self.line_num, msg)
-    error._dont_inline_ = True
 
     def add_char(self, field_builder, c):
         assert field_builder is not None
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.typedef import TypeDef, interp2app
@@ -21,12 +22,12 @@
             special += dialect.quotechar
         self.special_characters = special
 
+    @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
         w_error = space.getattr(w_module, space.wrap('Error'))
         raise OperationError(w_error, space.wrap(msg))
-    error._dont_inline_ = True
 
     def writerow(self, w_fields):
         """Construct and write a CSV record from a sequence of fields.
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -7,7 +7,7 @@
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
                                       interp_attrproperty)
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, always_inline
 from rpython.rlib.rtimer import read_timestamp, _is_64_bit
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -256,7 +256,7 @@
         return w_frame.wrap_string(space)
     return w_frame    # actually a PyCode object
 
-
+ at always_inline
 def prepare_spec(space, w_arg):
     if isinstance(w_arg, Method):
         return (w_arg.w_function, w_arg.w_class)
@@ -264,8 +264,6 @@
         return (w_arg, None)
     else:
         return (None, space.type(w_arg))
-prepare_spec._always_inline_ = True
-
 
 def lsprof_call(space, w_self, frame, event, w_arg):
     assert isinstance(w_self, W_Profiler)
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1,6 +1,6 @@
 import sys
 from rpython.rlib.rstring import StringBuilder
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, always_inline
 from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import oefmt
@@ -188,6 +188,7 @@
         self.pos = i
         return self.space.call_function(self.space.w_int, self.space.wrap(s))
 
+    @always_inline
     def parse_integer(self, i):
         "Parse a decimal number with an optional minus sign"
         sign = 1
@@ -218,7 +219,6 @@
         # overflowed
         ovf_maybe = (count >= OVF_DIGITS)
         return i, ovf_maybe, sign * intval
-    parse_integer._always_inline_ = True
 
     def decode_array(self, i):
         w_list = self.space.newlist([])
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -6,7 +6,7 @@
 import time
 from pypy.interpreter.error import OperationError
 from pypy.module._pypyjson.interp_decoder import loads
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, dont_inline
 
 
 ## MSG = open('msg.json').read()
@@ -69,11 +69,11 @@
         assert isinstance(w_x, W_String)
         return w_x.strval
 
+    @dont_inline
     def call_method(self, obj, name, arg):
         assert name == 'append'
         assert isinstance(obj, W_List)
         obj.listval.append(arg)
-    call_method._dont_inline_ = True
 
     def call_function(self, w_func, *args_w):
         return self.w_None # XXX
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from rpython.rlib import rmmap, rarithmetic
+from rpython.rlib import rmmap, rarithmetic, objectmodel
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
@@ -316,6 +316,7 @@
         self.w_error = space.new_exception_class("mmap.error",
                                                  space.w_EnvironmentError)
 
+ at objectmodel.dont_inline
 def mmap_error(space, e):
     if isinstance(e, RValueError):
         return OperationError(space.w_ValueError, space.wrap(e.message))
diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py
--- a/pypy/module/select/interp_select.py
+++ b/pypy/module/select/interp_select.py
@@ -2,6 +2,7 @@
 
 from rpython.rlib import _rsocket_rffi as _c, rpoll
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib import objectmodel
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
@@ -91,7 +92,7 @@
 
 # ____________________________________________________________
 
-
+ at objectmodel.always_inline  # get rid of the tuple result
 def _build_fd_set(space, list_w, ll_list, nfds):
     _c.FD_ZERO(ll_list)
     fdlist = []
@@ -105,7 +106,6 @@
         _c.FD_SET(fd, ll_list)
         fdlist.append(fd)
     return fdlist, nfds
-_build_fd_set._always_inline_ = True    # get rid of the tuple result
 
 
 def _unbuild_fd_set(space, list_w, fdlist, ll_list, reslist_w):

From pypy.commits at gmail.com  Mon Sep 26 02:50:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 25 Sep 2016 23:50:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix test
Message-ID: <57e8c537.079bc20a.b67a4.cdbc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87386:7472cf9b0cd7
Date: 2016-09-26 08:49 +0200
http://bitbucket.org/pypy/pypy/changeset/7472cf9b0cd7/

Log:	Fix test

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1030,8 +1030,8 @@
             # If we are running PyPy with a libpypy-c, the following
             # lines find the stdlib anyway.  Otherwise, it is not found.
             expected_found = (
-                '__pypy__' in sys.builtin_module_names and
-                sys.pypy_translation_info['translation.shared'])
+                getattr(sys, 'pypy_translation_info', {})
+                .get('translation.shared'))
 
             import app_main
             app_main.setup_bootstrap_path(tmp_pypy_c)

From pypy.commits at gmail.com  Mon Sep 26 03:21:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 00:21:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: If libpypy-c.so is a symlink,
 look first around the non-deferred path to
Message-ID: <57e8cc85.c22e1c0a.fa424.09f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87387:d1382b9d843b
Date: 2016-09-26 09:01 +0200
http://bitbucket.org/pypy/pypy/changeset/d1382b9d843b/

Log:	If libpypy-c.so is a symlink, look first around the non-deferred
	path to libpypy-c.so, and then look around where the symlink points
	to. Should fix virtualenv (e.g. seen in
	module.test_lib_pypy.cffi_tests.cffi0.test_zintegration)

diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -157,6 +157,22 @@
     return space.wrap(resolvedirof(filename))
 
 
+def find_stdlib_link(state, executable):
+    path, prefix = find_stdlib(state, executable)
+    if path is None:
+        if os.path.islink(executable):
+            # If we can't find the stdlib from the given 'executable',
+            # which is usually the path of libpypy-c.so, then we try
+            # to look if this libpypy-c.so is a symlink and follow that.
+            try:
+                link = _readlink_maybe(executable)
+            except OSError:
+                pass
+            else:
+                return find_stdlib_link(state, link)
+    return path, prefix
+
+
 @unwrap_spec(executable='str0', dynamic=int)
 def pypy_find_stdlib(space, executable, dynamic=1):
     if dynamic and space.config.translation.shared:
@@ -164,7 +180,7 @@
         if dynamic_location:
             executable = rffi.charp2str(dynamic_location)
             pypy_init_free(dynamic_location)
-    path, prefix = find_stdlib(get_state(space), executable)
+    path, prefix = find_stdlib_link(get_state(space), executable)
     if path is None:
         return space.w_None
     w_prefix = space.wrap(prefix)
@@ -229,11 +245,7 @@
                 dlerror());
         return NULL;
     }
-    char *p = realpath(info.dli_fname, NULL);
-    if (p == NULL) {
-        p = strdup(info.dli_fname);
-    }
-    return p;
+    return strdup(info.dli_fname);
 }
 """
 

From pypy.commits at gmail.com  Mon Sep 26 03:21:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 00:21:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fixes
Message-ID: <57e8cc87.85b81c0a.59021.1138@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87388:1057a7b1ad5f
Date: 2016-09-26 09:21 +0200
http://bitbucket.org/pypy/pypy/changeset/1057a7b1ad5f/

Log:	Fixes

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -328,7 +328,6 @@
     else:
         # bogus 'e'?
         return OperationError(space.w_SystemError, space.wrap('%s' % e))
-mmap_error._dont_inline_ = True
 
 
 class MMapBuffer(Buffer):
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -345,7 +345,7 @@
                         "unsupported format character '%s' (%s) at index %d",
                         s, hex(ord(c)), self.fmtpos - 1)
 
-        @specialize.argtype(2)
+        @specialize.argtype(1)
         def std_wp(self, r):
             length = len(r)
             if do_unicode and isinstance(r, str):

From pypy.commits at gmail.com  Mon Sep 26 04:05:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 01:05:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: backout d1382b9d843b: doesn't work,
 dladdr() doesn't return the symlink's
Message-ID: <57e8d6da.e50fc20a.b7e47.ed79@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87389:42797eb41b78
Date: 2016-09-26 10:05 +0200
http://bitbucket.org/pypy/pypy/changeset/42797eb41b78/

Log:	backout d1382b9d843b: doesn't work, dladdr() doesn't return the
	symlink's path anyway

diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -157,22 +157,6 @@
     return space.wrap(resolvedirof(filename))
 
 
-def find_stdlib_link(state, executable):
-    path, prefix = find_stdlib(state, executable)
-    if path is None:
-        if os.path.islink(executable):
-            # If we can't find the stdlib from the given 'executable',
-            # which is usually the path of libpypy-c.so, then we try
-            # to look if this libpypy-c.so is a symlink and follow that.
-            try:
-                link = _readlink_maybe(executable)
-            except OSError:
-                pass
-            else:
-                return find_stdlib_link(state, link)
-    return path, prefix
-
-
 @unwrap_spec(executable='str0', dynamic=int)
 def pypy_find_stdlib(space, executable, dynamic=1):
     if dynamic and space.config.translation.shared:
@@ -180,7 +164,7 @@
         if dynamic_location:
             executable = rffi.charp2str(dynamic_location)
             pypy_init_free(dynamic_location)
-    path, prefix = find_stdlib_link(get_state(space), executable)
+    path, prefix = find_stdlib(get_state(space), executable)
     if path is None:
         return space.w_None
     w_prefix = space.wrap(prefix)
@@ -245,7 +229,11 @@
                 dlerror());
         return NULL;
     }
-    return strdup(info.dli_fname);
+    char *p = realpath(info.dli_fname, NULL);
+    if (p == NULL) {
+        p = strdup(info.dli_fname);
+    }
+    return p;
 }
 """
 

From pypy.commits at gmail.com  Mon Sep 26 05:15:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 02:15:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Search first around the pypy executable
 (like previously), 
Message-ID: <57e8e720.657bc20a.1a3a4.0193@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87390:0e3b02bb98e5
Date: 2016-09-26 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/0e3b02bb98e5/

Log:	Search first around the pypy executable (like previously), and only
	when not found search around the libpypy-c.so

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -93,12 +93,10 @@
             home1 = rffi.charp2str(ll_home)
             home = os.path.join(home1, 'x') # <- so that 'll_home' can be
                                             # directly the root directory
-            dynamic = False
         else:
             home1 = "pypy's shared library location"
-            home = pypydir
-            dynamic = True
-        w_path = pypy_find_stdlib(space, home, dynamic)
+            home = '*'
+        w_path = pypy_find_stdlib(space, home)
         if space.is_none(w_path):
             if verbose:
                 debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -157,16 +157,20 @@
     return space.wrap(resolvedirof(filename))
 
 
- at unwrap_spec(executable='str0', dynamic=int)
-def pypy_find_stdlib(space, executable, dynamic=1):
-    if dynamic and space.config.translation.shared:
-        dynamic_location = pypy_init_home()
-        if dynamic_location:
-            executable = rffi.charp2str(dynamic_location)
-            pypy_init_free(dynamic_location)
-    path, prefix = find_stdlib(get_state(space), executable)
+ at unwrap_spec(executable='str0')
+def pypy_find_stdlib(space, executable):
+    path, prefix = None, None
+    if executable != '*':
+        path, prefix = find_stdlib(get_state(space), executable)
     if path is None:
-        return space.w_None
+        if space.config.translation.shared:
+            dynamic_location = pypy_init_home()
+            if dynamic_location:
+                dyn_path = rffi.charp2str(dynamic_location)
+                pypy_init_free(dynamic_location)
+                path, prefix = find_stdlib(get_state(space), dyn_path)
+        if path is None:
+            return space.w_None
     w_prefix = space.wrap(prefix)
     space.setitem(space.sys.w_dict, space.wrap('prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), w_prefix)

From pypy.commits at gmail.com  Mon Sep 26 05:15:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 02:15:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: Aargh,
 missing header meant that the "char *" was truncated to 32 bits
Message-ID: <57e8e722.a970c20a.6630c.0483@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87391:4e94cb209019
Date: 2016-09-26 10:24 +0100
http://bitbucket.org/pypy/pypy/changeset/4e94cb209019/

Log:	Aargh, missing header meant that the "char *" was truncated to 32
	bits (and *usually* it would fit in 32 bits, but of course, not
	always)

diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -241,7 +241,8 @@
 }
 """
 
-_eci = ExternalCompilationInfo(separate_module_sources=[_source_code])
+_eci = ExternalCompilationInfo(separate_module_sources=[_source_code],
+    post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);'])
 
 pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP,
                                  _nowrapper=True, compilation_info=_eci)

From pypy.commits at gmail.com  Mon Sep 26 05:44:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 02:44:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Aaargh again. Borked the locking here.
 Shows up as a rare crash
Message-ID: <57e8ee1b.02c11c0a.5b222.4829@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87392:c540b2c9656a
Date: 2016-09-26 10:53 +0100
http://bitbucket.org/pypy/pypy/changeset/c540b2c9656a/

Log:	Aaargh again. Borked the locking here. Shows up as a rare crash in
	test_lib_pypy/cffi_tests/embedding/test_thread.

diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -15,9 +15,10 @@
 static int check_valid(void);
 
 void _RPython_ThreadLocals_Acquire(void) {
-    while (!pypy_lock_test_and_set(&pypy_threadlocal_lock, 1)) {
-        /* busy loop */
-    }
+    long old_value;
+    do {
+        old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1);
+    } while (old_value != 0);   /* busy loop */
     assert(check_valid());
 }
 void _RPython_ThreadLocals_Release(void) {

From pypy.commits at gmail.com  Mon Sep 26 06:44:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 26 Sep 2016 03:44:49 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: move compiler-workshop slides into
 scipyUS2016 dir
Message-ID: <57e8fc21.d26b1c0a.13d7e.7290@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5714:3b81824d8ce8
Date: 2016-09-26 12:21 +0200
http://bitbucket.org/pypy/extradoc/changeset/3b81824d8ce8/

Log:	move compiler-workshop slides into scipyUS2016 dir

diff --git a/talk/compiler-workshop-2016/pypy.rst b/talk/scipyUS2016/compiler-workshop-2016/pypy.rst
rename from talk/compiler-workshop-2016/pypy.rst
rename to talk/scipyUS2016/compiler-workshop-2016/pypy.rst
diff --git a/talk/compiler-workshop-2016/slides/CONTRIBUTING.md b/talk/scipyUS2016/compiler-workshop-2016/slides/CONTRIBUTING.md
rename from talk/compiler-workshop-2016/slides/CONTRIBUTING.md
rename to talk/scipyUS2016/compiler-workshop-2016/slides/CONTRIBUTING.md
diff --git a/talk/compiler-workshop-2016/slides/Gruntfile.js b/talk/scipyUS2016/compiler-workshop-2016/slides/Gruntfile.js
rename from talk/compiler-workshop-2016/slides/Gruntfile.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/Gruntfile.js
diff --git a/talk/compiler-workshop-2016/slides/LICENSE b/talk/scipyUS2016/compiler-workshop-2016/slides/LICENSE
rename from talk/compiler-workshop-2016/slides/LICENSE
rename to talk/scipyUS2016/compiler-workshop-2016/slides/LICENSE
diff --git a/talk/compiler-workshop-2016/slides/README.md b/talk/scipyUS2016/compiler-workshop-2016/slides/README.md
rename from talk/compiler-workshop-2016/slides/README.md
rename to talk/scipyUS2016/compiler-workshop-2016/slides/README.md
diff --git a/talk/compiler-workshop-2016/slides/bower.json b/talk/scipyUS2016/compiler-workshop-2016/slides/bower.json
rename from talk/compiler-workshop-2016/slides/bower.json
rename to talk/scipyUS2016/compiler-workshop-2016/slides/bower.json
diff --git a/talk/compiler-workshop-2016/slides/css/print/paper.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/print/paper.css
rename from talk/compiler-workshop-2016/slides/css/print/paper.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/print/paper.css
diff --git a/talk/compiler-workshop-2016/slides/css/print/pdf.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/print/pdf.css
rename from talk/compiler-workshop-2016/slides/css/print/pdf.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/print/pdf.css
diff --git a/talk/compiler-workshop-2016/slides/css/reveal.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/reveal.css
rename from talk/compiler-workshop-2016/slides/css/reveal.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/reveal.css
diff --git a/talk/compiler-workshop-2016/slides/css/reveal.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/reveal.scss
rename from talk/compiler-workshop-2016/slides/css/reveal.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/reveal.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/README.md b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/README.md
rename from talk/compiler-workshop-2016/slides/css/theme/README.md
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/README.md
diff --git a/talk/compiler-workshop-2016/slides/css/theme/beige.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/beige.css
rename from talk/compiler-workshop-2016/slides/css/theme/beige.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/beige.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/black.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/black.css
rename from talk/compiler-workshop-2016/slides/css/theme/black.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/black.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/blood.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/blood.css
rename from talk/compiler-workshop-2016/slides/css/theme/blood.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/blood.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/league.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/league.css
rename from talk/compiler-workshop-2016/slides/css/theme/league.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/league.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/moon.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/moon.css
rename from talk/compiler-workshop-2016/slides/css/theme/moon.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/moon.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/night.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/night.css
rename from talk/compiler-workshop-2016/slides/css/theme/night.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/night.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/serif.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/serif.css
rename from talk/compiler-workshop-2016/slides/css/theme/serif.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/serif.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/simple.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/simple.css
rename from talk/compiler-workshop-2016/slides/css/theme/simple.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/simple.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/sky.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/sky.css
rename from talk/compiler-workshop-2016/slides/css/theme/sky.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/sky.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/solarized.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/solarized.css
rename from talk/compiler-workshop-2016/slides/css/theme/solarized.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/solarized.css
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/beige.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/beige.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/beige.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/beige.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/black.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/black.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/black.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/black.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/blood.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/blood.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/blood.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/blood.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/league.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/league.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/league.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/league.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/moon.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/moon.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/moon.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/moon.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/night.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/night.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/night.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/night.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/serif.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/serif.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/serif.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/serif.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/simple.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/simple.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/simple.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/simple.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/sky.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/sky.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/sky.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/sky.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/solarized.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/solarized.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/solarized.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/solarized.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/source/white.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/white.scss
rename from talk/compiler-workshop-2016/slides/css/theme/source/white.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/source/white.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/template/mixins.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/template/mixins.scss
rename from talk/compiler-workshop-2016/slides/css/theme/template/mixins.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/template/mixins.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/template/settings.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/template/settings.scss
rename from talk/compiler-workshop-2016/slides/css/theme/template/settings.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/template/settings.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/template/theme.scss b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/template/theme.scss
rename from talk/compiler-workshop-2016/slides/css/theme/template/theme.scss
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/template/theme.scss
diff --git a/talk/compiler-workshop-2016/slides/css/theme/white.css b/talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/white.css
rename from talk/compiler-workshop-2016/slides/css/theme/white.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/css/theme/white.css
diff --git a/talk/compiler-workshop-2016/slides/img/pypylogo.png b/talk/scipyUS2016/compiler-workshop-2016/slides/img/pypylogo.png
rename from talk/compiler-workshop-2016/slides/img/pypylogo.png
rename to talk/scipyUS2016/compiler-workshop-2016/slides/img/pypylogo.png
diff --git a/talk/compiler-workshop-2016/slides/index.html b/talk/scipyUS2016/compiler-workshop-2016/slides/index.html
rename from talk/compiler-workshop-2016/slides/index.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/index.html
diff --git a/talk/compiler-workshop-2016/slides/js/reveal.js b/talk/scipyUS2016/compiler-workshop-2016/slides/js/reveal.js
rename from talk/compiler-workshop-2016/slides/js/reveal.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/js/reveal.js
diff --git a/talk/compiler-workshop-2016/slides/lib/css/zenburn.css b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/css/zenburn.css
rename from talk/compiler-workshop-2016/slides/lib/css/zenburn.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/css/zenburn.css
diff --git a/talk/compiler-workshop-2016/slides/lib/font/league-gothic/LICENSE b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/LICENSE
rename from talk/compiler-workshop-2016/slides/lib/font/league-gothic/LICENSE
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/LICENSE
diff --git a/talk/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.css b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.css
rename from talk/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.css
diff --git a/talk/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.eot b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.eot
rename from talk/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.eot
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.eot
diff --git a/talk/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.ttf b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.ttf
rename from talk/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.ttf
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.ttf
diff --git a/talk/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.woff b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.woff
rename from talk/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.woff
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/league-gothic/league-gothic.woff
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/LICENSE b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/LICENSE
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/LICENSE
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/LICENSE
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.eot b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.eot
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.eot
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.eot
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.ttf b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.ttf
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.ttf
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.ttf
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.woff b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.woff
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.woff
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-italic.woff
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.eot b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.eot
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.eot
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.eot
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.ttf b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.ttf
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.ttf
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.ttf
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.woff b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.woff
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.woff
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-regular.woff
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.eot b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.eot
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.eot
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.eot
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.ttf b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.ttf
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.ttf
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.ttf
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.woff b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.woff
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.woff
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibold.woff
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.eot b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.eot
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.eot
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.eot
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.ttf b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.ttf
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.ttf
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.ttf
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.woff b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.woff
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.woff
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro-semibolditalic.woff
diff --git a/talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro.css b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro.css
rename from talk/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/font/source-sans-pro/source-sans-pro.css
diff --git a/talk/compiler-workshop-2016/slides/lib/js/classList.js b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/js/classList.js
rename from talk/compiler-workshop-2016/slides/lib/js/classList.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/js/classList.js
diff --git a/talk/compiler-workshop-2016/slides/lib/js/head.min.js b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/js/head.min.js
rename from talk/compiler-workshop-2016/slides/lib/js/head.min.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/js/head.min.js
diff --git a/talk/compiler-workshop-2016/slides/lib/js/html5shiv.js b/talk/scipyUS2016/compiler-workshop-2016/slides/lib/js/html5shiv.js
rename from talk/compiler-workshop-2016/slides/lib/js/html5shiv.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/lib/js/html5shiv.js
diff --git a/talk/compiler-workshop-2016/slides/package.json b/talk/scipyUS2016/compiler-workshop-2016/slides/package.json
rename from talk/compiler-workshop-2016/slides/package.json
rename to talk/scipyUS2016/compiler-workshop-2016/slides/package.json
diff --git a/talk/compiler-workshop-2016/slides/plugin/highlight/highlight.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/highlight/highlight.js
rename from talk/compiler-workshop-2016/slides/plugin/highlight/highlight.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/highlight/highlight.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/markdown/example.html b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/markdown/example.html
rename from talk/compiler-workshop-2016/slides/plugin/markdown/example.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/markdown/example.html
diff --git a/talk/compiler-workshop-2016/slides/plugin/markdown/example.md b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/markdown/example.md
rename from talk/compiler-workshop-2016/slides/plugin/markdown/example.md
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/markdown/example.md
diff --git a/talk/compiler-workshop-2016/slides/plugin/markdown/markdown.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/markdown/markdown.js
rename from talk/compiler-workshop-2016/slides/plugin/markdown/markdown.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/markdown/markdown.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/markdown/marked.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/markdown/marked.js
rename from talk/compiler-workshop-2016/slides/plugin/markdown/marked.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/markdown/marked.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/math/math.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/math/math.js
rename from talk/compiler-workshop-2016/slides/plugin/math/math.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/math/math.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/multiplex/client.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/multiplex/client.js
rename from talk/compiler-workshop-2016/slides/plugin/multiplex/client.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/multiplex/client.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/multiplex/index.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/multiplex/index.js
rename from talk/compiler-workshop-2016/slides/plugin/multiplex/index.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/multiplex/index.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/multiplex/master.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/multiplex/master.js
rename from talk/compiler-workshop-2016/slides/plugin/multiplex/master.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/multiplex/master.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/multiplex/package.json b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/multiplex/package.json
rename from talk/compiler-workshop-2016/slides/plugin/multiplex/package.json
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/multiplex/package.json
diff --git a/talk/compiler-workshop-2016/slides/plugin/notes-server/client.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes-server/client.js
rename from talk/compiler-workshop-2016/slides/plugin/notes-server/client.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes-server/client.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/notes-server/index.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes-server/index.js
rename from talk/compiler-workshop-2016/slides/plugin/notes-server/index.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes-server/index.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/notes-server/notes.html b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes-server/notes.html
rename from talk/compiler-workshop-2016/slides/plugin/notes-server/notes.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes-server/notes.html
diff --git a/talk/compiler-workshop-2016/slides/plugin/notes/notes.html b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes/notes.html
rename from talk/compiler-workshop-2016/slides/plugin/notes/notes.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes/notes.html
diff --git a/talk/compiler-workshop-2016/slides/plugin/notes/notes.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes/notes.js
rename from talk/compiler-workshop-2016/slides/plugin/notes/notes.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/notes/notes.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/print-pdf/print-pdf.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/print-pdf/print-pdf.js
rename from talk/compiler-workshop-2016/slides/plugin/print-pdf/print-pdf.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/print-pdf/print-pdf.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/search/search.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/search/search.js
rename from talk/compiler-workshop-2016/slides/plugin/search/search.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/search/search.js
diff --git a/talk/compiler-workshop-2016/slides/plugin/zoom-js/zoom.js b/talk/scipyUS2016/compiler-workshop-2016/slides/plugin/zoom-js/zoom.js
rename from talk/compiler-workshop-2016/slides/plugin/zoom-js/zoom.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/plugin/zoom-js/zoom.js
diff --git a/talk/compiler-workshop-2016/slides/reveal.js-3.3.0/.gitignore b/talk/scipyUS2016/compiler-workshop-2016/slides/reveal.js-3.3.0/.gitignore
rename from talk/compiler-workshop-2016/slides/reveal.js-3.3.0/.gitignore
rename to talk/scipyUS2016/compiler-workshop-2016/slides/reveal.js-3.3.0/.gitignore
diff --git a/talk/compiler-workshop-2016/slides/reveal.js-3.3.0/.travis.yml b/talk/scipyUS2016/compiler-workshop-2016/slides/reveal.js-3.3.0/.travis.yml
rename from talk/compiler-workshop-2016/slides/reveal.js-3.3.0/.travis.yml
rename to talk/scipyUS2016/compiler-workshop-2016/slides/reveal.js-3.3.0/.travis.yml
diff --git a/talk/compiler-workshop-2016/slides/test/examples/assets/image1.png b/talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/assets/image1.png
rename from talk/compiler-workshop-2016/slides/test/examples/assets/image1.png
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/assets/image1.png
diff --git a/talk/compiler-workshop-2016/slides/test/examples/assets/image2.png b/talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/assets/image2.png
rename from talk/compiler-workshop-2016/slides/test/examples/assets/image2.png
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/assets/image2.png
diff --git a/talk/compiler-workshop-2016/slides/test/examples/barebones.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/barebones.html
rename from talk/compiler-workshop-2016/slides/test/examples/barebones.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/barebones.html
diff --git a/talk/compiler-workshop-2016/slides/test/examples/embedded-media.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/embedded-media.html
rename from talk/compiler-workshop-2016/slides/test/examples/embedded-media.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/embedded-media.html
diff --git a/talk/compiler-workshop-2016/slides/test/examples/math.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/math.html
rename from talk/compiler-workshop-2016/slides/test/examples/math.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/math.html
diff --git a/talk/compiler-workshop-2016/slides/test/examples/slide-backgrounds.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/slide-backgrounds.html
rename from talk/compiler-workshop-2016/slides/test/examples/slide-backgrounds.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/slide-backgrounds.html
diff --git a/talk/compiler-workshop-2016/slides/test/examples/slide-transitions.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/slide-transitions.html
rename from talk/compiler-workshop-2016/slides/test/examples/slide-transitions.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/examples/slide-transitions.html
diff --git a/talk/compiler-workshop-2016/slides/test/qunit-1.12.0.css b/talk/scipyUS2016/compiler-workshop-2016/slides/test/qunit-1.12.0.css
rename from talk/compiler-workshop-2016/slides/test/qunit-1.12.0.css
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/qunit-1.12.0.css
diff --git a/talk/compiler-workshop-2016/slides/test/qunit-1.12.0.js b/talk/scipyUS2016/compiler-workshop-2016/slides/test/qunit-1.12.0.js
rename from talk/compiler-workshop-2016/slides/test/qunit-1.12.0.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/qunit-1.12.0.js
diff --git a/talk/compiler-workshop-2016/slides/test/test-markdown-element-attributes.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown-element-attributes.html
rename from talk/compiler-workshop-2016/slides/test/test-markdown-element-attributes.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown-element-attributes.html
diff --git a/talk/compiler-workshop-2016/slides/test/test-markdown-element-attributes.js b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown-element-attributes.js
rename from talk/compiler-workshop-2016/slides/test/test-markdown-element-attributes.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown-element-attributes.js
diff --git a/talk/compiler-workshop-2016/slides/test/test-markdown-slide-attributes.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown-slide-attributes.html
rename from talk/compiler-workshop-2016/slides/test/test-markdown-slide-attributes.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown-slide-attributes.html
diff --git a/talk/compiler-workshop-2016/slides/test/test-markdown-slide-attributes.js b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown-slide-attributes.js
rename from talk/compiler-workshop-2016/slides/test/test-markdown-slide-attributes.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown-slide-attributes.js
diff --git a/talk/compiler-workshop-2016/slides/test/test-markdown.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown.html
rename from talk/compiler-workshop-2016/slides/test/test-markdown.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown.html
diff --git a/talk/compiler-workshop-2016/slides/test/test-markdown.js b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown.js
rename from talk/compiler-workshop-2016/slides/test/test-markdown.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test-markdown.js
diff --git a/talk/compiler-workshop-2016/slides/test/test-pdf.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test-pdf.html
rename from talk/compiler-workshop-2016/slides/test/test-pdf.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test-pdf.html
diff --git a/talk/compiler-workshop-2016/slides/test/test-pdf.js b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test-pdf.js
rename from talk/compiler-workshop-2016/slides/test/test-pdf.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test-pdf.js
diff --git a/talk/compiler-workshop-2016/slides/test/test.html b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test.html
rename from talk/compiler-workshop-2016/slides/test/test.html
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test.html
diff --git a/talk/compiler-workshop-2016/slides/test/test.js b/talk/scipyUS2016/compiler-workshop-2016/slides/test/test.js
rename from talk/compiler-workshop-2016/slides/test/test.js
rename to talk/scipyUS2016/compiler-workshop-2016/slides/test/test.js

From pypy.commits at gmail.com  Mon Sep 26 06:44:52 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 26 Sep 2016 03:44:52 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: create new empty slide deck for my
 presentation in pyconza, rmeove test files for reveal.js
Message-ID: <57e8fc24.a2ddc20a.4209c.2c7a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5715:61d8bc530462
Date: 2016-09-26 12:41 +0200
http://bitbucket.org/pypy/extradoc/changeset/61d8bc530462/

Log:	create new empty slide deck for my presentation in pyconza, rmeove
	test files for reveal.js

diff too long, truncating to 2000 out of 20819 lines

diff --git a/talk/pyconza2016/pypy/CONTRIBUTING.md b/talk/pyconza2016/pypy/CONTRIBUTING.md
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy/CONTRIBUTING.md
@@ -0,0 +1,23 @@
+## Contributing
+
+Please keep the [issue tracker](http://github.com/hakimel/reveal.js/issues) limited to **bug reports**, **feature requests** and **pull requests**.
+
+
+### Personal Support
+If you have personal support or setup questions the best place to ask those are [StackOverflow](http://stackoverflow.com/questions/tagged/reveal.js).
+
+
+### Bug Reports
+When reporting a bug make sure to include information about which browser and operating system you are on as well as the necessary steps to reproduce the issue. If possible please include a link to a sample presentation where the bug can be tested.
+
+
+### Pull Requests
+- Should follow the coding style of the file you work in, most importantly:
+  - Tabs to indent
+  - Single-quoted strings
+- Should be made towards the **dev branch**
+- Should be submitted from a feature/topic branch (not your master)
+
+
+### Plugins
+Please do not submit plugins as pull requests. They should be maintained in their own separate repository. More information here: https://github.com/hakimel/reveal.js/wiki/Plugin-Guidelines
diff --git a/talk/pyconza2016/pypy/Gruntfile.js b/talk/pyconza2016/pypy/Gruntfile.js
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy/Gruntfile.js
@@ -0,0 +1,176 @@
+/* global module:false */
+module.exports = function(grunt) {
+	var port = grunt.option('port') || 8000;
+	var base = grunt.option('base') || '.';
+
+	// Project configuration
+	grunt.initConfig({
+		pkg: grunt.file.readJSON('package.json'),
+		meta: {
+			banner:
+				'/*!\n' +
+				' * reveal.js <%= pkg.version %> (<%= grunt.template.today("yyyy-mm-dd, HH:MM") %>)\n' +
+				' * http://lab.hakim.se/reveal-js\n' +
+				' * MIT licensed\n' +
+				' *\n' +
+				' * Copyright (C) 2016 Hakim El Hattab, http://hakim.se\n' +
+				' */'
+		},
+
+		qunit: {
+			files: [ 'test/*.html' ]
+		},
+
+		uglify: {
+			options: {
+				banner: '<%= meta.banner %>\n'
+			},
+			build: {
+				src: 'js/reveal.js',
+				dest: 'js/reveal.min.js'
+			}
+		},
+
+		sass: {
+			core: {
+				files: {
+					'css/reveal.css': 'css/reveal.scss',
+				}
+			},
+			themes: {
+				files: [
+					{
+						expand: true,
+						cwd: 'css/theme/source',
+						src: ['*.scss'],
+						dest: 'css/theme',
+						ext: '.css'
+					}
+				]
+			}
+		},
+
+		autoprefixer: {
+			dist: {
+				src: 'css/reveal.css'
+			}
+		},
+
+		cssmin: {
+			compress: {
+				files: {
+					'css/reveal.min.css': [ 'css/reveal.css' ]
+				}
+			}
+		},
+
+		jshint: {
+			options: {
+				curly: false,
+				eqeqeq: true,
+				immed: true,
+				latedef: true,
+				newcap: true,
+				noarg: true,
+				sub: true,
+				undef: true,
+				eqnull: true,
+				browser: true,
+				expr: true,
+				globals: {
+					head: false,
+					module: false,
+					console: false,
+					unescape: false,
+					define: false,
+					exports: false
+				}
+			},
+			files: [ 'Gruntfile.js', 'js/reveal.js' ]
+		},
+
+		connect: {
+			server: {
+				options: {
+					port: port,
+					base: base,
+					livereload: true,
+					open: true
+				}
+			}
+		},
+
+		zip: {
+			'reveal-js-presentation.zip': [
+				'index.html',
+				'css/**',
+				'js/**',
+				'lib/**',
+				'images/**',
+				'plugin/**',
+				'**.md'
+			]
+		},
+
+		watch: {
+			js: {
+				files: [ 'Gruntfile.js', 'js/reveal.js' ],
+				tasks: 'js'
+			},
+			theme: {
+				files: [ 'css/theme/source/*.scss', 'css/theme/template/*.scss' ],
+				tasks: 'css-themes'
+			},
+			css: {
+				files: [ 'css/reveal.scss' ],
+				tasks: 'css-core'
+			},
+			html: {
+				files: [ '*.html']
+			},
+			markdown: {
+				files: [ '*.md' ]
+			},
+			options: {
+				livereload: true
+			}
+		}
+
+	});
+
+	// Dependencies
+	grunt.loadNpmTasks( 'grunt-contrib-qunit' );
+	grunt.loadNpmTasks( 'grunt-contrib-jshint' );
+	grunt.loadNpmTasks( 'grunt-contrib-cssmin' );
+	grunt.loadNpmTasks( 'grunt-contrib-uglify' );
+	grunt.loadNpmTasks( 'grunt-contrib-watch' );
+	grunt.loadNpmTasks( 'grunt-sass' );
+	grunt.loadNpmTasks( 'grunt-contrib-connect' );
+	grunt.loadNpmTasks( 'grunt-autoprefixer' );
+	grunt.loadNpmTasks( 'grunt-zip' );
+
+	// Default task
+	grunt.registerTask( 'default', [ 'css', 'js' ] );
+
+	// JS task
+	grunt.registerTask( 'js', [ 'jshint', 'uglify', 'qunit' ] );
+
+	// Theme CSS
+	grunt.registerTask( 'css-themes', [ 'sass:themes' ] );
+
+	// Core framework CSS
+	grunt.registerTask( 'css-core', [ 'sass:core', 'autoprefixer', 'cssmin' ] );
+
+	// All CSS
+	grunt.registerTask( 'css', [ 'sass', 'autoprefixer', 'cssmin' ] );
+
+	// Package presentation to archive
+	grunt.registerTask( 'package', [ 'default', 'zip' ] );
+
+	// Serve presentation locally
+	grunt.registerTask( 'serve', [ 'connect', 'watch' ] );
+
+	// Run tests
+	grunt.registerTask( 'test', [ 'jshint', 'qunit' ] );
+
+};
diff --git a/talk/pyconza2016/pypy/LICENSE b/talk/pyconza2016/pypy/LICENSE
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy/LICENSE
@@ -0,0 +1,19 @@
+Copyright (C) 2016 Hakim El Hattab, http://hakim.se
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
\ No newline at end of file
diff --git a/talk/pyconza2016/pypy/README.md b/talk/pyconza2016/pypy/README.md
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy/README.md
@@ -0,0 +1,1104 @@
+# reveal.js [![Build Status](https://travis-ci.org/hakimel/reveal.js.svg?branch=master)](https://travis-ci.org/hakimel/reveal.js)
+
+A framework for easily creating beautiful presentations using HTML. [Check out the live demo](http://lab.hakim.se/reveal-js/).
+
+reveal.js comes with a broad range of features including [nested slides](https://github.com/hakimel/reveal.js#markup), [Markdown contents](https://github.com/hakimel/reveal.js#markdown), [PDF export](https://github.com/hakimel/reveal.js#pdf-export), [speaker notes](https://github.com/hakimel/reveal.js#speaker-notes) and a [JavaScript API](https://github.com/hakimel/reveal.js#api). There's also a fully featured visual editor and platform for sharing reveal.js presentations at [slides.com](https://slides.com).
+
+## Table of contents
+- [Online Editor](#online-editor)
+- [Instructions](#instructions)
+  - [Markup](#markup)
+  - [Markdown](#markdown)
+  - [Element Attributes](#element-attributes)
+  - [Slide Attributes](#slide-attributes)
+- [Configuration](#configuration)
+- [Presentation Size](#presentation-size)
+- [Dependencies](#dependencies)
+- [Ready Event](#ready-event)
+- [Auto-sliding](#auto-sliding)
+- [Keyboard Bindings](#keyboard-bindings)
+- [Touch Navigation](#touch-navigation)
+- [Lazy Loading](#lazy-loading)
+- [API](#api)
+  - [Slide Changed Event](#slide-changed-event)
+  - [Presentation State](#presentation-state)
+  - [Slide States](#slide-states)
+  - [Slide Backgrounds](#slide-backgrounds)
+  - [Parallax Background](#parallax-background)
+  - [Slide Transitions](#slide-transitions)
+  - [Internal links](#internal-links)
+  - [Fragments](#fragments)
+  - [Fragment events](#fragment-events)
+  - [Code syntax highlighting](#code-syntax-highlighting)
+  - [Slide number](#slide-number)
+  - [Overview mode](#overview-mode)
+  - [Fullscreen mode](#fullscreen-mode)
+  - [Embedded media](#embedded-media)
+  - [Stretching elements](#stretching-elements)
+  - [postMessage API](#postmessage-api)
+- [PDF Export](#pdf-export)
+- [Theming](#theming)
+- [Speaker Notes](#speaker-notes)
+  - [Share and Print Speaker Notes](#share-and-print-speaker-notes)
+  - [Server Side Speaker Notes](#server-side-speaker-notes)
+- [Multiplexing](#multiplexing)
+  - [Master presentation](#master-presentation)
+  - [Client presentation](#client-presentation)
+  - [Socket.io server](#socketio-server)
+- [MathJax](#mathjax)
+- [Installation](#installation)
+  - [Basic setup](#basic-setup)
+  - [Full setup](#full-setup)
+  - [Folder Structure](#folder-structure)
+- [License](#license)
+
+#### More reading
+- [Changelog](https://github.com/hakimel/reveal.js/releases): Up-to-date version history.
+- [Examples](https://github.com/hakimel/reveal.js/wiki/Example-Presentations): Presentations created with reveal.js, add your own!
+- [Browser Support](https://github.com/hakimel/reveal.js/wiki/Browser-Support): Explanation of browser support and fallbacks.
+- [Plugins](https://github.com/hakimel/reveal.js/wiki/Plugins,-Tools-and-Hardware): A list of plugins that can be used to extend reveal.js.
+
+## Online Editor
+
+Presentations are written using HTML or Markdown but there's also an online editor for those of you who prefer a graphical interface. Give it a try at [http://slides.com](http://slides.com?ref=github).
+
+
+## Instructions
+
+### Markup
+
+Here's a barebones example of a fully working reveal.js presentation:
+```html
+<html>
+	<head>
+		<link rel="stylesheet" href="css/reveal.css">
+		<link rel="stylesheet" href="css/theme/white.css">
+	</head>
+	<body>
+		<div class="reveal">
+			<div class="slides">
+				<section>Slide 1</section>
+				<section>Slide 2</section>
+			</div>
+		</div>
+		<script src="js/reveal.js"></script>
+		<script>
+			Reveal.initialize();
+		</script>
+	</body>
+</html>
+```
+
+The presentation markup hierarchy needs to be `.reveal > .slides > section` where the `section` represents one slide and can be repeated indefinitely. If you place multiple `section` elements inside of another `section` they will be shown as vertical slides. The first of the vertical slides is the "root" of the others (at the top), and will be included in the horizontal sequence. For example:
+
+```html
+<div class="reveal">
+	<div class="slides">
+		<section>Single Horizontal Slide</section>
+		<section>
+			<section>Vertical Slide 1</section>
+			<section>Vertical Slide 2</section>
+		</section>
+	</div>
+</div>
+```
+
+### Markdown
+
+It's possible to write your slides using Markdown. To enable Markdown, add the ```data-markdown``` attribute to your ```<section>``` elements and wrap the contents in a ```<script type="text/template">``` like the example below.
+
+This is based on [data-markdown](https://gist.github.com/1343518) from [Paul Irish](https://github.com/paulirish) modified to use [marked](https://github.com/chjj/marked) to support [GitHub Flavored Markdown](https://help.github.com/articles/github-flavored-markdown). Sensitive to indentation (avoid mixing tabs and spaces) and line breaks (avoid consecutive breaks).
+
+```html
+<section data-markdown>
+	<script type="text/template">
+		## Page title
+
+		A paragraph with some text and a [link](http://hakim.se).
+	</script>
+</section>
+```
+
+#### External Markdown
+
+You can write your content as a separate file and have reveal.js load it at runtime. Note the separator arguments which determine how slides are delimited in the external file. The ```data-charset``` attribute is optional and specifies which charset to use when loading the external file.
+
+When used locally, this feature requires that reveal.js [runs from a local web server](#full-setup).
+
+```html
+<section data-markdown="example.md"  
+         data-separator="^\n\n\n"  
+         data-separator-vertical="^\n\n"  
+         data-separator-notes="^Note:"  
+         data-charset="iso-8859-15">
+</section>
+```
+
+#### Element Attributes
+
+Special syntax (in html comment) is available for adding attributes to Markdown elements. This is useful for fragments, amongst other things.
+
+```html
+<section data-markdown>
+	<script type="text/template">
+		- Item 1 <!-- .element: class="fragment" data-fragment-index="2" -->
+		- Item 2 <!-- .element: class="fragment" data-fragment-index="1" -->
+	</script>
+</section>
+```
+
+#### Slide Attributes
+
+Special syntax (in html comment) is available for adding attributes to the slide `<section>` elements generated by your Markdown.
+
+```html
+<section data-markdown>
+	<script type="text/template">
+	<!-- .slide: data-background="#ff0000" -->
+		Markdown content
+	</script>
+</section>
+```
+
+
+### Configuration
+
+At the end of your page you need to initialize reveal by running the following code. Note that all config values are optional and will default as specified below.
+
+```javascript
+Reveal.initialize({
+
+	// Display controls in the bottom right corner
+	controls: true,
+
+	// Display a presentation progress bar
+	progress: true,
+
+	// Display the page number of the current slide
+	slideNumber: false,
+
+	// Push each slide change to the browser history
+	history: false,
+
+	// Enable keyboard shortcuts for navigation
+	keyboard: true,
+
+	// Enable the slide overview mode
+	overview: true,
+
+	// Vertical centering of slides
+	center: true,
+
+	// Enables touch navigation on devices with touch input
+	touch: true,
+
+	// Loop the presentation
+	loop: false,
+
+	// Change the presentation direction to be RTL
+	rtl: false,
+
+	// Randomizes the order of slides each time the presentation loads
+	shuffle: false,
+
+	// Turns fragments on and off globally
+	fragments: true,
+
+	// Flags if the presentation is running in an embedded mode,
+	// i.e. contained within a limited portion of the screen
+	embedded: false,
+
+	// Flags if we should show a help overlay when the questionmark
+	// key is pressed
+	help: true,
+
+	// Flags if speaker notes should be visible to all viewers
+	showNotes: false,
+
+	// Number of milliseconds between automatically proceeding to the
+	// next slide, disabled when set to 0, this value can be overwritten
+	// by using a data-autoslide attribute on your slides
+	autoSlide: 0,
+
+	// Stop auto-sliding after user input
+	autoSlideStoppable: true,
+
+	// Use this method for navigation when auto-sliding
+	autoSlideMethod: Reveal.navigateNext,
+
+	// Enable slide navigation via mouse wheel
+	mouseWheel: false,
+
+	// Hides the address bar on mobile devices
+	hideAddressBar: true,
+
+	// Opens links in an iframe preview overlay
+	previewLinks: false,
+
+	// Transition style
+	transition: 'default', // none/fade/slide/convex/concave/zoom
+
+	// Transition speed
+	transitionSpeed: 'default', // default/fast/slow
+
+	// Transition style for full page slide backgrounds
+	backgroundTransition: 'default', // none/fade/slide/convex/concave/zoom
+
+	// Number of slides away from the current that are visible
+	viewDistance: 3,
+
+	// Parallax background image
+	parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'"
+
+	// Parallax background size
+	parallaxBackgroundSize: '', // CSS syntax, e.g. "2100px 900px"
+
+	// Number of pixels to move the parallax background per slide
+	// - Calculated automatically unless specified
+	// - Set to 0 to disable movement along an axis
+	parallaxBackgroundHorizontal: null,
+	parallaxBackgroundVertical: null
+
+});
+```
+
+
+The configuration can be updated after initialization using the ```configure``` method:
+
+```javascript
+// Turn autoSlide off
+Reveal.configure({ autoSlide: 0 });
+
+// Start auto-sliding every 5s
+Reveal.configure({ autoSlide: 5000 });
+```
+
+
+### Presentation Size
+
+All presentations have a normal size, that is the resolution at which they are authored. The framework will automatically scale presentations uniformly based on this size to ensure that everything fits on any given display or viewport.
+
+See below for a list of configuration options related to sizing, including default values:
+
+```javascript
+Reveal.initialize({
+
+	...
+
+	// The "normal" size of the presentation, aspect ratio will be preserved
+	// when the presentation is scaled to fit different resolutions. Can be
+	// specified using percentage units.
+	width: 960,
+	height: 700,
+
+	// Factor of the display size that should remain empty around the content
+	margin: 0.1,
+
+	// Bounds for smallest/largest possible scale to apply to content
+	minScale: 0.2,
+	maxScale: 1.5
+
+});
+```
+
+
+### Dependencies
+
+Reveal.js doesn't _rely_ on any third party scripts to work but a few optional libraries are included by default. These libraries are loaded as dependencies in the order they appear, for example:
+
+```javascript
+Reveal.initialize({
+	dependencies: [
+		// Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/
+		{ src: 'lib/js/classList.js', condition: function() { return !document.body.classList; } },
+
+		// Interpret Markdown in <section> elements
+		{ src: 'plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
+		{ src: 'plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } },
+
+		// Syntax highlight for <code> elements
+		{ src: 'plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
+
+		// Zoom in and out with Alt+click
+		{ src: 'plugin/zoom-js/zoom.js', async: true },
+
+		// Speaker notes
+		{ src: 'plugin/notes/notes.js', async: true },
+
+		// MathJax
+		{ src: 'plugin/math/math.js', async: true }
+	]
+});
+```
+
+You can add your own extensions using the same syntax. The following properties are available for each dependency object:
+- **src**: Path to the script to load
+- **async**: [optional] Flags if the script should load after reveal.js has started, defaults to false
+- **callback**: [optional] Function to execute when the script has loaded
+- **condition**: [optional] Function which must return true for the script to be loaded
+
+
+### Ready Event
+
+A 'ready' event is fired when reveal.js has loaded all non-async dependencies and is ready to start navigating. To check if reveal.js is already 'ready' you can call `Reveal.isReady()`.
+
+```javascript
+Reveal.addEventListener( 'ready', function( event ) {
+	// event.currentSlide, event.indexh, event.indexv
+} );
+```
+
+
+### Auto-sliding
+
+Presentations can be configured to progress through slides automatically, without any user input. To enable this you will need to tell the framework how many milliseconds it should wait between slides:
+
+```javascript
+// Slide every five seconds
+Reveal.configure({
+  autoSlide: 5000
+});
+```
+When this is turned on a control element will appear that enables users to pause and resume auto-sliding. Alternatively, sliding can be paused or resumed by pressing &#187;a&#171; on the keyboard. Sliding is paused automatically as soon as the user starts navigating. You can disable these controls by specifying ```autoSlideStoppable: false``` in your reveal.js config.
+
+You can also override the slide duration for individual slides and fragments by using the ```data-autoslide``` attribute:
+
+```html
+<section data-autoslide="2000">
+	<p>After 2 seconds the first fragment will be shown.</p>
+	<p class="fragment" data-autoslide="10000">After 10 seconds the next fragment will be shown.</p>
+	<p class="fragment">Now, the fragment is displayed for 2 seconds before the next slide is shown.</p>
+</section>
+```
+
+To override the method used for navigation when auto-sliding, you can specify the ```autoSlideMethod``` setting. To only navigate along the top layer and ignore vertical slides, set this to ```Reveal.navigateRight```.
+
+Whenever the auto-slide mode is resumed or paused the ```autoslideresumed``` and ```autoslidepaused``` events are fired.
+
+
+### Keyboard Bindings
+
+If you're unhappy with any of the default keyboard bindings you can override them using the ```keyboard``` config option:
+
+```javascript
+Reveal.configure({
+  keyboard: {
+    13: 'next', // go to the next slide when the ENTER key is pressed
+    27: function() {}, // do something custom when ESC is pressed
+    32: null // don't do anything when SPACE is pressed (i.e. disable a reveal.js default binding)
+  }
+});
+```
+
+### Touch Navigation
+
+You can swipe to navigate through a presentation on any touch-enabled device. Horizontal swipes change between horizontal slides, vertical swipes change between vertical slides. If you wish to disable this you can set the `touch` config option to false when initializing reveal.js.
+
+If there's some part of your content that needs to remain accessible to touch events you'll need to highlight this by adding a `data-prevent-swipe` attribute to the element. One common example where this is useful is elements that need to be scrolled.
+
+
+### Lazy Loading
+
+When working on presentation with a lot of media or iframe content it's important to load lazily. Lazy loading means that reveal.js will only load content for the few slides nearest to the current slide. The number of slides that are preloaded is determined by the `viewDistance` configuration option.
+
+To enable lazy loading all you need to do is change your "src" attributes to "data-src" as shown below. This is supported for image, video, audio and iframe elements. Lazy loaded iframes will also unload when the containing slide is no longer visible.
+
+```html
+<section>
+  <img data-src="image.png">
+  <iframe data-src="http://hakim.se"></iframe>
+  <video>
+    <source data-src="video.webm" type="video/webm" />
+    <source data-src="video.mp4" type="video/mp4" />
+  </video>
+</section>
+```
+
+
+### API
+
+The ``Reveal`` object exposes a JavaScript API for controlling navigation and reading state:
+
+```javascript
+// Navigation
+Reveal.slide( indexh, indexv, indexf );
+Reveal.left();
+Reveal.right();
+Reveal.up();
+Reveal.down();
+Reveal.prev();
+Reveal.next();
+Reveal.prevFragment();
+Reveal.nextFragment();
+
+// Randomize the order of slides
+Reveal.shuffle();
+
+// Toggle presentation states, optionally pass true/false to force on/off
+Reveal.toggleOverview();
+Reveal.togglePause();
+Reveal.toggleAutoSlide();
+
+// Change a config value at runtime
+Reveal.configure({ controls: true });
+
+// Returns the present configuration options
+Reveal.getConfig();
+
+// Fetch the current scale of the presentation
+Reveal.getScale();
+
+// Retrieves the previous and current slide elements
+Reveal.getPreviousSlide();
+Reveal.getCurrentSlide();
+
+Reveal.getIndices(); // { h: 0, v: 0 } }
+Reveal.getProgress(); // 0-1
+Reveal.getTotalSlides();
+
+// Returns the speaker notes for the current slide
+Reveal.getSlideNotes();
+
+// State checks
+Reveal.isFirstSlide();
+Reveal.isLastSlide();
+Reveal.isOverview();
+Reveal.isPaused();
+Reveal.isAutoSliding();
+```
+
+### Slide Changed Event
+
+A 'slidechanged' event is fired each time the slide is changed (regardless of state). The event object holds the index values of the current slide as well as a reference to the previous and current slide HTML nodes.
+
+Some libraries, like MathJax (see [#226](https://github.com/hakimel/reveal.js/issues/226#issuecomment-10261609)), get confused by the transforms and display states of slides. Often times, this can be fixed by calling their update or render function from this callback.
+
+```javascript
+Reveal.addEventListener( 'slidechanged', function( event ) {
+	// event.previousSlide, event.currentSlide, event.indexh, event.indexv
+} );
+```
+
+### Presentation State
+
+The presentation's current state can be fetched by using the `getState` method. A state object contains all of the information required to put the presentation back as it was when `getState` was first called. Sort of like a snapshot. It's a simple object that can easily be stringified and persisted or sent over the wire.
+
+```javascript
+Reveal.slide( 1 );
+// we're on slide 1
+
+var state = Reveal.getState();
+
+Reveal.slide( 3 );
+// we're on slide 3
+
+Reveal.setState( state );
+// we're back on slide 1
+```
+
+### Slide States
+
+If you set ``data-state="somestate"`` on a slide ``<section>``, "somestate" will be applied as a class on the document element when that slide is opened. This allows you to apply broad style changes to the page based on the active slide.
+
+Furthermore you can also listen to these changes in state via JavaScript:
+
+```javascript
+Reveal.addEventListener( 'somestate', function() {
+	// TODO: Sprinkle magic
+}, false );
+```
+
+### Slide Backgrounds
+
+Slides are contained within a limited portion of the screen by default to allow them to fit any display and scale uniformly. You can apply full page backgrounds outside of the slide area by adding a ```data-background``` attribute to your ```<section>``` elements. Four different types of backgrounds are supported: color, image, video and iframe. Below are a few examples.
+
+```html
+<section data-background="#ff0000">
+	<h2>All CSS color formats are supported, like rgba() or hsl().</h2>
+</section>
+<section data-background="http://example.com/image.png">
+	<h2>This slide will have a full-size background image.</h2>
+</section>
+<section data-background="http://example.com/image.png" data-background-size="100px" data-background-repeat="repeat">
+	<h2>This background image will be sized to 100px and repeated.</h2>
+</section>
+<section data-background-video="https://s3.amazonaws.com/static.slid.es/site/homepage/v1/homepage-video-editor.mp4,https://s3.amazonaws.com/static.slid.es/site/homepage/v1/homepage-video-editor.webm" data-background-video-loop data-background-video-muted>
+	<h2>Video. Multiple sources can be defined using a comma separated list. Video will loop when the data-background-video-loop attribute is provided and can be muted with the data-background-video-muted attribute.</h2>
+</section>
+<section data-background-iframe="https://slides.com">
+	<h2>Embeds a web page as a background. Note that the page won't be interactive.</h2>
+</section>
+```
+
+Backgrounds transition using a fade animation by default. This can be changed to a linear sliding transition by passing ```backgroundTransition: 'slide'``` to the ```Reveal.initialize()``` call. Alternatively you can set ```data-background-transition``` on any section with a background to override that specific transition.
+
+
+### Parallax Background
+
+If you want to use a parallax scrolling background, set the first two config properties below when initializing reveal.js (the other two are optional).
+
+```javascript
+Reveal.initialize({
+
+	// Parallax background image
+	parallaxBackgroundImage: '', // e.g. "https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg"
+
+	// Parallax background size
+	parallaxBackgroundSize: '', // CSS syntax, e.g. "2100px 900px" - currently only pixels are supported (don't use % or auto)
+
+	// Number of pixels to move the parallax background per slide
+	// - Calculated automatically unless specified
+	// - Set to 0 to disable movement along an axis
+	parallaxBackgroundHorizontal: 200,
+	parallaxBackgroundVertical: 50
+
+});
+```
+
+Make sure that the background size is much bigger than screen size to allow for some scrolling. [View example](http://lab.hakim.se/reveal-js/?parallaxBackgroundImage=https%3A%2F%2Fs3.amazonaws.com%2Fhakim-static%2Freveal-js%2Freveal-parallax-1.jpg&parallaxBackgroundSize=2100px%20900px).
+
+
+
+### Slide Transitions
+The global presentation transition is set using the ```transition``` config value. You can override the global transition for a specific slide by using the ```data-transition``` attribute:
+
+```html
+<section data-transition="zoom">
+	<h2>This slide will override the presentation transition and zoom!</h2>
+</section>
+
+<section data-transition-speed="fast">
+	<h2>Choose from three transition speeds: default, fast or slow!</h2>
+</section>
+```
+
+You can also use different in and out transitions for the same slide:
+
+```html
+<section data-transition="slide">
+    The train goes on &#8230;
+</section>
+<section data-transition="slide">
+    and on &#8230;
+</section>
+<section data-transition="slide-in fade-out">
+    and stops.
+</section>
+<section data-transition="fade-in slide-out">
+    (Passengers entering and leaving)
+</section>
+<section data-transition="slide">
+    And it starts again.
+</section>
+```
+
+
+### Internal links
+
+It's easy to link between slides. The first example below targets the index of another slide whereas the second targets a slide with an ID attribute (```<section id="some-slide">```):
+
+```html
+<a href="#/2/2">Link</a>
+<a href="#/some-slide">Link</a>
+```
+
+You can also add relative navigation links, similar to the built in reveal.js controls, by appending one of the following classes on any element. Note that each element is automatically given an ```enabled``` class when it's a valid navigation route based on the current slide.
+
+```html
+<a href="#" class="navigate-left">
+<a href="#" class="navigate-right">
+<a href="#" class="navigate-up">
+<a href="#" class="navigate-down">
+<a href="#" class="navigate-prev"> <!-- Previous vertical or horizontal slide -->
+<a href="#" class="navigate-next"> <!-- Next vertical or horizontal slide -->
+```
+
+
+### Fragments
+Fragments are used to highlight individual elements on a slide. Every element with the class ```fragment``` will be stepped through before moving on to the next slide. Here's an example: http://lab.hakim.se/reveal-js/#/fragments
+
+The default fragment style is to start out invisible and fade in. This style can be changed by appending a different class to the fragment:
+
+```html
+<section>
+	<p class="fragment grow">grow</p>
+	<p class="fragment shrink">shrink</p>
+	<p class="fragment fade-out">fade-out</p>
+	<p class="fragment fade-up">fade-up (also down, left and right!)</p>
+	<p class="fragment current-visible">visible only once</p>
+	<p class="fragment highlight-current-blue">blue only once</p>
+	<p class="fragment highlight-red">highlight-red</p>
+	<p class="fragment highlight-green">highlight-green</p>
+	<p class="fragment highlight-blue">highlight-blue</p>
+</section>
+```
+
+Multiple fragments can be applied to the same element sequentially by wrapping it, this will fade in the text on the first step and fade it back out on the second.
+
+```html
+<section>
+	<span class="fragment fade-in">
+		<span class="fragment fade-out">I'll fade in, then out</span>
+	</span>
+</section>
+```
+
+The display order of fragments can be controlled using the ```data-fragment-index``` attribute.
+
+```html
+<section>
+	<p class="fragment" data-fragment-index="3">Appears last</p>
+	<p class="fragment" data-fragment-index="1">Appears first</p>
+	<p class="fragment" data-fragment-index="2">Appears second</p>
+</section>
+```
+
+### Fragment events
+
+When a slide fragment is either shown or hidden reveal.js will dispatch an event.
+
+Some libraries, like MathJax (see #505), get confused by the initially hidden fragment elements. Often times this can be fixed by calling their update or render function from this callback.
+
+```javascript
+Reveal.addEventListener( 'fragmentshown', function( event ) {
+	// event.fragment = the fragment DOM element
+} );
+Reveal.addEventListener( 'fragmenthidden', function( event ) {
+	// event.fragment = the fragment DOM element
+} );
+```
+
+### Code syntax highlighting
+
+By default, Reveal is configured with [highlight.js](https://highlightjs.org/) for code syntax highlighting. Below is an example with clojure code that will be syntax highlighted. When the `data-trim` attribute is present, surrounding whitespace is automatically removed.  HTML will be escaped by default. To avoid this, for example if you are using `<mark>` to call out a line of code, add the `data-noescape` attribute to the `<code>` element.
+
+```html
+<section>
+	<pre><code data-trim data-noescape>
+(def lazy-fib
+  (concat
+   [0 1]
+   <mark>((fn rfib [a b]</mark>
+        (lazy-cons (+ a b) (rfib b (+ a b)))) 0 1)))
+	</code></pre>
+</section>
+```
+
+### Slide number
+If you would like to display the page number of the current slide you can do so using the ```slideNumber``` configuration value.
+
+```javascript
+// Shows the slide number using default formatting
+Reveal.configure({ slideNumber: true });
+
+// Slide number formatting can be configured using these variables:
+//  "h.v": 	horizontal . vertical slide number (default)
+//  "h/v": 	horizontal / vertical slide number
+//    "c": 	flattened slide number
+//  "c/t": 	flattened slide number / total slides
+Reveal.configure({ slideNumber: 'c/t' });
+
+```
+
+
+### Overview mode
+
+Press "Esc" or "o" keys to toggle the overview mode on and off. While you're in this mode, you can still navigate between slides,
+as if you were at 1,000 feet above your presentation. The overview mode comes with a few API hooks:
+
+```javascript
+Reveal.addEventListener( 'overviewshown', function( event ) { /* ... */ } );
+Reveal.addEventListener( 'overviewhidden', function( event ) { /* ... */ } );
+
+// Toggle the overview mode programmatically
+Reveal.toggleOverview();
+```
+
+### Fullscreen mode
+Just press &#187;F&#171; on your keyboard to show your presentation in fullscreen mode. Press the &#187;ESC&#171; key to exit fullscreen mode.
+
+
+### Embedded media
+Embedded HTML5 `<video>`/`<audio>` and YouTube iframes are automatically paused when you navigate away from a slide. This can be disabled by decorating your element with a `data-ignore` attribute.
+
+Add `data-autoplay` to your media element if you want it to automatically start playing when the slide is shown:
+
+```html
+<video data-autoplay src="http://clips.vorwaerts-gmbh.de/big_buck_bunny.mp4"></video>
+```
+
+Additionally the framework automatically pushes two [post messages](https://developer.mozilla.org/en-US/docs/Web/API/Window.postMessage) to all iframes, ```slide:start``` when the slide containing the iframe is made visible and ```slide:stop``` when it is hidden.
+
+
+### Stretching elements
+Sometimes it's desirable to have an element, like an image or video, stretch to consume as much space as possible within a given slide. This can be done by adding the ```.stretch``` class to an element as seen below:
+
+```html
+<section>
+	<h2>This video will use up the remaining space on the slide</h2>
+    <video class="stretch" src="http://clips.vorwaerts-gmbh.de/big_buck_bunny.mp4"></video>
+</section>
+```
+
+Limitations:
+- Only direct descendants of a slide section can be stretched
+- Only one descendant per slide section can be stretched
+
+
+### postMessage API
+The framework has a built-in postMessage API that can be used when communicating with a presentation inside of another window. Here's an example showing how you'd make a reveal.js instance in the given window proceed to slide 2:
+
+```javascript
+<window>.postMessage( JSON.stringify({ method: 'slide', args: [ 2 ] }), '*' );
+```
+
+When reveal.js runs inside of an iframe it can optionally bubble all of its events to the parent. Bubbled events are stringified JSON with three fields: namespace, eventName and state. Here's how you subscribe to them from the parent window:
+
+```javascript
+window.addEventListener( 'message', function( event ) {
+	var data = JSON.parse( event.data );
+	if( data.namespace === 'reveal' && data.eventName ==='slidechanged' ) {
+		// Slide changed, see data.state for slide number
+	}
+} );
+```
+
+This cross-window messaging can be toggled on or off using configuration flags.
+
+```javascript
+Reveal.initialize({
+	...,
+
+	// Exposes the reveal.js API through window.postMessage
+	postMessage: true,
+
+	// Dispatches all reveal.js events to the parent window through postMessage
+	postMessageEvents: false
+});
+```
+
+
+## PDF Export
+
+Presentations can be exported to PDF via a special print stylesheet. This feature requires that you use [Google Chrome](http://google.com/chrome) or [Chromium](https://www.chromium.org/Home).
+Here's an example of an exported presentation that's been uploaded to SlideShare: http://www.slideshare.net/hakimel/revealjs-300.
+
+1. Open your presentation with `print-pdf` included anywhere in the query string. This triggers the default index HTML to load the PDF print stylesheet ([css/print/pdf.css](https://github.com/hakimel/reveal.js/blob/master/css/print/pdf.css)). You can test this with [lab.hakim.se/reveal-js?print-pdf](http://lab.hakim.se/reveal-js?print-pdf).
+2. Open the in-browser print dialog (CTRL/CMD+P).
+3. Change the **Destination** setting to **Save as PDF**.
+4. Change the **Layout** to **Landscape**.
+5. Change the **Margins** to **None**.
+6. Enable the **Background graphics** option.
+7. Click **Save**.
+
+![Chrome Print Settings](https://s3.amazonaws.com/hakim-static/reveal-js/pdf-print-settings-2.png)
+
+Alternatively you can use the [decktape](https://github.com/astefanutti/decktape) project.
+
+## Theming
+
+The framework comes with a few different themes included:
+
+- black: Black background, white text, blue links (default theme)
+- white: White background, black text, blue links
+- league: Gray background, white text, blue links (default theme for reveal.js < 3.0.0)
+- beige: Beige background, dark text, brown links
+- sky: Blue background, thin dark text, blue links
+- night: Black background, thick white text, orange links
+- serif: Cappuccino background, gray text, brown links
+- simple: White background, black text, blue links
+- solarized: Cream-colored background, dark green text, blue links
+
+Each theme is available as a separate stylesheet. To change theme you will need to replace **black** below with your desired theme name in index.html:
+
+```html
+<link rel="stylesheet" href="css/theme/black.css" id="theme">
+```
+
+If you want to add a theme of your own see the instructions here: [/css/theme/README.md](https://github.com/hakimel/reveal.js/blob/master/css/theme/README.md).
+
+
+## Speaker Notes
+
+reveal.js comes with a speaker notes plugin which can be used to present per-slide notes in a separate browser window. The notes window also gives you a preview of the next upcoming slide so it may be helpful even if you haven't written any notes. Press the 's' key on your keyboard to open the notes window.
+
+Notes are defined by appending an ```<aside>``` element to a slide as seen below. You can add the ```data-markdown``` attribute to the aside element if you prefer writing notes using Markdown.
+
+Alternatively you can add your notes in a `data-notes` attribute on the slide. Like `<section data-notes="Something important"></section>`.
+
+When used locally, this feature requires that reveal.js [runs from a local web server](#full-setup).
+
+```html
+<section>
+	<h2>Some Slide</h2>
+
+	<aside class="notes">
+		Oh hey, these are some notes. They'll be hidden in your presentation, but you can see them if you open the speaker notes window (hit 's' on your keyboard).
+	</aside>
+</section>
+```
+
+If you're using the external Markdown plugin, you can add notes with the help of a special delimiter:
+
+```html
+<section data-markdown="example.md" data-separator="^\n\n\n" data-separator-vertical="^\n\n" data-separator-notes="^Note:"></section>
+
+# Title
+## Sub-title
+
+Here is some content...
+
+Note:
+This will only display in the notes window.
+```
+
+#### Share and Print Speaker Notes
+
+Notes are only visible to the speaker inside of the speaker view. If you wish to share your notes with others you can initialize reveal.js with the `showNotes` config value set to `true`. Notes will appear along the bottom of the presentations.
+
+When `showNotes` is enabled notes are also included when you [export to PDF](https://github.com/hakimel/reveal.js#pdf-export).
+
+## Server Side Speaker Notes
+
+In some cases it can be desirable to run notes on a separate device from the one you're presenting on. The Node.js-based notes plugin lets you do this using the same note definitions as its client side counterpart. Include the required scripts by adding the following dependencies:
+
+```javascript
+Reveal.initialize({
+	...
+
+	dependencies: [
+		{ src: 'socket.io/socket.io.js', async: true },
+		{ src: 'plugin/notes-server/client.js', async: true }
+	]
+});
+```
+
+Then:
+
+1. Install [Node.js](http://nodejs.org/) (1.0.0 or later)
+2. Run ```npm install```
+3. Run ```node plugin/notes-server```
+
+
+## Multiplexing
+
+The multiplex plugin allows your audience to view the slides of the presentation you are controlling on their own phone, tablet or laptop. As the master presentation navigates the slides, all client presentations will update in real time. See a demo at [https://reveal-js-multiplex-ccjbegmaii.now.sh/](https://reveal-js-multiplex-ccjbegmaii.now.sh/).
+
+The multiplex plugin needs the following 3 things to operate:
+
+1. Master presentation that has control
+2. Client presentations that follow the master
+3. Socket.io server to broadcast events from the master to the clients
+
+More details:
+
+#### Master presentation
+Served from a static file server accessible (preferably) only to the presenter. This need only be on your (the presenter's) computer. (It's safer to run the master presentation from your own computer, so if the venue's Internet goes down it doesn't stop the show.) An example would be to execute the following commands in the directory of your master presentation:
+
+1. ```npm install node-static```
+2. ```static```
+
+If you want to use the speaker notes plugin with your master presentation then make sure you have the speaker notes plugin configured correctly along with the configuration shown below, then execute ```node plugin/notes-server``` in the directory of your master presentation. The configuration below will cause it to connect to the socket.io server as a master, as well as launch your speaker-notes/static-file server.
+
+You can then access your master presentation at ```http://localhost:1947```
+
+Example configuration:
+```javascript
+Reveal.initialize({
+	// other options...
+
+	multiplex: {
+		// Example values. To generate your own, see the socket.io server instructions.
+		secret: '13652805320794272084', // Obtained from the socket.io server. Gives this (the master) control of the presentation
+		id: '1ea875674b17ca76', // Obtained from socket.io server
+		url: 'https://reveal-js-multiplex-ccjbegmaii.now.sh' // Location of socket.io server
+	},
+
+	// Don't forget to add the dependencies
+	dependencies: [
+		{ src: '//cdn.socket.io/socket.io-1.3.5.js', async: true },
+		{ src: 'plugin/multiplex/master.js', async: true },
+
+		// and if you want speaker notes
+		{ src: 'plugin/notes-server/client.js', async: true }
+
+		// other dependencies...
+	]
+});
+```
+
+#### Client presentation
+Served from a publicly accessible static file server. Examples include: GitHub Pages, Amazon S3, Dreamhost, Akamai, etc. The more reliable, the better. Your audience can then access the client presentation via ```http://example.com/path/to/presentation/client/index.html```, with the configuration below causing them to connect to the socket.io server as clients.
+
+Example configuration:
+```javascript
+Reveal.initialize({
+	// other options...
+
+	multiplex: {
+		// Example values. To generate your own, see the socket.io server instructions.
+		secret: null, // null so the clients do not have control of the master presentation
+		id: '1ea875674b17ca76', // id, obtained from socket.io server
+		url: 'https://reveal-js-multiplex-ccjbegmaii.now.sh' // Location of socket.io server
+	},
+
+	// Don't forget to add the dependencies
+	dependencies: [
+		{ src: '//cdn.socket.io/socket.io-1.3.5.js', async: true },
+		{ src: 'plugin/multiplex/client.js', async: true }
+
+		// other dependencies...
+	]
+});
+```
+
+#### Socket.io server
+Server that receives the slideChanged events from the master presentation and broadcasts them out to the connected client presentations. This needs to be publicly accessible. You can run your own socket.io server with the commands:
+
+1. ```npm install```
+2. ```node plugin/multiplex```
+
+Or you use the socket.io server at [https://reveal-js-multiplex-ccjbegmaii.now.sh/](https://reveal-js-multiplex-ccjbegmaii.now.sh/).
+
+You'll need to generate a unique secret and token pair for your master and client presentations. To do so, visit ```http://example.com/token```, where ```http://example.com``` is the location of your socket.io server. Or if you're going to use the socket.io server at [https://reveal-js-multiplex-ccjbegmaii.now.sh/](https://reveal-js-multiplex-ccjbegmaii.now.sh/), visit [https://reveal-js-multiplex-ccjbegmaii.now.sh/token](https://reveal-js-multiplex-ccjbegmaii.now.sh/token).
+
+You are very welcome to point your presentations at the Socket.io server running at [https://reveal-js-multiplex-ccjbegmaii.now.sh/](https://reveal-js-multiplex-ccjbegmaii.now.sh/), but availability and stability are not guaranteed. For anything mission critical I recommend you run your own server. It is simple to deploy to nodejitsu, heroku, your own environment, etc.
+
+##### socket.io server as file static server
+
+The socket.io server can play the role of static file server for your client presentation, as in the example at [https://reveal-js-multiplex-ccjbegmaii.now.sh/](https://reveal-js-multiplex-ccjbegmaii.now.sh/). (Open [https://reveal-js-multiplex-ccjbegmaii.now.sh/](https://reveal-js-multiplex-ccjbegmaii.now.sh/) in two browsers. Navigate through the slides on one, and the other will update to match.)
+
+Example configuration:
+```javascript
+Reveal.initialize({
+	// other options...
+
+	multiplex: {
+		// Example values. To generate your own, see the socket.io server instructions.
+		secret: null, // null so the clients do not have control of the master presentation
+		id: '1ea875674b17ca76', // id, obtained from socket.io server
+		url: 'example.com:80' // Location of your socket.io server
+	},
+
+	// Don't forget to add the dependencies
+	dependencies: [
+		{ src: '//cdn.socket.io/socket.io-1.3.5.js', async: true },
+		{ src: 'plugin/multiplex/client.js', async: true }
+
+		// other dependencies...
+	]
+```
+
+It can also play the role of static file server for your master presentation and client presentations at the same time (as long as you don't want to use speaker notes). (Open [https://reveal-js-multiplex-ccjbegmaii.now.sh/](https://reveal-js-multiplex-ccjbegmaii.now.sh/) in two browsers. Navigate through the slides on one, and the other will update to match. Navigate through the slides on the second, and the first will update to match.) This is probably not desirable, because you don't want your audience to mess with your slides while you're presenting. ;)
+
+Example configuration:
+```javascript
+Reveal.initialize({
+	// other options...
+
+	multiplex: {
+		// Example values. To generate your own, see the socket.io server instructions.
+		secret: '13652805320794272084', // Obtained from the socket.io server. Gives this (the master) control of the presentation
+		id: '1ea875674b17ca76', // Obtained from socket.io server
+		url: 'example.com:80' // Location of your socket.io server
+	},
+
+	// Don't forget to add the dependencies
+	dependencies: [
+		{ src: '//cdn.socket.io/socket.io-1.3.5.js', async: true },
+		{ src: 'plugin/multiplex/master.js', async: true },
+		{ src: 'plugin/multiplex/client.js', async: true }
+
+		// other dependencies...
+	]
+});
+```
+
+## MathJax
+
+If you want to display math equations in your presentation you can easily do so by including this plugin. The plugin is a very thin wrapper around the [MathJax](http://www.mathjax.org/) library. To use it you'll need to include it as a reveal.js dependency, [find our more about dependencies here](#dependencies).
+
+The plugin defaults to using [LaTeX](http://en.wikipedia.org/wiki/LaTeX) but that can be adjusted through the ```math``` configuration object. Note that MathJax is loaded from a remote server. If you want to use it offline you'll need to download a copy of the library and adjust the ```mathjax``` configuration value.
+
+Below is an example of how the plugin can be configured. If you don't intend to change these values you do not need to include the ```math``` config object at all.
+
+```js
+Reveal.initialize({
+
+	// other options ...
+
+	math: {
+		mathjax: 'https://cdn.mathjax.org/mathjax/latest/MathJax.js',
+		config: 'TeX-AMS_HTML-full'  // See http://docs.mathjax.org/en/latest/config-files.html
+	},
+
+	dependencies: [
+		{ src: 'plugin/math/math.js', async: true }
+	]
+
+});
+```
+
+Read MathJax's documentation if you need [HTTPS delivery](http://docs.mathjax.org/en/latest/start.html#secure-access-to-the-cdn) or serving of [specific versions](http://docs.mathjax.org/en/latest/configuration.html#loading-mathjax-from-the-cdn) for stability.
+
+
+## Installation
+
+The **basic setup** is for authoring presentations only. The **full setup** gives you access to all reveal.js features and plugins such as speaker notes as well as the development tasks needed to make changes to the source.
+
+### Basic setup
+
+The core of reveal.js is very easy to install. You'll simply need to download a copy of this repository and open the index.html file directly in your browser.
+
+1. Download the latest version of reveal.js from <https://github.com/hakimel/reveal.js/releases>
+
+2. Unzip and replace the example contents in index.html with your own
+
+3. Open index.html in a browser to view it
+
+
+### Full setup
+
+Some reveal.js features, like external Markdown and speaker notes, require that presentations run from a local web server. The following instructions will set up such a server as well as all of the development tasks needed to make edits to the reveal.js source code.
+
+1. Install [Node.js](http://nodejs.org/) (1.0.0 or later)
+
+2. Install [Grunt](http://gruntjs.com/getting-started#installing-the-cli)
+
+4. Clone the reveal.js repository
+   ```sh
+   $ git clone https://github.com/hakimel/reveal.js.git
+   ```
+
+5. Navigate to the reveal.js folder
+   ```sh
+   $ cd reveal.js
+   ```
+
+6. Install dependencies
+   ```sh
+   $ npm install
+   ```
+
+7. Serve the presentation and monitor source files for changes
+   ```sh
+   $ grunt serve
+   ```
+
+8. Open <http://localhost:8000> to view your presentation
+
+   You can change the port by using `grunt serve --port 8001`.
+
+
+### Folder Structure
+- **css/** Core styles without which the project does not function
+- **js/** Like above but for JavaScript
+- **plugin/** Components that have been developed as extensions to reveal.js
+- **lib/** All other third party assets (JavaScript, CSS, fonts)
+
+
+## License
+
+MIT licensed
+
+Copyright (C) 2016 Hakim El Hattab, http://hakim.se
diff --git a/talk/pyconza2016/pypy/bower.json b/talk/pyconza2016/pypy/bower.json
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy/bower.json
@@ -0,0 +1,27 @@
+{
+  "name": "reveal.js",
+  "version": "3.3.0",
+  "main": [
+    "js/reveal.js",
+    "css/reveal.css"
+  ],
+  "homepage": "http://lab.hakim.se/reveal-js/",
+  "license": "MIT",
+  "description": "The HTML Presentation Framework",
+  "authors": [
+    "Hakim El Hattab <hakim.elhattab at gmail.com>"
+  ],
+  "dependencies": {
+    "headjs": "~1.0.3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/hakimel/reveal.js.git"
+  },
+  "ignore": [
+    "**/.*",
+    "node_modules",
+    "bower_components",
+    "test"
+  ]
+}
\ No newline at end of file
diff --git a/talk/pyconza2016/pypy/css/print/paper.css b/talk/pyconza2016/pypy/css/print/paper.css
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy/css/print/paper.css
@@ -0,0 +1,202 @@
+/* Default Print Stylesheet Template
+   by Rob Glazebrook of CSSnewbie.com
+   Last Updated: June 4, 2008
+
+   Feel free (nay, compelled) to edit, append, and
+   manipulate this file as you see fit. */
+
+
+ at media print {
+
+	/* SECTION 1: Set default width, margin, float, and
+	   background. This prevents elements from extending
+	   beyond the edge of the printed page, and prevents
+	   unnecessary background images from printing */
+	html {
+		background: #fff;
+		width: auto;
+		height: auto;
+		overflow: visible;
+	}
+	body {
+		background: #fff;
+		font-size: 20pt;
+		width: auto;
+		height: auto;
+		border: 0;
+		margin: 0 5%;
+		padding: 0;
+		overflow: visible;
+		float: none !important;
+	}
+
+	/* SECTION 2: Remove any elements not needed in print.
+	   This would include navigation, ads, sidebars, etc. */
+	.nestedarrow,
+	.controls,
+	.fork-reveal,
+	.share-reveal,
+	.state-background,
+	.reveal .progress,
+	.reveal .backgrounds {
+		display: none !important;
+	}
+
+	/* SECTION 3: Set body font face, size, and color.
+	   Consider using a serif font for readability. */
+	body, p, td, li, div {
+		font-size: 20pt!important;
+		font-family: Georgia, "Times New Roman", Times, serif !important;
+		color: #000;
+	}
+
+	/* SECTION 4: Set heading font face, sizes, and color.
+	   Differentiate your headings from your body text.
+	   Perhaps use a large sans-serif for distinction. */
+	h1,h2,h3,h4,h5,h6 {
+		color: #000!important;
+		height: auto;
+		line-height: normal;
+		font-family: Georgia, "Times New Roman", Times, serif !important;
+		text-shadow: 0 0 0 #000 !important;
+		text-align: left;
+		letter-spacing: normal;
+	}
+	/* Need to reduce the size of the fonts for printing */
+	h1 { font-size: 28pt !important;  }
+	h2 { font-size: 24pt !important; }
+	h3 { font-size: 22pt !important; }
+	h4 { font-size: 22pt !important; font-variant: small-caps; }
+	h5 { font-size: 21pt !important; }
+	h6 { font-size: 20pt !important; font-style: italic; }
+
+	/* SECTION 5: Make hyperlinks more usable.
+	   Ensure links are underlined, and consider appending
+	   the URL to the end of the link for usability. */
+	a:link,
+	a:visited {
+		color: #000 !important;
+		font-weight: bold;
+		text-decoration: underline;
+	}
+	/*
+	.reveal a:link:after,
+	.reveal a:visited:after {
+		content: " (" attr(href) ") ";
+		color: #222 !important;
+		font-size: 90%;
+	}
+	*/
+
+
+	/* SECTION 6: more reveal.js specific additions by @skypanther */
+	ul, ol, div, p {
+		visibility: visible;
+		position: static;
+		width: auto;
+		height: auto;
+		display: block;
+		overflow: visible;
+		margin: 0;
+		text-align: left !important;
+	}
+	.reveal pre,
+	.reveal table {
+		margin-left: 0;
+		margin-right: 0;
+	}
+	.reveal pre code {
+		padding: 20px;
+		border: 1px solid #ddd;
+	}
+	.reveal blockquote {
+		margin: 20px 0;
+	}
+	.reveal .slides {
+		position: static !important;
+		width: auto !important;
+		height: auto !important;
+
+		left: 0 !important;
+		top: 0 !important;
+		margin-left: 0 !important;
+		margin-top: 0 !important;
+		padding: 0 !important;
+		zoom: 1 !important;
+
+		overflow: visible !important;
+		display: block !important;
+
+		text-align: left !important;
+		-webkit-perspective: none;
+		   -moz-perspective: none;
+		    -ms-perspective: none;
+		        perspective: none;
+
+		-webkit-perspective-origin: 50% 50%;
+		   -moz-perspective-origin: 50% 50%;
+		    -ms-perspective-origin: 50% 50%;
+		        perspective-origin: 50% 50%;
+	}
+	.reveal .slides section {
+		visibility: visible !important;
+		position: static !important;
+		width: auto !important;
+		height: auto !important;
+		display: block !important;
+		overflow: visible !important;
+
+		left: 0 !important;
+		top: 0 !important;
+		margin-left: 0 !important;
+		margin-top: 0 !important;
+		padding: 60px 20px !important;
+		z-index: auto !important;
+
+		opacity: 1 !important;
+
+		page-break-after: always !important;
+
+		-webkit-transform-style: flat !important;
+		   -moz-transform-style: flat !important;
+		    -ms-transform-style: flat !important;
+		        transform-style: flat !important;
+
+		-webkit-transform: none !important;
+		   -moz-transform: none !important;
+		    -ms-transform: none !important;
+		        transform: none !important;
+
+		-webkit-transition: none !important;
+		   -moz-transition: none !important;
+		    -ms-transition: none !important;
+		        transition: none !important;
+	}
+	.reveal .slides section.stack {
+		padding: 0 !important;
+	}
+	.reveal section:last-of-type {
+		page-break-after: avoid !important;
+	}
+	.reveal section .fragment {
+		opacity: 1 !important;
+		visibility: visible !important;
+
+		-webkit-transform: none !important;
+		   -moz-transform: none !important;
+		    -ms-transform: none !important;
+		        transform: none !important;
+	}
+	.reveal section img {
+		display: block;
+		margin: 15px 0px;
+		background: rgba(255,255,255,1);
+		border: 1px solid #666;
+		box-shadow: none;
+	}
+
+	.reveal section small {
+		font-size: 0.8em;
+	}
+
+}
\ No newline at end of file
diff --git a/talk/pyconza2016/pypy/css/print/pdf.css b/talk/pyconza2016/pypy/css/print/pdf.css
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy/css/print/pdf.css
@@ -0,0 +1,160 @@
+/**
+ * This stylesheet is used to print reveal.js
+ * presentations to PDF.
+ *
+ * https://github.com/hakimel/reveal.js#pdf-export
+ */
+
+* {
+	-webkit-print-color-adjust: exact;
+}
+
+body {
+	margin: 0 auto !important;
+	border: 0;
+	padding: 0;
+	float: none !important;
+	overflow: visible;
+}
+
+html {
+	width: 100%;
+	height: 100%;
+	overflow: visible;
+}
+
+/* Remove any elements not needed in print. */
+.nestedarrow,
+.reveal .controls,
+.reveal .progress,
+.reveal .playback,
+.reveal.overview,
+.fork-reveal,
+.share-reveal,
+.state-background {
+	display: none !important;
+}
+
+h1, h2, h3, h4, h5, h6 {
+	text-shadow: 0 0 0 #000 !important;
+}
+
+.reveal pre code {
+	overflow: hidden !important;
+	font-family: Courier, 'Courier New', monospace !important;
+}
+
+ul, ol, div, p {
+	visibility: visible;
+	position: static;
+	width: auto;
+	height: auto;
+	display: block;
+	overflow: visible;
+	margin: auto;
+}
+.reveal {
+	width: auto !important;
+	height: auto !important;
+	overflow: hidden !important;
+}
+.reveal .slides {
+	position: static;
+	width: 100%;
+	height: auto;
+
+	left: auto;
+	top: auto;
+	margin: 0 !important;
+	padding: 0 !important;
+
+	overflow: visible;
+	display: block;
+
+	-webkit-perspective: none;
+	   -moz-perspective: none;
+	    -ms-perspective: none;
+	        perspective: none;
+
+	-webkit-perspective-origin: 50% 50%; /* there isn't a none/auto value but 50-50 is the default */
+	   -moz-perspective-origin: 50% 50%;
+	    -ms-perspective-origin: 50% 50%;
+	        perspective-origin: 50% 50%;
+}
+
+.reveal .slides section {
+	page-break-after: always !important;
+
+	visibility: visible !important;
+	position: relative !important;
+	display: block !important;
+	position: relative !important;
+
+	margin: 0 !important;
+	padding: 0 !important;
+	box-sizing: border-box !important;
+	min-height: 1px;
+
+	opacity: 1 !important;
+
+	-webkit-transform-style: flat !important;
+	   -moz-transform-style: flat !important;
+	    -ms-transform-style: flat !important;
+	        transform-style: flat !important;
+
+	-webkit-transform: none !important;
+	   -moz-transform: none !important;
+	    -ms-transform: none !important;
+	        transform: none !important;
+}
+
+.reveal section.stack {
+	margin: 0 !important;
+	padding: 0 !important;
+	page-break-after: avoid !important;
+	height: auto !important;
+	min-height: auto !important;
+}
+
+.reveal img {
+	box-shadow: none;
+}
+
+.reveal .roll {
+	overflow: visible;
+	line-height: 1em;
+}
+
+/* Slide backgrounds are placed inside of their slide when exporting to PDF */
+.reveal section .slide-background {
+	display: block !important;
+	position: absolute;
+	top: 0;
+	left: 0;
+	width: 100%;
+	z-index: -1;
+}
+
+/* All elements should be above the slide-background */
+.reveal section>* {
+	position: relative;
+	z-index: 1;
+}
+
+/* Display slide speaker notes when 'showNotes' is enabled */
+.reveal .speaker-notes-pdf {
+	display: block;
+	width: 100%;
+	max-height: none;
+	left: auto;
+	top: auto;
+	z-index: 100;
+}
+
+/* Display slide numbers when 'slideNumber' is enabled */
+.reveal .slide-number-pdf {
+	display: block;
+	position: absolute;
+	font-size: 14px;
+}
+
diff --git a/talk/pyconza2016/pypy/css/reveal.css b/talk/pyconza2016/pypy/css/reveal.css
new file mode 100644
--- /dev/null
+++ b/talk/pyconza2016/pypy/css/reveal.css
@@ -0,0 +1,1331 @@
+/*!
+ * reveal.js
+ * http://lab.hakim.se/reveal-js
+ * MIT licensed
+ *
+ * Copyright (C) 2016 Hakim El Hattab, http://hakim.se
+ */
+/*********************************************
+ * RESET STYLES
+ *********************************************/
+html, body, .reveal div, .reveal span, .reveal applet, .reveal object, .reveal iframe,
+.reveal h1, .reveal h2, .reveal h3, .reveal h4, .reveal h5, .reveal h6, .reveal p, .reveal blockquote, .reveal pre,
+.reveal a, .reveal abbr, .reveal acronym, .reveal address, .reveal big, .reveal cite, .reveal code,
+.reveal del, .reveal dfn, .reveal em, .reveal img, .reveal ins, .reveal kbd, .reveal q, .reveal s, .reveal samp,
+.reveal small, .reveal strike, .reveal strong, .reveal sub, .reveal sup, .reveal tt, .reveal var,
+.reveal b, .reveal u, .reveal center,
+.reveal dl, .reveal dt, .reveal dd, .reveal ol, .reveal ul, .reveal li,
+.reveal fieldset, .reveal form, .reveal label, .reveal legend,
+.reveal table, .reveal caption, .reveal tbody, .reveal tfoot, .reveal thead, .reveal tr, .reveal th, .reveal td,
+.reveal article, .reveal aside, .reveal canvas, .reveal details, .reveal embed,
+.reveal figure, .reveal figcaption, .reveal footer, .reveal header, .reveal hgroup,
+.reveal menu, .reveal nav, .reveal output, .reveal ruby, .reveal section, .reveal summary,
+.reveal time, .reveal mark, .reveal audio, .reveal video {
+  margin: 0;
+  padding: 0;
+  border: 0;
+  font-size: 100%;
+  font: inherit;
+  vertical-align: baseline; }
+
+.reveal article, .reveal aside, .reveal details, .reveal figcaption, .reveal figure,
+.reveal footer, .reveal header, .reveal hgroup, .reveal menu, .reveal nav, .reveal section {
+  display: block; }
+
+/*********************************************
+ * GLOBAL STYLES
+ *********************************************/
+html,
+body {
+  width: 100%;
+  height: 100%;
+  overflow: hidden; }
+
+body {
+  position: relative;
+  line-height: 1;
+  background-color: #fff;
+  color: #000; }
+
+html:-webkit-full-screen-ancestor {
+  background-color: inherit; }
+
+html:-moz-full-screen-ancestor {
+  background-color: inherit; }
+
+/*********************************************
+ * VIEW FRAGMENTS
+ *********************************************/
+.reveal .slides section .fragment {
+  opacity: 0;
+  visibility: hidden;
+  -webkit-transition: all .2s ease;
+          transition: all .2s ease; }
+  .reveal .slides section .fragment.visible {
+    opacity: 1;
+    visibility: visible; }
+
+.reveal .slides section .fragment.grow {
+  opacity: 1;
+  visibility: visible; }
+  .reveal .slides section .fragment.grow.visible {
+    -webkit-transform: scale(1.3);
+            transform: scale(1.3); }
+
+.reveal .slides section .fragment.shrink {
+  opacity: 1;
+  visibility: visible; }
+  .reveal .slides section .fragment.shrink.visible {
+    -webkit-transform: scale(0.7);
+            transform: scale(0.7); }
+
+.reveal .slides section .fragment.zoom-in {
+  -webkit-transform: scale(0.1);
+          transform: scale(0.1); }
+  .reveal .slides section .fragment.zoom-in.visible {
+    -webkit-transform: none;
+            transform: none; }
+
+.reveal .slides section .fragment.fade-out {
+  opacity: 1;
+  visibility: visible; }
+  .reveal .slides section .fragment.fade-out.visible {
+    opacity: 0;
+    visibility: hidden; }
+
+.reveal .slides section .fragment.semi-fade-out {
+  opacity: 1;
+  visibility: visible; }
+  .reveal .slides section .fragment.semi-fade-out.visible {
+    opacity: 0.5;
+    visibility: visible; }
+
+.reveal .slides section .fragment.strike {
+  opacity: 1;
+  visibility: visible; }
+  .reveal .slides section .fragment.strike.visible {
+    text-decoration: line-through; }
+
+.reveal .slides section .fragment.fade-up {
+  -webkit-transform: translate(0, 20%);
+          transform: translate(0, 20%); }
+  .reveal .slides section .fragment.fade-up.visible {
+    -webkit-transform: translate(0, 0);
+            transform: translate(0, 0); }
+
+.reveal .slides section .fragment.fade-down {
+  -webkit-transform: translate(0, -20%);
+          transform: translate(0, -20%); }
+  .reveal .slides section .fragment.fade-down.visible {
+    -webkit-transform: translate(0, 0);
+            transform: translate(0, 0); }
+
+.reveal .slides section .fragment.fade-right {
+  -webkit-transform: translate(-20%, 0);
+          transform: translate(-20%, 0); }
+  .reveal .slides section .fragment.fade-right.visible {
+    -webkit-transform: translate(0, 0);
+            transform: translate(0, 0); }
+
+.reveal .slides section .fragment.fade-left {
+  -webkit-transform: translate(20%, 0);
+          transform: translate(20%, 0); }
+  .reveal .slides section .fragment.fade-left.visible {
+    -webkit-transform: translate(0, 0);
+            transform: translate(0, 0); }
+
+.reveal .slides section .fragment.current-visible {
+  opacity: 0;
+  visibility: hidden; }
+  .reveal .slides section .fragment.current-visible.current-fragment {
+    opacity: 1;
+    visibility: visible; }
+
+.reveal .slides section .fragment.highlight-red,
+.reveal .slides section .fragment.highlight-current-red,
+.reveal .slides section .fragment.highlight-green,
+.reveal .slides section .fragment.highlight-current-green,
+.reveal .slides section .fragment.highlight-blue,
+.reveal .slides section .fragment.highlight-current-blue {
+  opacity: 1;
+  visibility: visible; }
+
+.reveal .slides section .fragment.highlight-red.visible {
+  color: #ff2c2d; }
+
+.reveal .slides section .fragment.highlight-green.visible {
+  color: #17ff2e; }
+
+.reveal .slides section .fragment.highlight-blue.visible {
+  color: #1b91ff; }
+
+.reveal .slides section .fragment.highlight-current-red.current-fragment {
+  color: #ff2c2d; }
+
+.reveal .slides section .fragment.highlight-current-green.current-fragment {
+  color: #17ff2e; }
+
+.reveal .slides section .fragment.highlight-current-blue.current-fragment {
+  color: #1b91ff; }
+
+/*********************************************
+ * DEFAULT ELEMENT STYLES
+ *********************************************/
+/* Fixes issue in Chrome where italic fonts did not appear when printing to PDF */
+.reveal:after {
+  content: '';
+  font-style: italic; }
+
+.reveal iframe {
+  z-index: 1; }
+
+/** Prevents layering issues in certain browser/transition combinations */
+.reveal a {
+  position: relative; }
+
+.reveal .stretch {
+  max-width: none;
+  max-height: none; }
+
+.reveal pre.stretch code {
+  height: 100%;
+  max-height: 100%;
+  box-sizing: border-box; }
+
+/*********************************************
+ * CONTROLS
+ *********************************************/
+.reveal .controls {
+  display: none;
+  position: fixed;
+  width: 110px;
+  height: 110px;
+  z-index: 30;
+  right: 10px;
+  bottom: 10px;
+  -webkit-user-select: none; }
+
+.reveal .controls button {
+  padding: 0;
+  position: absolute;
+  opacity: 0.05;
+  width: 0;
+  height: 0;
+  background-color: transparent;
+  border: 12px solid transparent;
+  -webkit-transform: scale(0.9999);
+          transform: scale(0.9999);
+  -webkit-transition: all 0.2s ease;
+          transition: all 0.2s ease;
+  -webkit-appearance: none;
+  -webkit-tap-highlight-color: transparent; }
+
+.reveal .controls .enabled {
+  opacity: 0.7;
+  cursor: pointer; }
+
+.reveal .controls .enabled:active {
+  margin-top: 1px; }
+
+.reveal .controls .navigate-left {
+  top: 42px;
+  border-right-width: 22px;
+  border-right-color: #000; }
+
+.reveal .controls .navigate-left.fragmented {
+  opacity: 0.3; }
+
+.reveal .controls .navigate-right {
+  left: 74px;
+  top: 42px;
+  border-left-width: 22px;
+  border-left-color: #000; }
+
+.reveal .controls .navigate-right.fragmented {
+  opacity: 0.3; }
+

From pypy.commits at gmail.com  Mon Sep 26 09:19:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 26 Sep 2016 06:19:58 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: assign me a new task
Message-ID: <57e9207e.8270c20a.8d572.181b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5716:16a9983d1c9a
Date: 2016-09-26 15:19 +0200
http://bitbucket.org/pypy/extradoc/changeset/16a9983d1c9a/

Log:	assign me a new task

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -5,14 +5,7 @@
 In-progress ("Lock" section)
 ----------------------------
 
-* richard: Implement changes to memory view. e.g. hex(): https://bugs.python.org/issue9951
-  Seems to work, but test suite hangs to verify the CPython tests.
-* richard: tuple indexing for memory view,
-  Comments: Stronly tied to numpy. Hard to implement, because most of the basics are missing (dimensions/strides)
-  We should make a plan to impl. that on default with cpyext support and merge it back to py3.5.
-  Matti's opinion on that would be great!
-* richard: extended slicing for memory view
-* richard: bytes % args, bytearray % args: PEP 461
+* richard: bz2, lzma, ... changes (cpython issue 15955)
 
 * arigo: faulthandler module
 
@@ -46,7 +39,6 @@
   'array' is typically an extension module; at least that's the case on
   Linux with default compilation settings).
 
-
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------
 

From pypy.commits at gmail.com  Mon Sep 26 11:37:41 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 26 Sep 2016 08:37:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: adding properties and new
 parameters in py3.5 (work in progress)
Message-ID: <57e940c5.669fc20a.465be.a45f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87393:3bc7c28ee3a9
Date: 2016-09-26 17:36 +0200
http://bitbucket.org/pypy/pypy/changeset/3bc7c28ee3a9/

Log:	adding properties and new parameters in py3.5 (work in progress)

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -96,9 +96,11 @@
 BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR
 
 if BUFSIZ < 8192:
-    SMALLCHUNK = 8192
+    INITIAL_BUFFER_SIZE = 8192
 else:
-    SMALLCHUNK = BUFSIZ
+    INITIAL_BUFFER_SIZE = 8192
+
+UINT_MAX = 2**32-1
 
 if rffi.sizeof(rffi.INT) > 4:
     BIGCHUNK = 512 * 32
@@ -187,12 +189,21 @@
     encapsulate the logic of setting up the fields of 'bzs' and
     allocating raw memory as needed.
     """
-    def __init__(self, bzs, initial_size=SMALLCHUNK):
+    def __init__(self, bzs, initial_size=INITIAL_BUFFER_SIZE, max_length=-1):
         # when the constructor is called, allocate a piece of memory
         # of length 'piece_size' and make bzs ready to dump there.
         self.temp = []
         self.bzs = bzs
-        self._allocate_chunk(initial_size)
+        self.max_length = max_length
+        if max_length < 0 or max_length >= initial_size:
+            size = initial_size
+        else:
+            size = max_length
+        self._allocate_chunk(size)
+        self.avail_in_real = 0
+
+    def get_data_size(self):
+        return 0
 
     def _allocate_chunk(self, size):
         self.raw_buf, self.gc_buf, self.case_num = rffi.alloc_buffer(size)
@@ -357,7 +368,6 @@
     W_BZ2Decompressor.__init__(x, space)
     return space.wrap(x)
 
-
 class W_BZ2Decompressor(W_Root):
     """BZ2Decompressor() -> decompressor object
 
@@ -372,6 +382,8 @@
         try:
             self.running = False
             self.unused_data = ""
+            self.needs_input = 1
+            self.input_buffer = None
 
             self._init_bz2decomp()
         except:
@@ -397,15 +409,47 @@
     def descr_getstate(self):
         raise oefmt(self.space.w_TypeError, "cannot serialize '%T' object", self)
 
+    def needs_input_w(self, space):
+        """ True if more input is needed before more decompressed
+            data can be produced. """
+        return space.wrap(self.needs_input)
+
     def eof_w(self, space):
         if self.running:
             return space.w_False
         else:
             return space.w_True
 
-    @unwrap_spec(data='bufferstr')
-    def decompress(self, data):
-        """decompress(data) -> string
+    def _decompress_buf(self, data, max_length):
+        in_bufsize = len(data)
+
+        with rffi.scoped_nonmovingbuffer(data) as in_buf:
+            self.bzs.c_next_in = in_buf
+            rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
+
+            with OutBuffer(self.bzs, max_length=max_length) as out:
+                while True:
+                    bzerror = BZ2_bzDecompress(self.bzs)
+                    if bzerror == BZ_STREAM_END:
+                        self.running = False
+                        break
+                    if bzerror != BZ_OK:
+                        _catch_bz2_error(self.space, bzerror)
+
+                    if rffi.getintfield(self.bzs, 'c_avail_in') == 0:
+                        break
+                    elif rffi.getintfield(self.bzs, 'c_avail_out') == 0:
+                        if out.get_data_size() == max_length:
+                            break
+                        out.prepare_next_chunk()
+                res = out.make_result_string()
+                # might be non zero if max_length has been specified
+                self.left_to_process = out.left
+                return self.space.newbytes(res)
+
+    @unwrap_spec(data='bufferstr', max_length=int)
+    def decompress(self, data, max_length=-1):
+        """decompress(data, max_length=-1) -> bytes
 
         Provide more data to the decompressor object. It will return chunks
         of decompressed data whenever possible. If you try to decompress data
@@ -419,34 +463,27 @@
         if data == '':
             return self.space.newbytes('')
 
-        in_bufsize = len(data)
+        bzs = self.bzs
+        if not self.input_buffer:
+            input_buffer_in_use = True
+            result = self._decompress_buf(self.input_buffer, max_length)
+        else:
+            input_buffer_in_use = False
+            result = self._decompress_buf(data, max_length)
 
-        with rffi.scoped_nonmovingbuffer(data) as in_buf:
-            self.bzs.c_next_in = in_buf
-            rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
+        if self.left_to_process == 0:
+            self.input_buffer = None
+            self.need_input = 1
+        else:
+            self.need_input = 0
+            if not input_buffer_in_use:
+                datalen = len(data)
+                self.input_buffer = data[datalen-self.left_to_process-1:]
 
-            with OutBuffer(self.bzs) as out:
-                while True:
-                    bzerror = BZ2_bzDecompress(self.bzs)
-                    if bzerror == BZ_STREAM_END:
-                        if rffi.getintfield(self.bzs, 'c_avail_in') != 0:
-                            unused = [self.bzs.c_next_in[i]
-                                      for i in range(
-                                          rffi.getintfield(self.bzs,
-                                                           'c_avail_in'))]
-                            self.unused_data = "".join(unused)
-                        self.running = False
-                        break
-                    if bzerror != BZ_OK:
-                        _catch_bz2_error(self.space, bzerror)
+        return result
 
-                    if rffi.getintfield(self.bzs, 'c_avail_in') == 0:
-                        break
-                    elif rffi.getintfield(self.bzs, 'c_avail_out') == 0:
-                        out.prepare_next_chunk()
 
-                res = out.make_result_string()
-                return self.space.newbytes(res)
+
 
 
 W_BZ2Decompressor.typedef = TypeDef("_bz2.BZ2Decompressor",
@@ -456,5 +493,6 @@
     unused_data = interp_attrproperty_bytes("unused_data", W_BZ2Decompressor),
     eof = GetSetProperty(W_BZ2Decompressor.eof_w),
     decompress = interp2app(W_BZ2Decompressor.decompress),
+    needs_input = GetSetProperty(W_BZ2Decompressor.needs_input_w),
 )
 W_BZ2Decompressor.typedef.acceptable_as_base_class = False

From pypy.commits at gmail.com  Mon Sep 26 11:52:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 08:52:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix what occurs near the limit
 MAX_STACK_DEPTH: I *think* it would
Message-ID: <57e94452.cacb1c0a.d5d13.a68d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87394:a6e7414f79c7
Date: 2016-09-26 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/a6e7414f79c7/

Log:	Fix what occurs near the limit MAX_STACK_DEPTH: I *think* it would
	appear to skip some frames then, in the JIT case

diff --git a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
@@ -4,17 +4,18 @@
                                 long *current_pos_addr);
 
 
+#define MAX_INLINE_DEPTH  384
+
+
 static long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
-                                             intptr_t ip, int max_depth)
+                                             intptr_t addr, int max_depth)
 {
 #ifdef PYPY_JIT_CODEMAP
     void *codemap;
     long current_pos = 0;
-    intptr_t ident;
+    intptr_t ident, local_stack[MAX_INLINE_DEPTH];
+    long m;
     long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    intptr_t tmp;
 
     codemap = pypy_find_codemap_at_addr(addr, &start_addr);
     if (codemap == NULL || n >= max_depth - 2)
@@ -25,24 +26,29 @@
     // in the middle
     result[n++] = VMPROF_ASSEMBLER_TAG;
     result[n++] = start_addr;
-    start = n;
-    while (n < max_depth) {
+
+    // build the list of code idents corresponding to the current
+    // position inside this particular piece of assembler.  If (very
+    // unlikely) we get more than MAX_INLINE_DEPTH recursion levels
+    // all inlined inside this single piece of assembler, then stop:
+    // there will be some missing frames then.  Otherwise, we need to
+    // first collect 'local_stack' and then write it to 'result' in the
+    // opposite order, stopping at 'max_depth'.  Previous versions of
+    // the code would write the oldest calls and then stop---whereas
+    // what we really need it to write the newest calls and then stop.
+    m = 0;
+    while (m < MAX_INLINE_DEPTH) {
         ident = pypy_yield_codemap_at_addr(codemap, addr, &current_pos);
         if (ident == -1)
             // finish
             break;
         if (ident == 0)
             continue; // not main codemap
+        local_stack[m++] = ident;
+    }
+    while (m > 0 && n < max_depth) {
         result[n++] = VMPROF_JITTED_TAG;
-        result[n++] = ident;
-    }
-    k = 1;
-
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k];
-        result[n - k] = tmp;
-        k += 2;
+        result[n++] = local_stack[--m];
     }
 #endif
     return n;

From pypy.commits at gmail.com  Mon Sep 26 12:03:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 09:03:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: add comment
Message-ID: <57e946c3.e70ec20a.eb5f2.6653@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87395:9e9d4ae8cad2
Date: 2016-09-26 18:02 +0200
http://bitbucket.org/pypy/pypy/changeset/9e9d4ae8cad2/

Log:	add comment

diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -138,6 +138,7 @@
             # shadowstack: change 'rpy_fastgil' to 0 (it should be
             # non-zero right now).
             self.change_extra_stack_depth = False
+            # ^^ note that set_extra_stack_depth() in this case is a no-op
             css_value = imm(0)
         else:
             from rpython.memory.gctransform import asmgcroot

From pypy.commits at gmail.com  Mon Sep 26 14:18:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 11:18:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: in-progress
Message-ID: <57e96674.e97bc20a.392d6.e843@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87396:80e8de7e078d
Date: 2016-09-26 20:17 +0200
http://bitbucket.org/pypy/pypy/changeset/80e8de7e078d/

Log:	in-progress

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -56,6 +56,11 @@
                                             [rffi.INT], lltype.Void,
                                             compilation_info=eci,
                                             _nowrapper=True)
+    vmprof_get_traceback = rffi.llexternal("vmprof_get_traceback",
+                                  [PVMPROFSTACK, lltype.Signed,
+                                   rffi.SIGNEDP, lltype.Signed],
+                                  lltype.Signed, compilation_info=eci,
+                                  _nowrapper=True)
 
     return CInterface(locals())
 
@@ -156,7 +161,7 @@
     vmprof_tl_stack.setraw(x)
 
 #
-# rvmprof.traceback support
+# traceback support
 
 def get_rvmprof_stack():
     return vmprof_tl_stack.get_or_make_raw()
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -8,3 +8,6 @@
 RPY_EXTERN int vmprof_stack_append(void*, long);
 RPY_EXTERN long vmprof_stack_pop(void*);
 RPY_EXTERN void vmprof_stack_free(void*);
+RPY_EXTERN intptr_t vmprof_get_traceback(void *, intptr_t, intptr_t*, intptr_t);
+
+#define RVMPROF_TRACEBACK_ESTIMATE_N(num_entries)  (2 * (num_entries) + 4)
diff --git a/rpython/rlib/rvmprof/src/vmprof_common.h b/rpython/rlib/rvmprof/src/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/vmprof_common.h
@@ -128,3 +128,14 @@
     return 0;
 }
 #endif
+
+RPY_EXTERN
+intptr_t vmprof_get_traceback(void *stack, intptr_t pc,
+                              intptr_t *result_p, intptr_t result_length)
+{
+    int n;
+    if (stack == NULL)
+        stack = get_vmprof_stack();
+    n = get_stack_trace(stack, result_p, result_length - 2, pc);
+    return (intptr_t)n;
+}
diff --git a/rpython/rlib/rvmprof/test/test_traceback.py b/rpython/rlib/rvmprof/test/test_traceback.py
--- a/rpython/rlib/rvmprof/test/test_traceback.py
+++ b/rpython/rlib/rvmprof/test/test_traceback.py
@@ -1,7 +1,8 @@
 import re
-from rpython.rlib import rvmprof
-from rpython.rlib.rvmprof.traceback import traceback
+from rpython.rlib import rvmprof, jit
+from rpython.rlib.rvmprof import traceback
 from rpython.translator.interactive import Translation
+from rpython.rtyper.lltypesystem import lltype
 
 
 def test_direct():
@@ -17,20 +18,21 @@
         if level > 0:
             mainloop(code, level - 1)
         else:
-            traceback(MyCode, my_callback, 42)
+            p, length = traceback.traceback(20)
+            traceback.walk_traceback(MyCode, my_callback, 42, p, length)
+            lltype.free(p, flavor='raw')
     #
     seen = []
-    def my_callback(depth, code, arg):
-        seen.append((depth, code, arg))
-        return 0
+    def my_callback(code, loc, arg):
+        seen.append((code, loc, arg))
     #
     code1 = MyCode()
     rvmprof.register_code(code1, "foo")
     mainloop(code1, 2)
     #
-    assert seen == [(0, code1, 42),
-                    (1, code1, 42),
-                    (2, code1, 42)]
+    assert seen == [(code1, traceback.LOC_INTERPRETED, 42),
+                    (code1, traceback.LOC_INTERPRETED, 42),
+                    (code1, traceback.LOC_INTERPRETED, 42)]
 
 def test_compiled():
     class MyCode:
@@ -44,10 +46,12 @@
         if level > 0:
             mainloop(code, level - 1)
         else:
-            traceback(MyCode, my_callback, 42)
+            p, length = traceback.traceback(20)
+            traceback.walk_traceback(MyCode, my_callback, 42, p, length)
+            lltype.free(p, flavor='raw')
 
-    def my_callback(depth, code, arg):
-        print depth, code, arg
+    def my_callback(code, loc, arg):
+        print code, loc, arg
         return 0
 
     def f(argv):
@@ -59,7 +63,49 @@
     t = Translation(f, None, gc="boehm")
     t.compile_c()
     stdout = t.driver.cbuilder.cmdexec('')
-    r = re.compile("(\d+) [<]MyCode object at 0x([0-9a-f]+)[>] 42\n")
+    r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] 0 42\n")
+    got = r.findall(stdout)
+    assert got == [got[0]] * 3
+
+def test_jitted():
+    class MyCode:
+        pass
+    def get_name(mycode):
+        raise NotImplementedError
+    rvmprof.register_code_object_class(MyCode, get_name)
+
+    jitdriver = jit.JitDriver(greens=[], reds='auto')
+
+    @rvmprof.vmprof_execute_code("mycode", lambda code, level, total_i: code)
+    def mainloop(code, level, total_i):
+        i = 20
+        while i > 0:
+            jitdriver.jit_merge_point()
+            i -= 1
+            if level > 0:
+                mainloop(code, level - 1, total_i + i)
+        if level == 0 and total_i == 0:
+            p, length = traceback.traceback(20)
+            traceback.walk_traceback(MyCode, my_callback, 42, p, length)
+            lltype.free(p, flavor='raw')
+
+    def my_callback(code, loc, arg):
+        print code, loc, arg
+        return 0
+
+    def f(argv):
+        jit.set_param(jitdriver, "inlining", 0)
+        code1 = MyCode()
+        rvmprof.register_code(code1, "foo")
+        mainloop(code1, 2, 0)
+        return 0
+
+    t = Translation(f, None, gc="boehm")
+    t.rtype()
+    t.driver.pyjitpl_lltype()
+    t.compile_c()
+    stdout = t.driver.cbuilder.cmdexec('')
+    r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] (\d) 42\n")
     got = r.findall(stdout)
     addr = got[0][1]
-    assert got == [("0", addr), ("1", addr), ("2", addr)]
+    assert got == XXX
diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py
--- a/rpython/rlib/rvmprof/traceback.py
+++ b/rpython/rlib/rvmprof/traceback.py
@@ -3,36 +3,62 @@
 from the faulthandler module.
 """
 
-from rpython.rlib.rvmprof import cintf
+from rpython.rlib.rvmprof import cintf, rvmprof
 from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+def traceback(estimate_number_of_entries):
+    """Build and return a vmprof-like traceback, as a pair (array_p,
+    array_length).  The caller must free array_p.  Not for signal handlers:
+    for these, call vmprof_get_traceback() from C code.
+    """
+    _cintf = rvmprof._get_vmprof().cintf
+    size = estimate_number_of_entries * 2 + 4
+    stack = cintf.get_rvmprof_stack()
+    array_p = lltype.malloc(rffi.SIGNEDP.TO, size, flavor='raw')
+    array_length = _cintf.vmprof_get_traceback(stack, 0, array_p, size)
+    return (array_p, array_length)
+
+
+LOC_INTERPRETED    = 0
+LOC_JITTED         = 1
+LOC_JITTED_INLINED = 2
 
 
 @specialize.arg(0, 1)
-def traceback(CodeClass, callback, arg):
-    """For each frame, invoke 'callback(depth, code_obj, arg)'.
-    'code_obj' is either a CodeClass, or None if we fail to determine
-    what it should be (shouldn't occur, but you never know).  If it
-    returns a non-null integer, stop and return that.  Otherwise,
-    continue.  If all callbacks return 0, this returns 0.
+def _traceback_one(CodeClass, callback, arg, code_id, loc):
+    found_code = None
+    if code_id != 0:
+        all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles()
+        i = len(all_code_wrefs) - 1
+        while i >= 0:
+            code = all_code_wrefs[i]()
+            if code is not None and code._vmprof_unique_id == code_id:
+                found_code = code
+                break
+            i -= 1
+    callback(found_code, loc, arg)
+
+ at specialize.arg(0, 1)
+def walk_traceback(CodeClass, callback, arg, array_p, array_length):
+    """Invoke 'callback(code_obj, loc, arg)' for every traceback entry.
+    'code_obj' may be None if it can't be determined.  'loc' is one
+    of the LOC_xxx constants.
     """
-    s = cintf.get_rvmprof_stack()
-    depth = 0
-    while s:
-        if s.c_kind == cintf.VMPROF_CODE_TAG:
-            code_id = s.c_value
-            found_code = None
-            if code_id != 0:
-                all_code_wrefs = CodeClass._vmprof_weak_list.get_all_handles()
-                i = len(all_code_wrefs) - 1
-                while i >= 0:
-                    code = all_code_wrefs[i]()
-                    if code is not None and code._vmprof_unique_id == code_id:
-                        found_code = code
-                        break
-                    i -= 1
-            result = callback(depth, found_code, arg)
-            if result != 0:
-                return result
-            depth += 1
-        s = s.c_next
-    return 0
+    i = 0
+    while i < array_length - 1:
+        tag = array_p[i]
+        tagged_value = array_p[i + 1]
+        if tag == rvmprof.VMPROF_CODE_TAG:
+            loc = LOC_INTERPRETED
+            _traceback_one(CodeClass, callback, arg, tagged_value, loc)
+        elif tag == rvmprof.VMPROF_JITTED_TAG:
+            if i + 2 >= array_length:  # skip last entry, can't determine if
+                break                  # it's LOC_JITTED_INLINED or LOC_JITTED
+            if array_p[i + 2] == rvmprof.VMPROF_JITTED_TAG:
+                loc = LOC_JITTED_INLINED
+            else:
+                loc = LOC_JITTED
+            _traceback_one(CodeClass, callback, arg, tagged_value, loc)
+        i += 2

From pypy.commits at gmail.com  Mon Sep 26 15:11:30 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 26 Sep 2016 12:11:30 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-null-slots: abandon making
 int(np.string_('abc')) raise a ValueError
Message-ID: <57e972e2.0b3b1c0a.dac6f.38dc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-null-slots
Changeset: r87397:d37191c0da88
Date: 2016-09-26 21:57 +0300
http://bitbucket.org/pypy/pypy/changeset/d37191c0da88/

Log:	abandon making int(np.string_('abc')) raise a ValueError

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -479,10 +479,6 @@
         if pto.c_tp_doc:
             self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
-    def lookup(self, name):
-        # do not traverse the mro, look only in self
-        return self.getdictvalue(self.space, name)
-
 @bootstrap_function
 def init_typeobject(space):
     make_typedescr(space.w_type.layout.typedef,
@@ -823,25 +819,17 @@
     # inheriting tp_as_* slots
     base = py_type.c_tp_base
     if base:
-        remap_slots = False
         if not py_type.c_tp_as_number: 
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
-            remap_slots = True
         if not py_type.c_tp_as_sequence:
             py_type.c_tp_as_sequence = base.c_tp_as_sequence
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
-            remap_slots = True
         if not py_type.c_tp_as_mapping:
             py_type.c_tp_as_mapping = base.c_tp_as_mapping
-            remap_slots = True
         if not py_type.c_tp_as_buffer:
             py_type.c_tp_as_buffer = base.c_tp_as_buffer
-            remap_slots = True
-        if remap_slots:
-            add_operators(space, w_obj.dict_w, py_type)
-
     return w_obj
 
 def finish_type_1(space, pto):

From pypy.commits at gmail.com  Mon Sep 26 15:11:33 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 26 Sep 2016 12:11:33 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head faca61b444b4
 on branch w-xor-x2
Message-ID: <57e972e5.0558c20a.a5182.ed20@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r87398:fd181c80ace7
Date: 2016-09-26 22:02 +0300
http://bitbucket.org/pypy/pypy/changeset/fd181c80ace7/

Log:	Merge closed head faca61b444b4 on branch w-xor-x2


From pypy.commits at gmail.com  Mon Sep 26 15:11:34 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 26 Sep 2016 12:11:34 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head ba39bd805638
 on branch zarch-simd-support
Message-ID: <57e972e6.c4371c0a.6b9ce.2d7d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r87399:c2e0070d0212
Date: 2016-09-26 22:02 +0300
http://bitbucket.org/pypy/pypy/changeset/c2e0070d0212/

Log:	Merge closed head ba39bd805638 on branch zarch-simd-support


From pypy.commits at gmail.com  Mon Sep 26 15:11:36 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 26 Sep 2016 12:11:36 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head d37191c0da88
 on branch cpyext-null-slots
Message-ID: <57e972e8.822e1c0a.adccd.32d3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r87400:3ded4861a366
Date: 2016-09-26 22:02 +0300
http://bitbucket.org/pypy/pypy/changeset/3ded4861a366/

Log:	Merge closed head d37191c0da88 on branch cpyext-null-slots


From pypy.commits at gmail.com  Mon Sep 26 15:11:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 26 Sep 2016 12:11:37 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: re-close this branch
Message-ID: <57e972e9.07d01c0a.c9eba.29a4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r87401:3c1fdb654106
Date: 2016-09-26 22:02 +0300
http://bitbucket.org/pypy/pypy/changeset/3c1fdb654106/

Log:	re-close this branch


From pypy.commits at gmail.com  Mon Sep 26 15:11:40 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 26 Sep 2016 12:11:40 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: merge default into branch
Message-ID: <57e972ec.e70ec20a.eb5f2.b247@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87402:46f3011f6b83
Date: 2016-09-26 22:07 +0300
http://bitbucket.org/pypy/pypy/changeset/46f3011f6b83/

Log:	merge default into branch

diff too long, truncating to 2000 out of 2477 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -13,6 +13,7 @@
 import sys
 import os
 import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -62,8 +63,7 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -75,8 +75,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,7 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break
+                break        
 
     if args:
         vals = []
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -22,7 +22,10 @@
     code, message = _ffi.getwinerror()
     raise WindowsError(code, message)
 
-_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+def _int2handle(val):
+    return _ffi.cast("HANDLE", val)
+
+_INVALID_HANDLE_VALUE = _int2handle(-1)
 
 class _handle(object):
     def __init__(self, c_handle):
@@ -70,9 +73,9 @@
     target = _ffi.new("HANDLE[1]")
 
     res = _kernel32.DuplicateHandle(
-        _ffi.cast("HANDLE", source_process),
-        _ffi.cast("HANDLE", source),
-        _ffi.cast("HANDLE", target_process),
+        _int2handle(source_process),
+        _int2handle(source),
+        _int2handle(target_process),
         target, access, inherit, options)
 
     if not res:
@@ -119,12 +122,14 @@
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
+    return (_handle(pi.hProcess),
+            _handle(pi.hThread),
+            pi.dwProcessId,
+            pi.dwThreadId)
 
 def WaitForSingleObject(handle, milliseconds):
     # CPython: the first argument is expected to be an integer.
-    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
-                                        milliseconds)
+    res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds)
     if res < 0:
         raise _WinError()
 
@@ -134,7 +139,7 @@
     # CPython: the first argument is expected to be an integer.
     code = _ffi.new("DWORD[1]")
 
-    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
+    res = _kernel32.GetExitCodeProcess(_int2handle(handle), code)
 
     if not res:
         raise _WinError()
@@ -144,7 +149,7 @@
 def TerminateProcess(handle, exitcode):
     # CPython: the first argument is expected to be an integer.
     # The second argument is silently wrapped in a UINT.
-    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+    res = _kernel32.TerminateProcess(_int2handle(handle),
                                      _ffi.cast("UINT", exitcode))
 
     if not res:
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.3
+Version: 1.8.4
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.3"
-__version_info__ = (1, 8, 3)
+__version__ = "1.8.4"
+__version_info__ = (1, 8, 4)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.3"
+                               "\ncompiled with cffi version: 1.8.4"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -332,7 +332,7 @@
                         realtype = model.unknown_ptr_type(decl.name)
                     else:
                         realtype, quals = self._get_type_and_quals(
-                            decl.type, name=decl.name)
+                            decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
                 else:
                     raise api.CDefError("unrecognized construct", decl)
@@ -781,11 +781,14 @@
                 exprnode.name in self._int_constants):
             return self._int_constants[exprnode.name]
         #
-        if partial_length_ok:
-            if (isinstance(exprnode, pycparser.c_ast.ID) and
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
                 self._partial_length = True
                 return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
         #
         raise api.FFIError(":%d: unsupported expression: expected a "
                            "simple numeric constant" % exprnode.coord.line)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -587,8 +587,11 @@
     # ----------
     # typedefs
 
+    def _typedef_type(self, tp, name):
+        return self._global_type(tp, "(*(%s *)0)" % (name,))
+
     def _generate_cpy_typedef_collecttype(self, tp, name):
-        self._do_collect_type(tp)
+        self._do_collect_type(self._typedef_type(tp, name))
 
     def _generate_cpy_typedef_decl(self, tp, name):
         pass
@@ -598,6 +601,7 @@
         self._lsts["typename"].append(TypenameExpr(name, type_index))
 
     def _generate_cpy_typedef_ctx(self, tp, name):
+        tp = self._typedef_type(tp, name)
         self._typedef_ctx(tp, name)
         if getattr(tp, "origin", None) == "unknown_type":
             self._struct_ctx(tp, tp.name, approxname=None)
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -34,9 +34,11 @@
    This function searches the PyPy standard library starting from the given
    "PyPy home directory".  The arguments are:
 
-   * ``home``: NULL terminated path to an executable inside the pypy directory
+   * ``home``: path to an executable inside the pypy directory
      (can be a .so name, can be made up).  Used to look up the standard
-     library, and is also set as ``sys.executable``.
+     library, and is also set as ``sys.executable``.  From PyPy 5.5, you can
+     just say NULL here, as long as the ``libpypy-c.so/dylib/dll`` is itself
+     inside this directory.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
@@ -82,18 +84,14 @@
 
 Note that this API is a lot more minimal than say CPython C API, so at first
 it's obvious to think that you can't do much. However, the trick is to do
-all the logic in Python and expose it via `cffi`_ callbacks. Let's assume
-we're on linux and pypy is installed in ``/opt/pypy`` (with
-subdirectories like ``lib-python`` and ``lib_pypy``), and with the
-library in ``/opt/pypy/bin/libpypy-c.so``.  (It doesn't need to be
-installed; you can also replace these paths with a local extract of the
-installation tarballs, or with your local checkout of pypy.) We write a
-little C program:
+all the logic in Python and expose it via `cffi`_ callbacks.
+We write a little C program:
 
 .. code-block:: c
 
     #include "PyPy.h"
     #include <stdio.h>
+    #include <stdlib.h>
 
     static char source[] = "print 'hello from pypy'";
 
@@ -102,9 +100,9 @@
         int res;
 
         rpython_startup_code();
-        /* note: in the path /opt/pypy/x, the final x is ignored and
-           replaced with lib-python and lib_pypy. */
-        res = pypy_setup_home("/opt/pypy/x", 1);
+        /* Before PyPy 5.5, you may need to say e.g. "/opt/pypy/bin" instead
+         * of NULL. */
+        res = pypy_setup_home(NULL, 1);
         if (res) {
             printf("Error setting pypy home!\n");
             return 1;
@@ -123,11 +121,6 @@
     $ LD_LIBRARY_PATH=/opt/pypy/bin ./x
     hello from pypy
 
-.. note:: If the compilation fails because of missing PyPy.h header file,
-          you are running PyPy <= 2.2.1.  Get it here__.
-
-.. __: https://bitbucket.org/pypy/pypy/raw/c4cd6eca9358066571500ac82aaacfdaa3889e8c/include/PyPy.h
-
 On OSX it is necessary to set the rpath of the binary if one wants to link to it,
 with a command like::
 
@@ -181,6 +174,7 @@
     /* C example */
     #include "PyPy.h"
     #include <stdio.h>
+    #include <stdlib.h>
 
     struct API {
         double (*add_numbers)(double x, double y);
@@ -196,7 +190,7 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/x", 1);
+        res = pypy_setup_home(NULL, 1);
         if (res) {
             fprintf(stderr, "Error setting pypy home!\n");
             return -1;
@@ -237,6 +231,8 @@
 Finding pypy_home
 -----------------
 
+**You can usually skip this section if you are running PyPy >= 5.5.**
+
 The function pypy_setup_home() takes as first parameter the path to a
 file from which it can deduce the location of the standard library.
 More precisely, it tries to remove final components until it finds
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -21,3 +21,14 @@
 JIT residual calls: if the called function starts with a fast-path
 like "if x.foo != 0: return x.foo", then inline the check before
 doing the CALL.  For now, string hashing is about the only case.
+
+.. branch: search-path-from-libpypy
+
+The compiled pypy now looks for its lib-python/lib_pypy path starting
+from the location of the *libpypy-c* instead of the executable. This is
+arguably more consistent, and also it is what occurs anyway if you're
+embedding pypy.  Linux distribution packagers, take note!  At a minimum,
+the ``libpypy-c.so`` must really be inside the path containing
+``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
+from somewhere else.  You no longer have to do the same with the
+``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -89,17 +89,20 @@
     def pypy_setup_home(ll_home, verbose):
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
-        if ll_home:
+        if ll_home and ord(ll_home[0]):
             home1 = rffi.charp2str(ll_home)
             home = os.path.join(home1, 'x') # <- so that 'll_home' can be
                                             # directly the root directory
+            dynamic = False
         else:
-            home = home1 = pypydir
-        w_path = pypy_find_stdlib(space, home)
+            home1 = "pypy's shared library location"
+            home = pypydir
+            dynamic = True
+        w_path = pypy_find_stdlib(space, home, dynamic)
         if space.is_none(w_path):
             if verbose:
                 debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
-                      " not found in '%s' or in any parent directory" % home1)
+                      " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.appexec([w_path], """(path):
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -374,11 +374,8 @@
         self._value = value
         self.setup(w_type)
 
-    def get_w_value(self, space):
-        w_value = self._w_value
-        if w_value is None:
-            self._w_value = w_value = space.wrap(self._value)
-        return w_value
+    def _compute_value(self, space):
+        return self._value
 
 @specialize.memo()
 def get_operr_class(valuefmt):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1019,23 +1019,32 @@
         old_sys_path = sys.path[:]
         old_cwd = os.getcwd()
 
-        sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
         if self.tmp_dir.startswith(self.trunkdir):
             skip('TMPDIR is inside the PyPy source')
-        os.chdir(self.tmp_dir)
+        sys.path.append(self.goal_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
+            os.chdir(self.tmp_dir)
+
+            # If we are running PyPy with a libpypy-c, the following
+            # lines find the stdlib anyway.  Otherwise, it is not found.
+            expected_found = (
+                '__pypy__' in sys.builtin_module_names and
+                sys.pypy_translation_info['translation.shared'])
+
             import app_main
-            app_main.setup_bootstrap_path(tmp_pypy_c)  # stdlib not found
+            app_main.setup_bootstrap_path(tmp_pypy_c)
             assert sys.executable == ''
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not expected_found:
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
             if not sys.platform == 'win32':
                 # an existing file is always 'executable' on windows
                 assert sys.executable == ''      # not executable!
-                assert sys.path == old_sys_path + [self.goal_dir]
+                if not expected_found:
+                    assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0755)
             app_main.setup_bootstrap_path(self.fake_exe)
@@ -1046,7 +1055,8 @@
             if newpath[0].endswith('__extensions__'):
                 newpath = newpath[1:]
             # we get at least 'expected_path', and maybe more (e.g.plat-linux2)
-            assert newpath[:len(self.expected_path)] == self.expected_path
+            if not expected_found:
+                assert newpath[:len(self.expected_path)] == self.expected_path
         finally:
             sys.path[:] = old_sys_path
             os.chdir(old_cwd)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.3"
+VERSION = "1.8.4"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -112,29 +112,7 @@
 #define _WIN32_WINNT 0x0501
 #include <windows.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  _MAX_PATH
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    HMODULE hModule = 0;
-    DWORD res;
-
-    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | 
-                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-                       (LPCTSTR)&_cffi_init, &hModule);
-
-    if (hModule == 0 ) {
-        _cffi_init_error("GetModuleHandleEx() failed", "");
-        return -1;
-    }
-    res = GetModuleFileName(hModule, output_home_path, CFFI_INIT_HOME_PATH_MAX);
-    if (res >= CFFI_INIT_HOME_PATH_MAX) {
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -155,28 +133,9 @@
 else:
 
     do_includes = r"""
-#include <dlfcn.h>
 #include <pthread.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  PATH_MAX
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    Dl_info info;
-    dlerror();   /* reset */
-    if (dladdr(&_cffi_init, &info) == 0) {
-        _cffi_init_error("dladdr() failed: ", dlerror());
-        return -1;
-    }
-    if (realpath(info.dli_fname, output_home_path) == NULL) {
-        perror("realpath() failed");
-        _cffi_init_error("realpath() failed", "");
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -201,14 +160,10 @@
 
 static void _cffi_init(void)
 {
-    char home[CFFI_INIT_HOME_PATH_MAX + 1];
-
     rpython_startup_code();
     RPyGilAllocate();
 
-    if (_cffi_init_home(home) != 0)
-        return;
-    if (pypy_setup_home(home, 1) != 0) {
+    if (pypy_setup_home(NULL, 1) != 0) {
         _cffi_init_error("pypy_setup_home() failed", "");
         return;
     }
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.3", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -507,6 +507,11 @@
     def test_bug_1(self):
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
-        q = ffi.new("char[]", "abcd")
+        q = ffi.new("char[]", b"abcd")
         p = ffi.cast("char(*)(void)", q)
         raises(TypeError, ffi.string, p)
+
+    def test_negative_array_size(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -8,7 +8,7 @@
 
 @unwrap_spec(cdef=str, module_name=str, source=str)
 def prepare(space, cdef, module_name, source, w_includes=None,
-            w_extra_source=None):
+            w_extra_source=None, w_min_version=None):
     try:
         import cffi
         from cffi import FFI            # <== the system one, which
@@ -16,8 +16,13 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 4, 0):
-        py.test.skip("system cffi module needs to be at least 1.4.0")
+    if w_min_version is None:
+        min_version = (1, 4, 0)
+    else:
+        min_version = tuple(space.unwrap(w_min_version))
+    if cffi.__version_info__ < min_version:
+        py.test.skip("system cffi module needs to be at least %s, got %s" % (
+            min_version, cffi.__version_info__))
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -1790,3 +1795,28 @@
                                 "void f(void) { }")
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
+
+    def test_typedef_array_dotdotdot(self):
+        ffi, lib = self.prepare("""
+            typedef int foo_t[...], bar_t[...];
+            int gv[...];
+            typedef int mat_t[...][...];
+            typedef int vmat_t[][...];
+            """,
+            "test_typedef_array_dotdotdot", """
+            typedef int foo_t[50], bar_t[50];
+            int gv[23];
+            typedef int mat_t[6][7];
+            typedef int vmat_t[][8];
+        """, min_version=(1, 8, 4))
+        assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+        assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+        assert len(ffi.new("foo_t")) == 50
+        assert len(ffi.new("bar_t")) == 50
+        assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+        assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+        assert len(ffi.new("mat_t")) == 6
+        assert len(ffi.new("mat_t")[3]) == 7
+        raises(ffi.error, ffi.sizeof, "vmat_t")
+        p = ffi.new("vmat_t", 4)
+        assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -30,28 +30,25 @@
             raise oefmt(space.w_TypeError,
                         "array.array() does not take keyword arguments")
 
-    w_initializer_type = None
-    w_initializer = None
-    if len(__args__.arguments_w) > 0:
-        w_initializer = __args__.arguments_w[0]
-        w_initializer_type = space.type(w_initializer)
     for tc in unroll_typecodes:
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-            if w_initializer is not None:
-                if w_initializer_type is space.w_str:
-                    a.descr_fromstring(space, w_initializer)
-                elif w_initializer_type is space.w_list:
-                    a.descr_fromlist(space, w_initializer)
-                else:
-                    a.extend(w_initializer, True)
             break
     else:
         raise oefmt(space.w_ValueError,
                     "bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or "
                     "d)")
 
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
+        if w_initializer_type is space.w_str:
+            a.descr_fromstring(space, w_initializer)
+        elif w_initializer_type is space.w_list:
+            a.descr_fromlist(space, w_initializer)
+        else:
+            a.extend(w_initializer, True)
     return a
 
 
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,6 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import widen
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -28,25 +27,25 @@
     """Return a new PyLongObject object from a C size_t, or NULL on
     failure.
     """
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.LONGLONG], PyObject)
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.newlong(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
     """Return a new PyLongObject object from a C unsigned long, or
     NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONGLONG], PyObject)
 def PyLong_FromUnsignedLongLong(space, val):
     """Return a new PyLongObject object from a C unsigned long long,
     or NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([PyObject], rffi.ULONG, error=-1)
 def PyLong_AsUnsignedLong(space, w_long):
@@ -203,7 +202,10 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.newlong(rffi.cast(ADDR, p))
+    value = rffi.cast(ADDR, p)    # signed integer
+    if value < 0:
+        return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
+    return space.wrap(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -1,5 +1,6 @@
 import sys, py
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import maxint
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
@@ -108,10 +109,26 @@
         lltype.free(overflow, flavor='raw')
 
     def test_as_voidptr(self, space, api):
+        # CPython returns an int (not a long) depending on the value
+        # passed to PyLong_FromVoidPtr().  In all cases, NULL becomes
+        # the int 0.
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
-        assert isinstance(w_l, W_LongObject)
-        assert space.unwrap(w_l) == 0L
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == 0
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
+        # Positive values also return an int (assuming, like always in
+        # PyPy, that an int is big enough to store any pointer).
+        p = rffi.cast(rffi.VOIDP, maxint)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == maxint
+        assert api.PyLong_AsVoidPtr(w_l) == p
+        # Negative values always return a long.
+        p = rffi.cast(rffi.VOIDP, -maxint-1)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_long)
+        assert space.unwrap(w_l) == maxint+1
+        assert api.PyLong_AsVoidPtr(w_l) == p
 
     def test_sign_and_bits(self, space, api):
         if space.is_true(space.lt(space.sys.get('version_info'),
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -226,13 +226,13 @@
     w_keywords = space.newdict()
     stat_float_times = space.fromcache(StatState).stat_float_times
     for i, (name, TYPE) in FIELDS:
-        value = getattr(st, name)
-        if name in ('st_atime', 'st_mtime', 'st_ctime'):
-            value = int(value)   # rounded to an integer for indexed access
-        w_value = space.wrap(value)
         if i < rposix_stat.N_INDEXABLE_FIELDS:
+            # get the first 10 items by indexing; this gives us
+            # 'st_Xtime' as an integer, too
+            w_value = space.wrap(st[i])
             lst[i] = w_value
-        else:
+        elif name.startswith('st_'):    # exclude 'nsec_Xtime'
+            w_value = space.wrap(getattr(st, name))
             space.setitem(w_keywords, space.wrap(name), w_value)
 
     # non-rounded values for name-based access
@@ -243,13 +243,8 @@
                       space.wrap('st_mtime'), space.wrap(st.st_mtime))
         space.setitem(w_keywords,
                       space.wrap('st_ctime'), space.wrap(st.st_ctime))
-    else:
-        space.setitem(w_keywords,
-                      space.wrap('st_atime'), space.wrap(int(st.st_atime)))
-        space.setitem(w_keywords,
-                      space.wrap('st_mtime'), space.wrap(int(st.st_mtime)))
-        space.setitem(w_keywords,
-                      space.wrap('st_ctime'), space.wrap(int(st.st_ctime)))
+    #else:
+    #   filled by the __init__ method
 
     w_tuple = space.newtuple(lst)
     w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -129,9 +129,9 @@
         assert st[4] == st.st_uid
         assert st[5] == st.st_gid
         assert st[6] == st.st_size
-        assert st[7] == int(st.st_atime)
-        assert st[8] == int(st.st_mtime)
-        assert st[9] == int(st.st_ctime)
+        assert st[7] == int(st.st_atime)   # in complete corner cases, rounding
+        assert st[8] == int(st.st_mtime)   # here could maybe get the wrong
+        assert st[9] == int(st.st_ctime)   # integer...
 
         assert stat.S_IMODE(st.st_mode) & stat.S_IRUSR
         assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR
@@ -141,13 +141,12 @@
         assert st.st_size == 14
         assert st.st_nlink == 1
 
-        #if sys.platform.startswith('linux'):
-        #    # expects non-integer timestamps - it's unlikely that they are
-        #    # all three integers
-        #    assert ((st.st_atime, st.st_mtime, st.st_ctime) !=
-        #            (st[7],       st[8],       st[9]))
-        #    assert st.st_blksize * st.st_blocks >= st.st_size
+        assert not hasattr(st, 'nsec_atime')
+
         if sys.platform.startswith('linux'):
+            assert isinstance(st.st_atime, float)
+            assert isinstance(st.st_mtime, float)
+            assert isinstance(st.st_ctime, float)
             assert hasattr(st, 'st_rdev')
 
     def test_stat_float_times(self):
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -9,6 +9,8 @@
 
 from rpython.rlib import rpath
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.sys.state import get as get_state
@@ -155,8 +157,13 @@
     return space.wrap(resolvedirof(filename))
 
 
- at unwrap_spec(executable='str0')
-def pypy_find_stdlib(space, executable):
+ at unwrap_spec(executable='str0', dynamic=int)
+def pypy_find_stdlib(space, executable, dynamic=1):
+    if dynamic and space.config.translation.shared:
+        dynamic_location = pypy_init_home()
+        if dynamic_location:
+            executable = rffi.charp2str(dynamic_location)
+            pypy_init_free(dynamic_location)
     path, prefix = find_stdlib(get_state(space), executable)
     if path is None:
         return space.w_None
@@ -164,3 +171,75 @@
     space.setitem(space.sys.w_dict, space.wrap('prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), w_prefix)
     return space.newlist([space.wrap(p) for p in path])
+
+
+# ____________________________________________________________
+
+
+if os.name == 'nt':
+
+    _source_code = r"""
+#define _WIN32_WINNT 0x0501
+#include <windows.h>
+#include <stdio.h>
+
+RPY_EXPORTED
+char *_pypy_init_home(void)
+{
+    HMODULE hModule = 0;
+    DWORD res;
+    char *p;
+
+    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                       (LPCTSTR)&_pypy_init_home, &hModule);
+
+    if (hModule == 0 ) {
+        fprintf(stderr, "PyPy initialization: GetModuleHandleEx() failed\n");
+        return NULL;
+    }
+    p = malloc(_MAX_PATH);
+    if (p == NULL)
+        return NULL;
+    res = GetModuleFileName(hModule, p, _MAX_PATH);
+    if (res >= _MAX_PATH || res <= 0) {
+        free(p);
+        fprintf(stderr, "PyPy initialization: GetModuleFileName() failed\n");
+        return NULL;
+    }
+    return p;
+}
+"""
+
+else:
+
+    _source_code = r"""
+#include <dlfcn.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+RPY_EXPORTED
+char *_pypy_init_home(void)
+{
+    Dl_info info;
+    dlerror();   /* reset */
+    if (dladdr(&_pypy_init_home, &info) == 0) {
+        fprintf(stderr, "PyPy initialization: dladdr() failed: %s\n",
+                dlerror());
+        return NULL;
+    }
+    char *p = realpath(info.dli_fname, NULL);
+    if (p == NULL) {
+        p = strdup(info.dli_fname);
+    }
+    return p;
+}
+"""
+
+_eci = ExternalCompilationInfo(separate_module_sources=[_source_code])
+
+pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP,
+                                 _nowrapper=True, compilation_info=_eci)
+pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void,
+                                 _nowrapper=True, compilation_info=_eci)
diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py
--- a/pypy/module/sys/test/test_initpath.py
+++ b/pypy/module/sys/test/test_initpath.py
@@ -1,8 +1,10 @@
 import py
 import os.path
-from pypy.module.sys.initpath import (compute_stdlib_path, find_executable, find_stdlib,
-                                      resolvedirof)
+from pypy.module.sys.initpath import (compute_stdlib_path, find_executable,
+                                      find_stdlib, resolvedirof,
+                                      pypy_init_home, pypy_init_free)
 from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
+from rpython.rtyper.lltypesystem import rffi
 
 def build_hierarchy(prefix):
     dirname = '%d.%d' % CPYTHON_VERSION[:2]
@@ -10,7 +12,7 @@
     b = prefix.join('lib-python', dirname).ensure(dir=1)
     return a, b
 
-def test_find_stdlib(tmpdir, monkeypatch):
+def test_find_stdlib(tmpdir):
     bin_dir = tmpdir.join('bin').ensure(dir=True)
     pypy = bin_dir.join('pypy').ensure(file=True)
     build_hierarchy(tmpdir)
@@ -33,6 +35,14 @@
     path, prefix = find_stdlib(None, str(pypy_sym))
     assert prefix == pypydir
 
+def test_pypy_init_home():
+    p = pypy_init_home()
+    assert p
+    s = rffi.charp2str(p)
+    pypy_init_free(p)
+    print s
+    assert os.path.exists(s)
+
 def test_compute_stdlib_path(tmpdir):
     dirs = build_hierarchy(tmpdir)
     path = compute_stdlib_path(None, str(tmpdir))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -480,3 +480,7 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_negative_array_size(self):
+        ffi = FFI()
+        py.test.raises(ValueError, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py
@@ -503,3 +503,7 @@
     assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
     p = ffi.new("int[]", [-123456789])
     assert ffi.unpack(p, 1) == [-123456789]
+
+def test_negative_array_size():
+    ffi = _cffi1_backend.FFI()
+    py.test.raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1981,3 +1981,29 @@
         static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
     """)
     assert lib.f1(52).a == 52
+
+def test_typedef_array_dotdotdot():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef int foo_t[...], bar_t[...];
+        int gv[...];
+        typedef int mat_t[...][...];
+        typedef int vmat_t[][...];
+        """)
+    lib = verify(ffi, "test_typedef_array_dotdotdot", """
+        typedef int foo_t[50], bar_t[50];
+        int gv[23];
+        typedef int mat_t[6][7];
+        typedef int vmat_t[][8];
+    """)
+    assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+    assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+    assert len(ffi.new("foo_t")) == 50
+    assert len(ffi.new("bar_t")) == 50
+    assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+    assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+    assert len(ffi.new("mat_t")) == 6
+    assert len(ffi.new("mat_t")[3]) == 7
+    py.test.raises(ffi.error, ffi.sizeof, "vmat_t")
+    p = ffi.new("vmat_t", 4)
+    assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -270,6 +270,10 @@
             return W_SmallLongObject.fromint(val)
         return W_LongObject.fromint(self, val)
 
+    @specialize.argtype(1)
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+        return W_LongObject.fromrarith_int(val)
+
     def newlong_from_rbigint(self, val):
         return newlong(self, val)
 
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -25,7 +25,6 @@
         space.raises_w(space.w_OverflowError, space.float_w, w_big)
 
     def test_rint_variants(self):
-        py.test.skip("XXX broken!")
         from rpython.rtyper.tool.rfficache import platform
         space = self.space
         for r in platform.numbertype_to_rclass.values():
@@ -36,8 +35,8 @@
             for x in values:
                 if not r.SIGNED:
                     x &= r.MASK
-                w_obj = space.wrap(r(x))
-                assert space.bigint_w(w_obj).eq(rbigint.fromint(x))
+                w_obj = space.newlong_from_rarith_int(r(x))
+                assert space.bigint_w(w_obj).eq(rbigint.fromlong(x))
 
 
 class AppTestLong:
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -797,9 +797,7 @@
         class AA(object):
             __slots__ = ('a',)
         aa = AA()
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "aa.__class__ = A")
+        aa.__class__ = A
         raises(TypeError, "aa.__class__ = object")
         class Z1(A):
             pass
@@ -861,9 +859,13 @@
             __slots__ = ['a', 'b']
         class Order2(object):
             __slots__ = ['b', 'a']
-        # the following line works on CPython >= 2.6 but not on PyPy.
-        # but see below for more
-        raises(TypeError, "Order1().__class__ = Order2")
+        Order1().__class__ = Order2
+
+        # like CPython, the order of slot names doesn't matter
+        x = Order1()
+        x.a, x.b = 1, 2
+        x.__class__ = Order2
+        assert (x.a, x.b) == (1, 2)
 
         class U1(object):
             __slots__ = ['a', 'b']
@@ -873,10 +875,11 @@
             __slots__ = ['a', 'b']
         class V2(V1):
             __slots__ = ['c', 'd', 'e']
-        # the following line does not work on CPython >= 2.6 either.
-        # that's just obscure.  Really really.  So we just ignore
-        # the whole issue until someone comes complaining.  Then we'll
-        # just kill slots altogether apart from maybe doing a few checks.
+        # the following line does not work on CPython either: we can't
+        # change a class if the old and new class have different layouts
+        # that look compatible but aren't, because they don't have the
+        # same base-layout class (even if these base classes are
+        # themselves compatible)...  obscure.
         raises(TypeError, "U2().__class__ = V2")
 
     def test_name(self):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,9 +102,10 @@
     """
     _immutable_ = True
 
-    def __init__(self, typedef, nslots, base_layout=None):
+    def __init__(self, typedef, nslots, newslotnames=[], base_layout=None):
         self.typedef = typedef
         self.nslots = nslots
+        self.newslotnames = newslotnames[:]    # make a fixed-size list
         self.base_layout = base_layout
 
     def issublayout(self, parent):
@@ -114,6 +115,12 @@
                 return False
         return True
 
+    def expand(self, hasdict, weakrefable):
+        """Turn this Layout into a tuple.  If two classes get equal
+        tuples, it means their instances have a fully compatible layout."""
+        return (self.typedef, self.newslotnames, self.base_layout,
+                hasdict, weakrefable)
+
 
 # possible values of compares_by_identity_status
 UNKNOWN = 0
@@ -289,8 +296,7 @@
 
     # compute a tuple that fully describes the instance layout
     def get_full_instance_layout(self):
-        layout = self.layout
-        return (layout, self.hasdict, self.weakrefable)
+        return self.layout.expand(self.hasdict, self.weakrefable)
 
     def compute_default_mro(self):
         return compute_C3_mro(self.space, self)
@@ -1001,11 +1007,15 @@
         w_self.weakrefable = w_self.weakrefable or w_base.weakrefable
     return hasoldstylebase
 
+
 def create_all_slots(w_self, hasoldstylebase, w_bestbase, force_new_layout):
+    from pypy.objspace.std.listobject import StringSort
+
     base_layout = w_bestbase.layout
     index_next_extra_slot = base_layout.nslots
     space = w_self.space
     dict_w = w_self.dict_w
+    newslotnames = []
     if '__slots__' not in dict_w:
         wantdict = True
         wantweakref = True
@@ -1031,8 +1041,22 @@
                                 "__weakref__ slot disallowed: we already got one")
                 wantweakref = True
             else:
-                index_next_extra_slot = create_slot(w_self, slot_name,
-                                                    index_next_extra_slot)
+                newslotnames.append(slot_name)
+        # Sort the list of names collected so far
+        sorter = StringSort(newslotnames, len(newslotnames))
+        sorter.sort()
+        # Try to create all slots in order.  The creation of some of
+        # them might silently fail; then we delete the name from the
+        # list.  At the end, 'index_next_extra_slot' has been advanced
+        # by the final length of 'newslotnames'.
+        i = 0
+        while i < len(newslotnames):
+            if create_slot(w_self, newslotnames[i], index_next_extra_slot):
+                index_next_extra_slot += 1
+                i += 1
+            else:
+                del newslotnames[i]
+    #
     wantdict = wantdict or hasoldstylebase
     if wantdict:
         create_dict_slot(w_self)
@@ -1041,11 +1065,12 @@
     if '__del__' in dict_w:
         w_self.hasuserdel = True
     #
+    assert index_next_extra_slot == base_layout.nslots + len(newslotnames)
     if index_next_extra_slot == base_layout.nslots and not force_new_layout:
         return base_layout
     else:
         return Layout(base_layout.typedef, index_next_extra_slot,
-                      base_layout=base_layout)
+                      newslotnames, base_layout=base_layout)
 
 def create_slot(w_self, slot_name, index_next_extra_slot):
     space = w_self.space
@@ -1058,9 +1083,10 @@
         slot_name = space.str_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
         member = Member(index_next_extra_slot, slot_name, w_self)
-        index_next_extra_slot += 1
         w_self.dict_w[slot_name] = space.wrap(member)
-    return index_next_extra_slot
+        return True
+    else:
+        return False
 
 def create_dict_slot(w_self):
     if not w_self.hasdict:
diff --git a/rpython/doc/jit/backend.rst b/rpython/doc/jit/backend.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/jit/backend.rst
@@ -0,0 +1,263 @@
+=========================
+PyPy's assembler backends
+=========================
+
+Draft notes about the organization of assembler backends in the PyPy JIT, in 2016
+=================================================================================
+
+
+input: linear sequence of instructions, called a "trace".
+
+A trace is a sequence of instructions in SSA form.  Most instructions
+correspond to one or a few CPU-level instructions.  There are a few
+meta-instructions like `label` and debugging stuff.  All branching is
+done with guards, which are instructions that check that a condition is
+true and exit the trace if not.  A failing guard can have a new trace
+added to it later, called a "bridge".  A patched guard becomes a direct
+`Jcond` instruction going to the bridge, with no indirection, no
+register spilling, etc.
+
+A trace ends with either a `return` or a `jump to label`.  The target
+label is either inside the same trace, or in some older one.  For
+historical reasons we call a "loop" a trace that is not a bridge.  The
+machine code that we generate is organized as a forest of trees; the
+trunk of the tree is a "loop", and the branches are all bridges
+(branching off the trunk or off another branch).
+
+* every trunk or branch that ends in a `jump to label` can target a
+  label from a different tree, too.
+
+* the whole process of assembling a loop or a branch is basically
+  single-threaded, so no synchronization issue there (including to patch
+  older generated instructions).
+
+* the generated assembler has got a "frame" in %rbp, which is actually
+  not on the stack at all, but is a GC object (called a "jitframe").
+  Spilling goes there.
+
+* the guards are `Jcond` to a very small piece of generated code, which
+  is basically pushing a couple of constants on the stack and then
+  jumping to the general guard-recovery code.  That code will save the
+  registers into the jitframe and then exit the whole generated
+  function.  The caller of that generated function checks how it
+  finished: if it finished by hitting a guard, then the caller is
+  responsible for calling the "blackhole interpreter".  This is the part
+  of the front-end that recovers from failing guards and finishes
+  running the frame (including, possibly, by jumping again into
+  generated assembler).
+
+
+Details about the JITting process:
+
+* front-end and optimization pass
+
+* rewrite (includes gc related transformation as well as simplifactions)
+
+* assembler generation
+
+
+Front-end and optimization pass
+-------------------------------
+
+Not discussed here in detail.  This produces loops and bridges using an
+instruction set that is "high-level" in some sense: it contains
+intructions like "new"/"new_array", and
+"setfield"/"setarrayitem"/"setinteriorfield" which describe the action
+of storing a value in a precise field of the structure or array.  For
+example, the "setfield" action might require implicitly a GC write
+barrier.  This is the high-level trace that we send to the following
+step.
+
+
+Rewrite
+-------
+
+A mostly but not completely CPU-independent phase: lowers some
+instructions.  For example, the variants of "new" are lowered to
+"malloc" and a few "gc_store": it bumps the pointer of the GC and then
+sets a few fields explicitly in the newly allocated structure.  The
+"setfield" is replaced with a "cond_gc_wb_call" (conditional call to the
+write barrier) if needed, followed by a "gc_store".
+
+The "gc_store" instruction can be encoded in a single MOV assembler
+instruction, but is not as flexible as a MOV.  The address is always
+specified as "some GC pointer + an offset".  We don't have the notion of
+interior pointer for GC objects.
+
+A different instruction, "gc_store_indexed", offers additional operands,
+which can be mapped to a single MOV instruction using forms like
+`[rax+8*rcx+24]`.
+
+Some other complex instructions pass through to the backend, which must
+deal with them: for example, "card marking" in the GC.  (Writing an
+object pointer inside an array would require walking the whole array
+later to find "young" references. Instead of that, we flip a bit for
+every range of 128 entries.  This is a common GC optimization.)  Setting
+the card bit of a GC object requires a sequence of assembler
+instructions that depends too much on the target CPU to be expressed
+explicitly here (moreover, it contains a few branches, which are hard to
+express at this level).
+
+
+Assembly
+--------
+
+No fancy code generation technique, but greedy forward pass that tries
+to avoid some pitfalls
+
+
+Handling instructions
+~~~~~~~~~~~~~~~~~~~~~
+
+* One by one (forward direction).   Each instruction asks the register
+  allocator to ensure that some arguments are in registers (not in the
+  jitframe); asks for a register to put its result into; and asks for
+  additional scratch registers that will be freed at  the end of the
+  instruction.  There is a special case for boolean variables: they are
+  stored in the condition code flags instead of being materialized as a
+  0/1 value.  (They are materialized later, except in the common case
+  where they are only used by the next `guard_false` or `guard_true` and
+  then forgotten.)
+
+* Instruction arguments are loaded into a register on demand.  This
+  makes the backend quite easy to write, but leads do some bad
+  decisions.
+
+
+Linear scan register allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Although it's always a linear trace that we consider, we don't use
+advanced techniques for register allocation: we do forward, on-demand
+allocation as the backend produces the assembler.  When it asks for a
+register to put some value into, we give it any free register, without
+consideration for what will be done with it later.  We compute the
+longevity of all variables, but only use it when choosing which register
+to spill (we spill the variable with the longest longevity).
+
+This works to some extend because it is well integrated with the earlier
+optimization pass. Loops are unrolled once by the optimization pass to
+allow more powerful optimizations---the optimization pass itself is the
+place that benefits the most, but it also has benefits here in the
+assembly pass.  These are:
+
+* The first peeling initializes the register binding on the first use.
+
+* This leads to an already allocated register of the trace loop.
+
+* As well as allocated registers when exiting bridges
+
+[Try to better allocate registers to match the ABI (minor to non benefit
+in the current state)]
+
+
+More complex mappings
+~~~~~~~~~~~~~~~~~~~~~
+
+Some instructions generate more complex code.  These are either or both of:
+
+* complex instructions generating some local control flow, like
+  "cond_gc_wb_call" (for write barriers), "call_assembler" (a call
+  followed by a few checks).
+
+* instructions that invoke custom assembler helpers, like the slow-path
+  of write barriers or the slow-path of allocations.  These slow-paths
+  are typically generated too, so that we are not constrained by the
+  usual calling conventions.
+
+
+GC pointers
+~~~~~~~~~~~
+
+Around most CALL instructions, we need to record a description of where
+the GC pointers are (registers and stack frame).  This is needed in case
+the CALL invokes a garbage collection.  The GC pointers can move; the
+pointers in the registers and stack frame are updated by the GC.  That's
+a reason for why we don't have explicit interior pointers.
+
+GC pointers can appear as constants in the trace.  We are busy changing
+that to use a constant table and `MOV REG, (%RIP+offset)`.  The
+"constant" in the table is actually updated by the GC if the object
+move.
+
+
+Vectorization
+~~~~~~~~~~~~~
+
+Optimization developed to use SIMD instructions for trace loops. Primary
+idea was to use it as an optimization of micro numpy. It has several
+passes on the already optimized trace.
+
+Shortly explained: It builds dependencies for an unrolled trace loop,
+gathering pairs/packs of operations that could be executed in parallel
+and finally schedules the operations.
+
+What did it add to the code base:
+
+* Dependencies can be constructed
+
+* Code motion of guards to relax dependencies
+
+* Scheduler to reorder trace
+
+* Array bound check removal (especially for unrolled traces)
+
+What can it do:
+
+* Transform vector loops (element wise operations)
+
+* Accumulation (`reduce([...],operator,0)`). Requires Operation to be
+  associative and commutative
+
+* SSE 4.1 as "vector backend"
+
+
+We do not
+~~~~~~~~~
+
+* Keep tracing data around to reoptimize the trace tree. (Once a trace
+  is compiled, minimal data is kept.)  This is one reason (there are
+  others in the front-end) for the following result: JIT-compiling a
+  small loop with two common paths ends up as one "loop" and one bridge
+  assembled, and the bridge-following path is slightly less efficient.
+  This is notably because this bridge is assembled with two constraints:
+  the input registers are fixed (from the guard), and the output
+  registers are fixed (from the jump target); usually these two sets of
+  fixed registers are different, and copying around is needed.
+
+* We don't join trace tails: we only assemble *trees*.
+
+* We don't do any reordering (neither of trace instructions nor of
+  individual assembler instructions)
+
+* We don't do any cross-instruction optimization that makes sense only
+  for the backend and can't easily be expressed at a higher level.  I'm
+  sure there are tons of examples of that, but e.g. loading a large
+  constant in a register that will survive for several instructions;
+  moving out of loops *parts* of some instruction like the address
+  calculation; etc. etc.
+
+* Other optimization opportunities I can think about: look at the
+  function prologue/epilogue; look at the overhead (small but not zero)
+  at the start of a bridge.  Also check if the way guards are
+  implemented makes sense.  Also, we generate large-ish sequences of
+  assembler instructions with tons of `Jcond` that are almost never
+  followed; any optimization opportunity there?  (They all go forward,
+  if it changes anything.)  In theory we could also replace some of
+  these with a signal handler on segfault (e.g. `guard_nonnull_class`).
+
+
+a GCC or LLVM backend?
+~~~~~~~~~~~~~~~~~~~~~~
+
+At least for comparison we'd like a JIT backend that emits its code
+using GCC or LLVM (irrespective of the time it would take).  But it's
+hard to map reasonably well the guards to the C language or to LLVM IR.
+The problems are: (1) we have many guards, we would like to avoid having
+many paths that each do a full
+saving-all-local-variables-that-are-still-alive; (2) it's hard to patch
+a guard when a bridge is compiled from it; (3) instructions like a CALL
+need to expose the local variables that are GC pointers; CALL_MAY_FORCE
+need to expose *all* local variables for optional off-line
+reconstruction of the interpreter state.
+
diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst
--- a/rpython/doc/jit/index.rst
+++ b/rpython/doc/jit/index.rst
@@ -26,6 +26,7 @@
    optimizer
    virtualizable
    vectorization
+   backend
 
 - :doc:`Overview <overview>`: motivating our approach
 
@@ -34,5 +35,8 @@
 - :doc:`Optimizer <optimizer>`: the step between tracing and writing
   machine code
 
-- :doc:`Virtulizable <virtualizable>` how virtualizables work and what they are
-  (in other words how to make frames more efficient).
+- :doc:`Virtualizable <virtualizable>`: how virtualizables work and what
+  they are (in other words how to make frames more efficient).
+
+- :doc:`Assembler backend <backend>`: draft notes about the organization
+  of the assembler backends
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -710,6 +710,12 @@
 
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
+
+    def filter(self, builder):
+        if not builder.cpu.supports_cond_call_value and \
+           self.opnum == rop.COND_CALL_VALUE_I:
+            raise test_random.CannotProduceOperation
+
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4558,3 +4558,20 @@
         self.meta_interp(f, [])
         self.check_resops(guard_nonnull=0)
 
+    def test_loop_before_main_loop(self):
+        fdriver = JitDriver(greens=[], reds='auto')
+        gdriver = JitDriver(greens=[], reds='auto')
+        def f(i, j):
+            while j > 0:   # this loop unrolls because it is in the same
+                j -= 1     # function as a jit_merge_point()
+            while i > 0:
+                fdriver.jit_merge_point()
+                i -= 1
+        def g(i, j, k):
+            while k > 0:
+                gdriver.jit_merge_point()
+                f(i, j)
+                k -= 1
+
+        self.meta_interp(g, [5, 5, 5])
+        self.check_resops(guard_true=10)   # 5 unrolled, plus 5 unrelated
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -359,12 +359,13 @@
     tpe.members[n] = lltype.nullptr(FFI_TYPE_P.TO)
     return tpe
 
+ at specialize.memo()
 def cast_type_to_ffitype(tp):
     """ This function returns ffi representation of rpython type tp
     """
     return TYPE_MAP[tp]
-cast_type_to_ffitype._annspecialcase_ = 'specialize:memo'
 
+ at specialize.argtype(1)
 def push_arg_as_ffiptr(ffitp, arg, ll_buf):
     # This is for primitive types.  Note that the exact type of 'arg' may be
     # different from the expected 'c_size'.  To cope with that, we fall back
@@ -396,7 +397,6 @@
                 arg >>= 8
         else:
             raise AssertionError
-push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)'
 
 
 # type defs for callback and closure userdata
@@ -470,12 +470,12 @@
 FUNCFLAG_USE_ERRNO = 8
 FUNCFLAG_USE_LASTERROR = 16
 
+ at specialize.arg(1)     # hack :-/
 def get_call_conv(flags, from_jit):
     if _WIN32 and not _WIN64 and (flags & FUNCFLAG_CDECL == 0):
         return FFI_STDCALL
     else:
         return FFI_DEFAULT_ABI
-get_call_conv._annspecialcase_ = 'specialize:arg(1)'     # hack :-/
 
 
 class AbstractFuncPtr(object):
@@ -599,6 +599,7 @@
         else:
             self.restype_size = -1
 
+    @specialize.argtype(1)
     def push_arg(self, value):
         #if self.pushed_args == self.argnum:
         #    raise TypeError("Too many arguments, eats %d, pushed %d" %
@@ -618,7 +619,6 @@
         push_arg_as_ffiptr(self.argtypes[self.pushed_args], value,
                            self.ll_args[self.pushed_args])
         self.pushed_args += 1
-    push_arg._annspecialcase_ = 'specialize:argtype(1)'
 
     def _check_args(self):
         if self.pushed_args < self.argnum:
@@ -627,6 +627,7 @@
     def _clean_args(self):
         self.pushed_args = 0
 
+    @specialize.arg(1)
     def call(self, RES_TP):
         self._check_args()
         ffires = c_ffi_call(self.ll_cif, self.funcsym,
@@ -645,7 +646,6 @@
         self._clean_args()
         check_fficall_result(ffires, self.flags)
         return res
-    call._annspecialcase_ = 'specialize:arg(1)'
 
     def __del__(self):
         if self.ll_args:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -280,6 +280,7 @@
 
 
 @oopspec("jit.isconstant(value)")
+ at specialize.call_location()
 def isconstant(value):
     """
     While tracing, returns whether or not the value is currently known to be
@@ -289,9 +290,9 @@
     This is for advanced usage only.
     """
     return NonConstant(False)
-isconstant._annspecialcase_ = "specialize:call_location"
 
 @oopspec("jit.isvirtual(value)")
+ at specialize.call_location()
 def isvirtual(value):
     """
     Returns if this value is virtual, while tracing, it's relatively
@@ -300,7 +301,6 @@
     This is for advanced usage only.
     """
     return NonConstant(False)
-isvirtual._annspecialcase_ = "specialize:call_location"
 
 @specialize.call_location()
 def loop_unrolling_heuristic(lst, size, cutoff=2):
@@ -401,28 +401,27 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Signed, _we_are_jitted)
 
-
+ at oopspec('jit.current_trace_length()')
 def current_trace_length():
     """During JIT tracing, returns the current trace length (as a constant).
     If not tracing, returns -1."""
     if NonConstant(False):
         return 73
     return -1
-current_trace_length.oopspec = 'jit.current_trace_length()'
 
+ at oopspec('jit.debug(string, arg1, arg2, arg3, arg4)')
 def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1,
                       arg3=-sys.maxint-1, arg4=-sys.maxint-1):
     """When JITted, cause an extra operation JIT_DEBUG to appear in
     the graphs.  Should not be left after debugging."""
     keepalive_until_here(string) # otherwise the whole function call is removed
-jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)'
 
+ at oopspec('jit.assert_green(value)')
+ at specialize.argtype(0)
 def assert_green(value):
     """Very strong assert: checks that 'value' is a green
     (a JIT compile-time constant)."""
     keepalive_until_here(value)
-assert_green._annspecialcase_ = 'specialize:argtype(0)'
-assert_green.oopspec = 'jit.assert_green(value)'
 
 class AssertGreenFailed(Exception):
     pass
@@ -457,6 +456,7 @@
 # ____________________________________________________________
 # VRefs
 
+ at oopspec('virtual_ref(x)')
 @specialize.argtype(0)
 def virtual_ref(x):
     """Creates a 'vref' object that contains a reference to 'x'.  Calls
@@ -467,14 +467,13 @@
     dereferenced (by the call syntax 'vref()'), it returns 'x', which is
     then forced."""
     return DirectJitVRef(x)
-virtual_ref.oopspec = 'virtual_ref(x)'
 
+ at oopspec('virtual_ref_finish(x)')
 @specialize.argtype(1)
 def virtual_ref_finish(vref, x):
     """See docstring in virtual_ref(x)"""
     keepalive_until_here(x)   # otherwise the whole function call is removed
     _virtual_ref_finish(vref, x)
-virtual_ref_finish.oopspec = 'virtual_ref_finish(x)'
 
 def non_virtual_ref(x):
     """Creates a 'vref' that just returns x when called; nothing more special.
@@ -831,6 +830,7 @@
     jit_opencoder_model
     """
 
+ at specialize.arg(0)
 def set_user_param(driver, text):
     """Set the tunable JIT parameters from a user-supplied string
     following the format 'param=value,param=value', or 'off' to
@@ -866,7 +866,6 @@
                     break
             else:
                 raise ValueError
-set_user_param._annspecialcase_ = 'specialize:arg(0)'
 
 # ____________________________________________________________
 #
diff --git a/rpython/rlib/listsort.py b/rpython/rlib/listsort.py
--- a/rpython/rlib/listsort.py
+++ b/rpython/rlib/listsort.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rlib.objectmodel import specialize
 
 
 ## ------------------------------------------------------------------------
@@ -141,6 +142,12 @@
         # or, IOW, the first k elements of a should precede key, and the last
         # n-k should follow key.
 
+        # hint for the annotator: the argument 'rightmost' is always passed in as
+        # a constant (either True or False), so we can specialize the function for
+        # the two cases.  (This is actually needed for technical reasons: the
+        # variable 'lower' must contain a known method, which is the case in each
+        # specialized version but not in the unspecialized one.)
+        @specialize.arg(4)
         def gallop(self, key, a, hint, rightmost):
             assert 0 <= hint < a.len
             if rightmost:
@@ -212,12 +219,6 @@
             assert lastofs == ofs         # so a[ofs-1] < key <= a[ofs]
             return ofs
 
-        # hint for the annotator: the argument 'rightmost' is always passed in as
-        # a constant (either True or False), so we can specialize the function for
-        # the two cases.  (This is actually needed for technical reasons: the
-        # variable 'lower' must contain a known method, which is the case in each
-        # specialized version but not in the unspecialized one.)
-        gallop._annspecialcase_ = "specialize:arg(4)"
 
         # ____________________________________________________________
 
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -37,6 +37,7 @@
 from rpython.rlib import objectmodel
 from rpython.flowspace.model import Constant, const
 from rpython.flowspace.specialcase import register_flow_sc
+from rpython.rlib.objectmodel import specialize
 
 """
 Long-term target:
@@ -135,14 +136,15 @@
     # We deal directly with overflow there anyway.
     return r_longlonglong(n)
 
+ at specialize.argtype(0)
 def widen(n):
     from rpython.rtyper.lltypesystem import lltype
     if _should_widen_type(lltype.typeOf(n)):
         return intmask(n)
     else:
         return n
-widen._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def _should_widen_type(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Bool:
@@ -153,19 +155,18 @@
     assert issubclass(r_class, base_int)
     return r_class.BITS < LONG_BIT or (
         r_class.BITS == LONG_BIT and r_class.SIGNED)
-_should_widen_type._annspecialcase_ = 'specialize:memo'
 
 # the replacement for sys.maxint
 maxint = int(LONG_TEST - 1)
 # for now, it should be equal to sys.maxint on all supported platforms
 assert maxint == sys.maxint
 
+ at specialize.argtype(0)
 def is_valid_int(r):
     if objectmodel.we_are_translated():
         return isinstance(r, int)
     return isinstance(r, (base_int, int, long, bool)) and (
         -maxint - 1 <= r <= maxint)
-is_valid_int._annspecialcase_ = 'specialize:argtype(0)'
 
 def ovfcheck(r):
     "NOT_RPYTHON"
@@ -225,12 +226,12 @@
         return build_int(None, self_type.SIGNED, max(self_type.BITS, other_type.BITS))
     raise AssertionError("Merging these types (%s, %s) is not supported" % (self_type, other_type))
 
+ at specialize.memo()
 def signedtype(t):
     if t in (bool, int, long):
         return True
     else:
         return t.SIGNED
-signedtype._annspecialcase_ = 'specialize:memo'
 
 def normalizedinttype(t):
     if t is int:
@@ -241,11 +242,12 @@
         assert t.BITS <= r_longlong.BITS
         return build_int(None, t.SIGNED, r_longlong.BITS)
 
+ at specialize.argtype(0)
 def most_neg_value_of_same_type(x):
     from rpython.rtyper.lltypesystem import lltype
     return most_neg_value_of(lltype.typeOf(x))
-most_neg_value_of_same_type._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def most_neg_value_of(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -256,13 +258,13 @@
         return r_class(-(r_class.MASK >> 1) - 1)
     else:
         return r_class(0)
-most_neg_value_of._annspecialcase_ = 'specialize:memo'
 
+ at specialize.argtype(0)
 def most_pos_value_of_same_type(x):
     from rpython.rtyper.lltypesystem import lltype
     return most_pos_value_of(lltype.typeOf(x))
-most_pos_value_of_same_type._annspecialcase_ = 'specialize:argtype(0)'
 
+ at specialize.memo()
 def most_pos_value_of(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -273,8 +275,8 @@
         return r_class(r_class.MASK >> 1)
     else:
         return r_class(r_class.MASK)
-most_pos_value_of._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def is_signed_integer_type(tp):
     from rpython.rtyper.lltypesystem import lltype, rffi
     if tp is lltype.Signed:
@@ -284,7 +286,6 @@
         return r_class.SIGNED
     except KeyError:
         return False   # not an integer type
-is_signed_integer_type._annspecialcase_ = 'specialize:memo'
 
 def highest_bit(n):
     """
@@ -676,7 +677,7 @@
     from rpython.rtyper.lltypesystem.lloperation import llop
     return llop.int_mod(lltype.Signed, x, y)
 
- at objectmodel.specialize.ll()
+ at specialize.ll()
 def byteswap(arg):
     """ Convert little->big endian and the opposite
     """
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -90,16 +90,16 @@
 
 FIVEARY_CUTOFF = 8
 
+ at specialize.argtype(0)
 def _mask_digit(x):
     return UDIGIT_MASK(x & MASK)
-_mask_digit._annspecialcase_ = 'specialize:argtype(0)'
 
 def _widen_digit(x):
     return rffi.cast(LONG_TYPE, x)
 
+ at specialize.argtype(0)
 def _store_digit(x):
     return rffi.cast(STORE_TYPE, x)
-_store_digit._annspecialcase_ = 'specialize:argtype(0)'
 
 def _load_unsigned_digit(x):
     return rffi.cast(UNSIGNED_TYPE, x)
@@ -175,11 +175,11 @@
         return _load_unsigned_digit(self._digits[x])
     udigit._always_inline_ = True
 
+    @specialize.argtype(2)
     def setdigit(self, x, val):
         val = _mask_digit(val)
         assert val >= 0
         self._digits[x] = _store_digit(val)
-    setdigit._annspecialcase_ = 'specialize:argtype(2)'
     setdigit._always_inline_ = True
 
     def numdigits(self):
@@ -1312,6 +1312,7 @@
 
     return res
 
+ at specialize.argtype(0)
 def digits_from_nonneg_long(l):
     digits = []
     while True:
@@ -1319,8 +1320,8 @@
         l = l >> SHIFT
         if not l:
             return digits[:] # to make it non-resizable
-digits_from_nonneg_long._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def digits_for_most_neg_long(l):
     # This helper only works if 'l' is the most negative integer of its
     # type, which in base 2 looks like: 1000000..0000
@@ -1335,8 +1336,8 @@
     assert l & MASK == l
     digits.append(_store_digit(l))
     return digits[:] # to make it non-resizable
-digits_for_most_neg_long._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def args_from_rarith_int1(x):
     if x > 0:
         return digits_from_nonneg_long(x), 1
@@ -1348,11 +1349,10 @@
     else:
         # the most negative integer! hacks needed...
         return digits_for_most_neg_long(x), -1
-args_from_rarith_int1._annspecialcase_ = "specialize:argtype(0)"
-
+
+ at specialize.argtype(0)
 def args_from_rarith_int(x):
     return args_from_rarith_int1(widen(x))
-args_from_rarith_int._annspecialcase_ = "specialize:argtype(0)"
 # ^^^ specialized by the precise type of 'x', which is typically a r_xxx
 #     instance from rlib.rarithmetic
 
@@ -1909,6 +1909,7 @@
         i += 1
     return borrow
 
+ at specialize.argtype(2)
 def _muladd1(a, n, extra=0):
     """Multiply by a single digit and add a single digit, ignoring the sign.
     """
@@ -1926,7 +1927,7 @@
     z.setdigit(i, carry)
     z._normalize()
     return z
-_muladd1._annspecialcase_ = "specialize:argtype(2)"
+
 def _v_lshift(z, a, m, d):
     """ Shift digit vector a[0:m] d bits left, with 0 <= d < SHIFT. Put
         * result in z[0:m], and return the d bits shifted out of the top.
@@ -2178,6 +2179,7 @@
         ad = -ad
     return ad
 
+ at specialize.arg(0)
 def _loghelper(func, arg):
     """
     A decent logarithm is easy to compute even for huge bigints, but libm can't
@@ -2195,7 +2197,6 @@
     # CAUTION:  e*SHIFT may overflow using int arithmetic,
     # so force use of double. */
     return func(x) + (e * float(SHIFT) * func(2.0))
-_loghelper._annspecialcase_ = 'specialize:arg(0)'
 
 # ____________________________________________________________
 
@@ -2519,6 +2520,7 @@
     return output.build()
 
 
+ at specialize.arg(1)
 def _bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
@@ -2598,8 +2600,8 @@
         return z
 
     return z.invert()
-_bitwise._annspecialcase_ = "specialize:arg(1)"
-
+
+ at specialize.arg(1)
 def _int_bitwise(a, op, b): # '&', '|', '^'
     """ Bitwise and/or/xor operations """
 
@@ -2682,7 +2684,6 @@
         return z
 
     return z.invert()
-_int_bitwise._annspecialcase_ = "specialize:arg(1)"
 
 ULONGLONG_BOUND = r_ulonglong(1L << (r_longlong.BITS-1))
 LONGLONG_MIN = r_longlong(-(1L << (r_longlong.BITS-1)))
diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py
--- a/rpython/rlib/rmmap.py
+++ b/rpython/rlib/rmmap.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.nonconst import NonConstant
 from rpython.rlib.rarithmetic import intmask
 
@@ -239,12 +239,12 @@
     _, _VirtualProtect_safe = winexternal('VirtualProtect',
                                   [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD],
                                   BOOL)
+    @specialize.ll()
     def VirtualProtect(addr, size, mode, oldmode_ptr):
         return _VirtualProtect_safe(addr,
                                rffi.cast(rffi.SIZE_T, size),
                                rffi.cast(DWORD, mode),
                                oldmode_ptr)
-    VirtualProtect._annspecialcase_ = 'specialize:ll'
     VirtualFree, VirtualFree_safe = winexternal('VirtualFree',
                               [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL)
 
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.error import TyperError

From pypy.commits at gmail.com  Mon Sep 26 16:42:40 2016
From: pypy.commits at gmail.com (sbauman)
Date: Mon, 26 Sep 2016 13:42:40 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Cleanup
Message-ID: <57e98840.43e61c0a.8b044.71a1@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87403:e927b403b073
Date: 2016-09-26 13:05 -0400
http://bitbucket.org/pypy/pypy/changeset/e927b403b073/

Log:	Cleanup

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -132,7 +132,6 @@
     def debug_header(self, indent):
         raise NotImplementedError
 
-
 class AbstractVirtualStructStateInfo(AbstractVirtualStateInfo):
     _attrs_ = ('typedescr', 'fielddescrs')
 
@@ -141,6 +140,9 @@
         self.fielddescrs = fielddescrs
 
     def _generate_guards(self, other, box, runtime_box, state):
+        if isinstance(other, NotVirtualStateInfo):
+            return self._generate_guards_non_virtual(other, box, runtime_box, state)
+
         if not self._generalization_of_structpart(other):
             raise VirtualStatesCantMatch("different kinds of structs")
 
@@ -178,7 +180,13 @@
                                                    fieldbox_runtime, state)
 
     def _generate_guards_non_virtual(self, other, box, runtime_box, state):
-        pass
+        """
+        Generate guards for the case where a virtual object is expected, but
+        a non-virtual is given. When the underlying type is a value class,
+        the non-virtual may be virtualized by unpacking it and forwarding the
+        components elementwise.
+        """
+        raise VirtualStatesCantMatch("different kinds of structs")
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
@@ -227,6 +235,7 @@
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
 
 class VArrayStateInfo(AbstractVirtualStateInfo):
+    _attrs_ = ('arraydescr',)
 
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
@@ -277,6 +286,8 @@
 
 
 class VArrayStructStateInfo(AbstractVirtualStateInfo):
+    _attrs_ = ('arraydescr', 'fielddescrs', 'length')
+
     def __init__(self, arraydescr, fielddescrs, length):
         self.arraydescr = arraydescr
         self.fielddescrs = fielddescrs
@@ -355,10 +366,12 @@
         return NotVirtualStateInfoInt(cpu, type, info)
     if type == 'r':
         return NotVirtualStateInfoPtr(cpu, type, info)
+    assert type == 'f'
     return NotVirtualStateInfo(cpu, type, info)
 
 
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
+    _attrs_ = ('level', 'constbox', 'position_in_notvirtuals')
     level = LEVEL_UNKNOWN
     constbox = None
 
@@ -458,6 +471,7 @@
         debug_print(result)
 
 class NotVirtualStateInfoInt(NotVirtualStateInfo):
+    _attrs_ = ('intbound')
     intbound = None
 
     def __init__(self, cpu, type, info):
@@ -492,6 +506,7 @@
 
 
 class NotVirtualStateInfoPtr(NotVirtualStateInfo):
+    _attrs_ = ('lenbound', 'known_class')
     lenbound = None
     known_class = None
 
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1048,12 +1048,7 @@
         # virtual X object. The side exit in this case will contain a non-virtual
         # value class, which should be unpacked into the
 
-        # self.check_resops(jump=3, label=2, new_with_vtable=2)
-        # self.check_target_token_count(2)
-        # self.check_trace_count(3)
-
-
-    def test_conflated_virtual_states(self):
+    def test_aliased_virtual_states(self):
         # All cases are covered when forcing one component of the virtual state
         # also forces an as yet unseen component.
         # i.e. expect [NotVirtual, Virtual] and given a pair of aliasing virtual

From pypy.commits at gmail.com  Mon Sep 26 16:42:44 2016
From: pypy.commits at gmail.com (sbauman)
Date: Mon, 26 Sep 2016 13:42:44 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Kill dead rop function
Message-ID: <57e98844.0a21c20a.baef2.e892@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87405:da2ccb17ee1d
Date: 2016-09-26 16:24 -0400
http://bitbucket.org/pypy/pypy/changeset/da2ccb17ee1d/

Log:	Kill dead rop function

diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1265,14 +1265,6 @@
         return rop.CALL_LOOPINVARIANT_N
 
     @staticmethod
-    def getfield_pure_for_descr(descr):
-        if descr.is_pointer_field():
-            return rop.GETFIELD_GC_PURE_R
-        elif descr.is_float_field():
-            return rop.GETFIELD_GC_PURE_F
-        return rop.GETFIELD_GC_PURE_I
-
-    @staticmethod
     def getfield_for_descr(descr):
         if descr.is_pointer_field():
             return rop.GETFIELD_GC_R

From pypy.commits at gmail.com  Mon Sep 26 16:42:43 2016
From: pypy.commits at gmail.com (sbauman)
Date: Mon, 26 Sep 2016 13:42:43 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Work on merging virtual and
 non-virtual states
Message-ID: <57e98843.a3adc20a.e8481.1415@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87404:460dc18502fe
Date: 2016-09-26 16:14 -0400
http://bitbucket.org/pypy/pypy/changeset/460dc18502fe/

Log:	Work on merging virtual and non-virtual states

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -33,8 +33,11 @@
 class FakeDescr(object):
     def __init__(self, vtable):
         self.vtable = vtable
+    def is_value_class(self):
+        return False
     def get_vtable(self):
         return self.vtable
+
 FakeDescr.value = FakeDescr(42)
 
 class BaseTestGenerateGuards(BaseTest):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -179,15 +179,6 @@
                                                    fieldbox,
                                                    fieldbox_runtime, state)
 
-    def _generate_guards_non_virtual(self, other, box, runtime_box, state):
-        """
-        Generate guards for the case where a virtual object is expected, but
-        a non-virtual is given. When the underlying type is a value class,
-        the non-virtual may be virtualized by unpacking it and forwarding the
-        components elementwise.
-        """
-        raise VirtualStatesCantMatch("different kinds of structs")
-
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
@@ -222,6 +213,63 @@
         return (isinstance(other, VirtualStateInfo) and
                 self.typedescr is other.typedescr)
 
+    def _generate_guards_non_virtual(self, other, box, runtime_box, state):
+        """
+        Generate guards for the case where a virtual object is expected, but
+        a non-virtual is given. When the underlying type is a value class,
+        the non-virtual may be virtualized by unpacking it and forwarding the
+        components elementwise.
+        """
+        assert isinstance(other, NotVirtualStateInfoPtr)
+
+        known_class = self.typedescr
+        if not known_class.is_value_class():
+            raise VirtualStatesCantMatch("different kinds of structs")
+
+        raise VirtualStatesCantMatch("different kinds of structs")
+
+        # TODO: Probably should rename state.extra_guards to extra_ops
+        extra_guards = state.extra_guards
+        cpu = state.cpu
+
+        # Emit any guards that are needed to validate the class of the supplied
+        # object, so that we know it is safe to unpack.
+        if other.level == LEVEL_UNKNOWN:
+            op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class])
+            extra_guards.append(op)
+        elif other.level == LEVEL_NONULL:
+            op = ResOperation(rop.GUARD_CLASS, [box, self.known_class])
+            extra_guards.append(op)
+        elif other.level == LEVEL_KNOWNCLASS:
+            if not self.known_class.same_constant(other.known_class):
+                raise VirtualStatesCantMatch("classes don't match")
+        else:
+            assert other.level == LEVEL_CONSTANT
+            known = self.known_class
+            const = other.constbox
+            if const.nonnull() and known.same_constant(cpu.ts.cls_of_box(const)):
+                pass
+            else:
+                raise VirtualStatesCantMatch("classes don't match")
+
+        # Things to do...
+        # 1. Generate new_with_vtable operation to allocate the new virtual object
+        # 2. Generate getfield operations which populate the fields of the new virtual
+        # 3. Recursively generate guards for each portion of the virtual state
+        #    (we don't have a VirtualStateInfo objects for the subfields of the
+        #     non-virtual object which we are promoting to a virtual. How do we
+        #     generate this new virtual state so we can operate recursively)
+
+        # We need to build a virtual version which conforms with the expected
+        # virtual object.
+        # This will probably look a lot like
+        # AbstractVirtualStateInfo._generate_guards
+        fields = [None] * len(self.fielddescrs)
+        for i, descr in enumerate(self.fielddescrs):
+            opnum = rop.getfield_for_descr(descr)
+            fields[i] = getfield_op = ResOperation(opnum, [box], descr=descr)
+            extra_guards.append(getfield_op)
+
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
 
@@ -231,6 +279,9 @@
         return (isinstance(other, VStructStateInfo) and
                 self.typedescr is other.typedescr)
 
+    def _generate_guards_non_virtual(self, other, box, runtime_box, state):
+        raise VirtualStatesCantMatch("can't unify virtual struct with non-virtual")
+
     def debug_header(self, indent):
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
 

From pypy.commits at gmail.com  Mon Sep 26 17:20:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Sep 2016 14:20:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: fix the test
Message-ID: <57e99129.c50e1c0a.71dc9.613c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87406:986e0e2abaa1
Date: 2016-09-26 23:20 +0200
http://bitbucket.org/pypy/pypy/changeset/986e0e2abaa1/

Log:	fix the test

diff --git a/rpython/rlib/rvmprof/test/test_traceback.py b/rpython/rlib/rvmprof/test/test_traceback.py
--- a/rpython/rlib/rvmprof/test/test_traceback.py
+++ b/rpython/rlib/rvmprof/test/test_traceback.py
@@ -74,13 +74,15 @@
         raise NotImplementedError
     rvmprof.register_code_object_class(MyCode, get_name)
 
-    jitdriver = jit.JitDriver(greens=[], reds='auto')
+    jitdriver = jit.JitDriver(greens=['code'], reds='auto',
+                   is_recursive=True,
+                   get_unique_id=lambda code: rvmprof.get_unique_id(code))
 
     @rvmprof.vmprof_execute_code("mycode", lambda code, level, total_i: code)
     def mainloop(code, level, total_i):
         i = 20
         while i > 0:
-            jitdriver.jit_merge_point()
+            jitdriver.jit_merge_point(code=code)
             i -= 1
             if level > 0:
                 mainloop(code, level - 1, total_i + i)
@@ -107,5 +109,5 @@
     stdout = t.driver.cbuilder.cmdexec('')
     r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] (\d) 42\n")
     got = r.findall(stdout)
-    addr = got[0][1]
-    assert got == XXX
+    addr = got[0][0]
+    assert got == [(addr, '1'), (addr, '1'), (addr, '0')]

From pypy.commits at gmail.com  Tue Sep 27 01:14:02 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Mon, 26 Sep 2016 22:14:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Update int(x,
 base=10) behaviour for Python 3.4.
Message-ID: <57ea001a.898b1c0a.d61c1.bc2d@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87407:1fc357671011
Date: 2016-09-26 21:00 +0100
http://bitbucket.org/pypy/pypy/changeset/1fc357671011/

Log:	Update int(x, base=10) behaviour for Python 3.4.

	"Changed in version 3.4: If base is not an instance of int and the
	base object has a base.__index__ method, that method is called to
	obtain an integer for the base. Previous versions used base.__int__
	instead of base.__index__." (from
	https://docs.python.org/3/library/functions.html#int)

	Note that in CPython an error is returned if base.__index__ does not
	exist on the non-int object, even if base.__int__ exists. Test
	amended to confirm this error is raised.

diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -892,7 +892,7 @@
                 return _string_to_int_or_long(space, w_inttype, w_value, buf)
     else:
         try:
-            base = space.int_w(w_base)
+            base = space.getindex_w(w_base, None)
         except OperationError as e:
             if not e.match(space, space.w_OverflowError):
                 raise
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -525,12 +525,23 @@
         class MyInt(object):
             def __init__(self, x):
                 self.x = x
-            def __int__(self):
+            def __index__(self):
                 return self.x
 
         base = MyInt(24)
         assert int('10', base) == 24
 
+        class MyNonIndexable(object):
+            def __init__(self, x):
+                self.x = x
+            def __int__(self):
+                return self.x
+
+        base = MyNonIndexable(24)
+        e = raises(TypeError, int, '10', base)
+        assert str(e.value) == ("'MyNonIndexable' object cannot be interpreted "
+                                "as an integer")
+
     def test_truediv(self):
         import operator
         x = 1000000

From pypy.commits at gmail.com  Tue Sep 27 01:17:44 2016
From: pypy.commits at gmail.com (sbauman)
Date: Mon, 26 Sep 2016 22:17:44 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Fix handling of intbounds in
 virtuastate
Message-ID: <57ea00f8.8aabc20a.33724.2765@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87409:c5b9c438b580
Date: 2016-09-27 00:46 -0400
http://bitbucket.org/pypy/pypy/changeset/c5b9c438b580/

Log:	Fix handling of intbounds in virtuastate

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -226,10 +226,8 @@
         if not known_class.is_value_class():
             raise VirtualStatesCantMatch("different kinds of structs")
 
+        raise VirtualStatesCantMatch("different kinds of structs")
         # import pdb; pdb.set_trace()
-        raise VirtualStatesCantMatch("different kinds of structs")
-
-        import pdb; pdb.set_trace()
 
         # TODO: Probably should rename state.extra_guards to extra_ops
         extra_guards = state.extra_guards
@@ -561,9 +559,8 @@
             other_intbound = other.intbound
         if self.intbound is None:
             return
-        if other.intbound is None:
-            return
-        if self.intbound.contains_bound(other_intbound):
+        if (other_intbound is not None and
+            self.intbound.contains_bound(other_intbound)):
             return
         if (runtime_box is not None and
             self.intbound.contains(runtime_box.getint())):

From pypy.commits at gmail.com  Tue Sep 27 01:17:42 2016
From: pypy.commits at gmail.com (sbauman)
Date: Mon, 26 Sep 2016 22:17:42 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Generate fieldboxes and faux
 virtual state
Message-ID: <57ea00f6.02c11c0a.5b222.bb57@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87408:3d7fa60a7ac3
Date: 2016-09-26 23:19 -0400
http://bitbucket.org/pypy/pypy/changeset/3d7fa60a7ac3/

Log:	Generate fieldboxes and faux virtual state

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -226,8 +226,11 @@
         if not known_class.is_value_class():
             raise VirtualStatesCantMatch("different kinds of structs")
 
+        # import pdb; pdb.set_trace()
         raise VirtualStatesCantMatch("different kinds of structs")
 
+        import pdb; pdb.set_trace()
+
         # TODO: Probably should rename state.extra_guards to extra_ops
         extra_guards = state.extra_guards
         cpu = state.cpu
@@ -260,15 +263,31 @@
         #     non-virtual object which we are promoting to a virtual. How do we
         #     generate this new virtual state so we can operate recursively)
 
+        if runtime_box is not None:
+            opinfo = state.optimizer.getptrinfo(box)
+            assert opinfo is None or isinstance(opinfo, AbstractStructPtrInfo)
+        else:
+            opinfo = None
+
         # We need to build a virtual version which conforms with the expected
         # virtual object.
         # This will probably look a lot like
         # AbstractVirtualStateInfo._generate_guards
-        fields = [None] * len(self.fielddescrs)
+        fields = []
         for i, descr in enumerate(self.fielddescrs):
-            opnum = rop.getfield_for_descr(descr)
-            fields[i] = getfield_op = ResOperation(opnum, [box], descr=descr)
-            extra_guards.append(getfield_op)
+            if runtime_box is not None and opinfo is not None:
+                fieldbox = opinfo._fields[descr.get_index()]
+                fieldbox_runtime = state.get_runtime_field(runtime_box, descr)
+            else:
+                opnum = rop.getfield_for_descr(descr)
+                fieldbox = ResOperation(opnum, [box], descr=descr)
+                extra_guards.append(fieldbox)
+                fieldbox_runtime = None
+            fields.append(fieldbox)
+            faux_field_state = not_virtual(cpu, fieldbox.type, opinfo)
+            fieldstate = self.fieldstate[i]
+            fieldstate.generate_guards(faux_field_state, fieldbox,
+                                       fieldbox_runtime, state)
 
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
@@ -542,6 +561,8 @@
             other_intbound = other.intbound
         if self.intbound is None:
             return
+        if other.intbound is None:
+            return
         if self.intbound.contains_bound(other_intbound):
             return
         if (runtime_box is not None and

From pypy.commits at gmail.com  Tue Sep 27 04:10:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 27 Sep 2016 01:10:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix translation with
 --no-allworkingmodules
Message-ID: <57ea2966.8aabc20a.33724.62e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87410:04821987a10e
Date: 2016-09-27 09:58 +0200
http://bitbucket.org/pypy/pypy/changeset/04821987a10e/

Log:	fix translation with --no-allworkingmodules

diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -7,7 +7,7 @@
 import stat
 import sys
 
-from rpython.rlib import rpath
+from rpython.rlib import rpath, rdynload
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -243,6 +243,7 @@
 
 _eci = ExternalCompilationInfo(separate_module_sources=[_source_code],
     post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);'])
+_eci = _eci.merge(rdynload.eci)
 
 pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP,
                                  _nowrapper=True, compilation_info=_eci)
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -34,6 +34,7 @@
 else:
     libraries = ['dl']
 
+# this 'eci' is also used in pypy/module/sys/initpath.py
 eci = ExternalCompilationInfo(
     pre_include_bits = pre_include_bits,
     includes = includes,

From pypy.commits at gmail.com  Tue Sep 27 04:36:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 27 Sep 2016 01:36:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: Port pypy to the new interface
Message-ID: <57ea2f71.448e1c0a.c6a43.83a4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87411:bc08e4bce5dd
Date: 2016-09-27 10:35 +0200
http://bitbucket.org/pypy/pypy/changeset/bc08e4bce5dd/

Log:	Port pypy to the new interface

diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -1,5 +1,5 @@
 import py
-from rpython.rtyper.lltypesystem import lltype, rffi, rstr
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.translator import cdir
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
@@ -15,7 +15,8 @@
     kwargs.setdefault('compilation_info', eci)
     return rffi.llexternal(*args, **kwargs)
 
-DUMP_CALLBACK = lltype.Ptr(lltype.FuncType([], lltype.Void))
+DUMP_CALLBACK = lltype.Ptr(lltype.FuncType(
+                     [rffi.INT, rffi.SIGNEDP, lltype.Signed], lltype.Void))
 
 pypy_faulthandler_setup = direct_llexternal(
     'pypy_faulthandler_setup', [DUMP_CALLBACK], rffi.CCHARP)
@@ -33,13 +34,14 @@
     'pypy_faulthandler_is_enabled', [], rffi.INT)
 
 pypy_faulthandler_write = direct_llexternal(
-    'pypy_faulthandler_write', [rffi.CCHARP], lltype.Void)
+    'pypy_faulthandler_write', [rffi.INT, rffi.CCHARP], lltype.Void)
 
 pypy_faulthandler_write_int = direct_llexternal(
-    'pypy_faulthandler_write_int', [lltype.Signed], lltype.Void)
+    'pypy_faulthandler_write_int', [rffi.INT, lltype.Signed], lltype.Void)
 
 pypy_faulthandler_dump_traceback = direct_llexternal(
-    'pypy_faulthandler_dump_traceback', [rffi.INT, rffi.INT], lltype.Void)
+    'pypy_faulthandler_dump_traceback',
+    [rffi.INT, rffi.INT, llmemory.Address], lltype.Void)
 
 # for tests...
 
diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py
--- a/pypy/module/faulthandler/dumper.py
+++ b/pypy/module/faulthandler/dumper.py
@@ -1,24 +1,18 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import rgc
-from rpython.rlib.rvmprof import enum_all_code_objs
-from rpython.rlib.rvmprof import cintf as rvmprof_cintf
+from rpython.rlib.rvmprof import traceback
 
 from pypy.interpreter.pycode import PyCode
 from pypy.module.faulthandler.cintf import pypy_faulthandler_write
 from pypy.module.faulthandler.cintf import pypy_faulthandler_write_int
 
-#
-# xxx The dumper is tied to the internals of rvmprof. xxx
-#
 
-
-MAX_FRAME_DEPTH = 100
 MAX_STRING_LENGTH = 500
 
 global_buf = lltype.malloc(rffi.CCHARP.TO, MAX_STRING_LENGTH, flavor='raw',
                            immortal=True, zero=True)
 
-def _dump(s):
+def _dump(fd, s):
     assert isinstance(s, str)
     l = len(s)
     if l >= MAX_STRING_LENGTH:
@@ -28,43 +22,34 @@
         global_buf[i] = s[i]
         i += 1
     global_buf[l] = '\x00'
-    pypy_faulthandler_write(global_buf)
+    pypy_faulthandler_write(fd, global_buf)
 
-def _dump_int(i):
-    pypy_faulthandler_write_int(i)
+def _dump_int(fd, i):
+    pypy_faulthandler_write_int(fd, i)
 
 
-def dump_code(pycode, this_code_id, search_code_id):
-    if this_code_id != search_code_id:
-        return 0
-    _dump('"')
-    _dump(pycode.co_filename)
-    _dump('" in ')
-    _dump(pycode.co_name)
-    _dump(", from line ")
-    _dump_int(pycode.co_firstlineno)
-    _dump("\n")
-    return 1
+def dump_code(pycode, loc, fd):
+    if pycode is None:
+        _dump(fd, "  File ???")
+    else:
+        _dump(fd, '  File "')
+        _dump(fd, pycode.co_filename)
+        _dump(fd, '" in ')
+        _dump(fd, pycode.co_name)
+        _dump(fd, ", from line ")
+        _dump_int(fd, pycode.co_firstlineno)
+    if loc == traceback.LOC_JITTED:
+        _dump(fd, " [jitted]")
+    elif loc == traceback.LOC_JITTED_INLINED:
+        _dump(fd, " [jit inlined]")
+    _dump(fd, "\n")
 
 
 @rgc.no_collect
-def _dump_callback():
+def _dump_callback(fd, array_p, array_length):
     """We are as careful as we can reasonably be here (i.e. not 100%,
     but hopefully close enough).  In particular, this is written as
     RPython but shouldn't allocate anything.
     """
-    _dump("Current stack (most recent call first):\n")
-
-    s = rvmprof_cintf.get_rvmprof_stack()
-    depth = 0
-    while s:
-        if depth >= MAX_FRAME_DEPTH:
-            _dump("  ...\n")
-            break
-        if s.c_kind == rvmprof_cintf.VMPROF_CODE_TAG:
-            code_id = s.c_value
-            _dump("  File ")
-            if enum_all_code_objs(PyCode, dump_code, code_id) == 0:
-                _dump("???\n")
-        s = s.c_next
-        depth += 1
+    _dump(fd, "Current stack (most recent call first):\n")
+    traceback.walk_traceback(PyCode, dump_code, fd, array_p, array_length)
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -9,6 +9,11 @@
 #include <sys/resource.h>
 #include <math.h>
 
+#include "rvmprof.h"
+
+#define MAX_FRAME_DEPTH   100
+#define FRAME_DEPTH_N     RVMPROF_TRACEBACK_ESTIMATE_N(MAX_FRAME_DEPTH)
+
 
 typedef struct sigaction _Py_sighandler_t;
 
@@ -23,8 +28,7 @@
     int initialized;
     int enabled;
     volatile int fd, all_threads;
-    void (*volatile dump_traceback)(void);
-    int _current_fd;
+    volatile pypy_faulthandler_cb_t dump_traceback;
 } fatal_error;
 
 static stack_t stack;
@@ -46,45 +50,46 @@
 static const int faulthandler_nsignals =
     sizeof(faulthandler_handlers) / sizeof(fault_handler_t);
 
-static void
-fh_write(int fd, const char *str)
+RPY_EXTERN
+void pypy_faulthandler_write(int fd, const char *str)
 {
     (void)write(fd, str, strlen(str));
 }
 
 RPY_EXTERN
-void pypy_faulthandler_write(char *str)
+void pypy_faulthandler_write_int(int fd, long value)
 {
-    fh_write(fatal_error._current_fd, str);
+    char buf[32];
+    sprintf(buf, "%ld", value);
+    pypy_faulthandler_write(fd, buf);
 }
 
+
 RPY_EXTERN
-void pypy_faulthandler_write_int(long x)
+void pypy_faulthandler_dump_traceback(int fd, int all_threads,
+                                      void *ucontext)
 {
-    char buf[32];
-    sprintf(buf, "%ld", x);
-    fh_write(fatal_error._current_fd, buf);
+    pypy_faulthandler_cb_t fn;
+    intptr_t array_p[FRAME_DEPTH_N], array_length;
+
+    /* XXX 'all_threads' ignored */
+    fn = fatal_error.dump_traceback;
+    if (fn) {
+        array_length = vmprof_get_traceback(NULL, ucontext,
+                                            array_p, FRAME_DEPTH_N);
+        fn(fd, array_p, array_length);
+    }
 }
 
-
-RPY_EXTERN
-void pypy_faulthandler_dump_traceback(int fd, int all_threads)
-{
-    fatal_error._current_fd = fd;
-
-    /* XXX 'all_threads' ignored */
-    if (fatal_error.dump_traceback)
-        fatal_error.dump_traceback();
-}
-
-void faulthandler_dump_traceback(int fd, int all_threads)
+static void
+faulthandler_dump_traceback(int fd, int all_threads, void *ucontext)
 {
     static volatile int reentrant = 0;
 
     if (reentrant)
         return;
     reentrant = 1;
-    pypy_faulthandler_dump_traceback(fd, all_threads);
+    pypy_faulthandler_dump_traceback(fd, all_threads, ucontext);
     reentrant = 0;
 }
 
@@ -103,7 +108,7 @@
    This function is signal-safe and should only call signal-safe functions. */
 
 static void
-faulthandler_fatal_error(int signum)
+faulthandler_fatal_error(int signum, siginfo_t *info, void *ucontext)
 {
     int fd = fatal_error.fd;
     int i;
@@ -123,11 +128,11 @@
         handler->enabled = 0;
     }
 
-    fh_write(fd, "Fatal Python error: ");
-    fh_write(fd, handler->name);
-    fh_write(fd, "\n\n");
+    pypy_faulthandler_write(fd, "Fatal Python error: ");
+    pypy_faulthandler_write(fd, handler->name);
+    pypy_faulthandler_write(fd, "\n\n");
 
-    faulthandler_dump_traceback(fd, fatal_error.all_threads);
+    faulthandler_dump_traceback(fd, fatal_error.all_threads, ucontext);
 
     errno = save_errno;
 #ifdef MS_WINDOWS
@@ -144,7 +149,7 @@
 
 
 RPY_EXTERN
-char *pypy_faulthandler_setup(void dump_callback(void))
+char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback)
 {
     if (fatal_error.initialized)
         return NULL;
@@ -201,11 +206,11 @@
             struct sigaction action;
             fault_handler_t *handler = &faulthandler_handlers[i];
 
-            action.sa_handler = faulthandler_fatal_error;
+            action.sa_sigaction = faulthandler_fatal_error;
             sigemptyset(&action.sa_mask);
             /* Do not prevent the signal from being received from within
                its own signal handler */
-            action.sa_flags = SA_NODEFER;
+            action.sa_flags = SA_NODEFER | SA_SIGINFO;
             if (stack.ss_sp != NULL) {
                 /* Call the signal handler on an alternate signal stack
                    provided by sigaltstack() */
diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -2,19 +2,24 @@
 #define PYPY_FAULTHANDLER_H
 
 #include "src/precommondefs.h"
+#include <stdint.h>
 
 
-RPY_EXTERN char *pypy_faulthandler_setup(void dump_callback(void));
+typedef void (*pypy_faulthandler_cb_t)(int fd, intptr_t *array_p,
+                                       intptr_t length);
+
+RPY_EXTERN char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback);
 RPY_EXTERN void pypy_faulthandler_teardown(void);
 
 RPY_EXTERN char *pypy_faulthandler_enable(int fd, int all_threads);
 RPY_EXTERN void pypy_faulthandler_disable(void);
 RPY_EXTERN int pypy_faulthandler_is_enabled(void);
 
-RPY_EXTERN void pypy_faulthandler_write(char *);
-RPY_EXTERN void pypy_faulthandler_write_int(long);
+RPY_EXTERN void pypy_faulthandler_write(int fd, const char *str);
+RPY_EXTERN void pypy_faulthandler_write_int(int fd, long value);
 
-RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads);
+RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads,
+                                                 void *ucontext);
 
 
 RPY_EXTERN int pypy_faulthandler_read_null(void);
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -1,5 +1,5 @@
 import os
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import llmemory, rffi
 from rpython.rlib.rposix import is_valid_fd
 from rpython.rlib.rarithmetic import widen
 from rpython.rlib.objectmodel import keepalive_until_here
@@ -71,7 +71,8 @@
         self.setup()
         cintf.pypy_faulthandler_dump_traceback(
             rffi.cast(rffi.INT, fileno),
-            rffi.cast(rffi.INT, all_threads))
+            rffi.cast(rffi.INT, all_threads),
+            llmemory.NULL)
         keepalive_until_here(w_file)
 
     def finish(self):
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -57,7 +57,7 @@
                                             compilation_info=eci,
                                             _nowrapper=True)
     vmprof_get_traceback = rffi.llexternal("vmprof_get_traceback",
-                                  [PVMPROFSTACK, lltype.Signed,
+                                  [PVMPROFSTACK, llmemory.Address,
                                    rffi.SIGNEDP, lltype.Signed],
                                   lltype.Signed, compilation_info=eci,
                                   _nowrapper=True)
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,3 +1,4 @@
+#include <stdint.h>
 
 RPY_EXTERN char *vmprof_init(int, double, char *);
 RPY_EXTERN void vmprof_ignore_signals(int);
@@ -8,6 +9,6 @@
 RPY_EXTERN int vmprof_stack_append(void*, long);
 RPY_EXTERN long vmprof_stack_pop(void*);
 RPY_EXTERN void vmprof_stack_free(void*);
-RPY_EXTERN intptr_t vmprof_get_traceback(void *, intptr_t, intptr_t*, intptr_t);
+RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t);
 
 #define RVMPROF_TRACEBACK_ESTIMATE_N(num_entries)  (2 * (num_entries) + 4)
diff --git a/rpython/rlib/rvmprof/src/vmprof_common.h b/rpython/rlib/rvmprof/src/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/vmprof_common.h
@@ -130,10 +130,11 @@
 #endif
 
 RPY_EXTERN
-intptr_t vmprof_get_traceback(void *stack, intptr_t pc,
+intptr_t vmprof_get_traceback(void *stack, void *ucontext,
                               intptr_t *result_p, intptr_t result_length)
 {
     int n;
+    intptr_t pc = ucontext ? GetPC((ucontext_t *)ucontext) : 0;
     if (stack == NULL)
         stack = get_vmprof_stack();
     n = get_stack_trace(stack, result_p, result_length - 2, pc);
diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py
--- a/rpython/rlib/rvmprof/traceback.py
+++ b/rpython/rlib/rvmprof/traceback.py
@@ -5,7 +5,7 @@
 
 from rpython.rlib.rvmprof import cintf, rvmprof
 from rpython.rlib.objectmodel import specialize
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 
 
 def traceback(estimate_number_of_entries):
@@ -17,7 +17,8 @@
     size = estimate_number_of_entries * 2 + 4
     stack = cintf.get_rvmprof_stack()
     array_p = lltype.malloc(rffi.SIGNEDP.TO, size, flavor='raw')
-    array_length = _cintf.vmprof_get_traceback(stack, 0, array_p, size)
+    NULL = llmemory.NULL
+    array_length = _cintf.vmprof_get_traceback(stack, NULL, array_p, size)
     return (array_p, array_length)
 
 

From pypy.commits at gmail.com  Tue Sep 27 05:27:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 27 Sep 2016 02:27:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Tweak the build_cffi_imports output:
 clarify the status in case of
Message-ID: <57ea3b70.06b0c20a.630a7.8385@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r87412:70287b5cb024
Date: 2016-09-27 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/70287b5cb024/

Log:	Tweak the build_cffi_imports output: clarify the status in case of
	errors, remove any message in case everything works, and make sure
	we get the same explanations when running rpython from the Makefile

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py
+	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -303,37 +303,20 @@
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_* to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.tool.runsubprocess import run_subprocess
         from rpython.translator.driver import taskdef
         import types
 
-        class Options(object):
-            pass
-
-
-        def mkexename(name):
-            if sys.platform == 'win32':
-                name = name.new(ext='exe')
-            return name
-
         compile_goal, = driver.backend_select_goals(['compile'])
         @taskdef([compile_goal], "Create cffi bindings for modules")
         def task_build_cffi_imports(self):
-            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
             ''' Use cffi to compile cffi interfaces to modules'''
-            exename = mkexename(driver.compute_exe_name())
-            basedir = exename
-            while not basedir.join('include').exists():
-                _basedir = basedir.dirpath()
-                if _basedir == basedir:
-                    raise ValueError('interpreter %s not inside pypy repo',
-                                     str(exename))
-                basedir = _basedir
-            modules = self.config.objspace.usemodules.getpaths()
-            options = Options()
-            # XXX possibly adapt options using modules
-            failures = create_cffi_import_libraries(exename, options, basedir)
-            # if failures, they were already printed
-            print  >> sys.stderr, str(exename),'successfully built (errors, if any, while building the above modules are ignored)'
+            filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py')
+            status, out, err = run_subprocess(str(driver.compute_exe_name()),
+                                              [filename])
+            sys.stdout.write(out)
+            sys.stderr.write(err)
+            # otherwise, ignore errors
         driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
         driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal]
         driver.default_goal = 'build_cffi_imports'
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -44,8 +44,8 @@
 if __name__ == '__main__':
     import py, os
     if '__pypy__' not in sys.builtin_module_names:
-        print 'Call with a pypy interpreter'
-        sys.exit(-1)
+        print >> sys.stderr, 'Call with a pypy interpreter'
+        sys.exit(1)
 
     class Options(object):
         pass
@@ -59,18 +59,26 @@
                                  str(exename))
         basedir = _basedir
     options = Options()
-    print >> sys.stderr, "There should be no failures here"
     failures = create_cffi_import_libraries(exename, options, basedir)
     if len(failures) > 0:
-        print '*** failed to build', [f[1] for f in failures]
+        print >> sys.stderr, '*** failed to build the CFFI modules %r' % (
+            [f[1] for f in failures],)
+        print >> sys.stderr, '''
+PyPy can still be used as long as you don't need the corresponding
+modules.  If you do need them, please install the missing headers and
+libraries (see error messages just above) and then re-run the command:
+
+    %s %s
+''' % (sys.executable, ' '.join(sys.argv))
         sys.exit(1)
 
-    # monkey patch a failure, just to test
-    print >> sys.stderr, 'This line should be followed by a traceback'
-    for k in cffi_build_scripts:
-        setattr(options, 'no_' + k, True)
-    must_fail = '_missing_build_script.py'
-    assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
-    cffi_build_scripts['should_fail'] = must_fail
-    failures = create_cffi_import_libraries(exename, options, basedir)
-    assert len(failures) == 1
+    if len(sys.argv) > 1 and sys.argv[1] == '--test':
+        # monkey patch a failure, just to test
+        print >> sys.stderr, 'This line should be followed by a traceback'
+        for k in cffi_build_scripts:
+            setattr(options, 'no_' + k, True)
+        must_fail = '_missing_build_script.py'
+        assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+        cffi_build_scripts['should_fail'] = must_fail
+        failures = create_cffi_import_libraries(exename, options, basedir)
+        assert len(failures) == 1

From pypy.commits at gmail.com  Tue Sep 27 05:37:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 27 Sep 2016 02:37:47 -0700 (PDT)
Subject: [pypy-commit] cffi default: Fix leak. Remaining issue: if the user
 close()s the file object in Python, 
Message-ID: <57ea3deb.04141c0a.ac6c0.124b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2779:630f22d1d446
Date: 2016-09-27 11:37 +0200
http://bitbucket.org/cffi/cffi/changeset/630f22d1d446/

Log:	Fix leak. Remaining issue: if the user close()s the file object in
	Python, the __cffi_FILE attribute remains there and the dup'ed file
	descriptor remains open.

diff --git a/c/file_emulator.h b/c/file_emulator.h
--- a/c/file_emulator.h
+++ b/c/file_emulator.h
@@ -31,7 +31,7 @@
 static FILE *PyFile_AsFile(PyObject *ob_file)
 {
     PyObject *ob, *ob_capsule = NULL, *ob_mode = NULL;
-    FILE *f = NULL;
+    FILE *f;
     int fd;
     char *mode;
 
@@ -80,7 +80,11 @@
         if (PyObject_SetAttrString(ob_file, "__cffi_FILE", ob_capsule) < 0)
             goto fail;
     }
-    return PyCapsule_GetPointer(ob_capsule, "FILE");
+    else {
+        f = PyCapsule_GetPointer(ob_capsule, "FILE");
+    }
+    Py_DECREF(ob_capsule);   /* assumes still at least one reference */
+    return f;
 
  fail:
     Py_XDECREF(ob_mode);

From pypy.commits at gmail.com  Tue Sep 27 07:46:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 27 Sep 2016 04:46:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: unused data is now added again
 for decompression
Message-ID: <57ea5c29.e50fc20a.c9259.bee5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87414:aab9dfb33c91
Date: 2016-09-27 13:46 +0200
http://bitbucket.org/pypy/pypy/changeset/aab9dfb33c91/

Log:	unused data is now added again for decompression

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -450,6 +450,13 @@
                         if out.get_data_size() == max_length:
                             break
                         out.prepare_next_chunk()
+
+                if not self.running:
+                    self.needs_input = False
+                    if self.left_to_process != 0:
+                        end = len(data)
+                        start = end - self.left_to_process
+                        self.unused_data = data[start:]
                 res = out.make_result_string()
                 return self.space.newbytes(res)
 
@@ -469,7 +476,6 @@
         if data == '':
             return self.space.newbytes('')
         datalen = len(data)
-        import pdb; pdb.set_trace()
         if self.input_buffer:
             input_buffer_in_use = True
             result = self._decompress_buf(self.input_buffer, max_length)
diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -1,6 +1,8 @@
 import os
 
 import py
+import glob
+import bz2
 
 from pypy.module.bz2.test.support import CheckAllocation
 from pypy.module.bz2 import interp_bz2
@@ -37,6 +39,15 @@
     mod.OLD_SMALLCHUNK = interp_bz2.INITIAL_BUFFER_SIZE
     interp_bz2.INITIAL_BUFFER_SIZE = 32
 
+    test_size = 0
+    mod.BIG_TEXT = bytearray(128*1024)
+    for fname in glob.glob(os.path.join(os.path.dirname(__file__), '*.py')):
+        with open(fname, 'rb') as fh:
+            test_size += fh.readinto(memoryview(BIG_TEXT)[test_size:])
+        if test_size > 128*1024:
+            break
+    mod.BIG_DATA = bz2.compress(BIG_TEXT, compresslevel=1)
+
 def teardown_module(mod):
     interp_bz2.INITIAL_BUFFER_SIZE = mod.OLD_SMALLCHUNK
 
@@ -122,6 +133,8 @@
     def setup_class(cls):
         cls.w_TEXT = cls.space.newbytes(TEXT)
         cls.w_DATA = cls.space.newbytes(DATA)
+        cls.w_BIG_DATA = cls.space.newbytes(BIG_DATA)
+        cls.w_BIG_TEXT = cls.space.newbytes(BIG_TEXT)
         cls.w_BUGGY_DATA = cls.space.newbytes(BUGGY_DATA)
 
         cls.space.appexec([], """(): import warnings""")  # Work around a recursion limit
@@ -206,8 +219,8 @@
         bz2d = BZ2Decompressor()
         decomp= []
 
-        length = len(self.DATA)
-        decomp.append(bz2d.decompress(self.DATA[:length-64]))
+        length = len(self.BIG_DATA)
+        decomp.append(bz2d.decompress(self.BIG_DATA[:length-64]), max_length=100)
         assert bz2d.needs_input == False
         assert len(decomp[-1]) == 100
 
@@ -215,7 +228,7 @@
         assert bz2d.needs_input == False
         assert len(decomp[-1]) == 50
 
-        decomp.append(bz2d.decompress(self.DATA[length-64:], max_length=50))
+        decomp.append(bz2d.decompress(self.BIG_DATA[length-64:], max_length=50))
         assert bz2d.needs_input == False
         assert len(decomp[-1]) == 50
 
@@ -223,7 +236,7 @@
             decomp.append(bz2d.decompress(b"", max_length=50))
             assert len(decomp[-1]) <= 50
 
-        assert ''.join(decomp) == self.TEXT
+        assert ''.join(decomp) == self.BIG_TEXT
 
 
 class AppTestBZ2ModuleFunctions(CheckAllocation):

From pypy.commits at gmail.com  Tue Sep 27 07:51:01 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 27 Sep 2016 04:51:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: py3.5 modify test,
 read accepts None as first argument
Message-ID: <57ea5d25.29a9c20a.ebe31.bf79@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87415:e07e6c2860a6
Date: 2016-09-27 13:50 +0200
http://bitbucket.org/pypy/pypy/changeset/e07e6c2860a6/

Log:	py3.5 modify test, read accepts None as first argument

diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -256,7 +256,6 @@
         self.create_temp_file()
 
         bz2f = BZ2File(self.temppath)
-        raises(TypeError, bz2f.read, None)
         text_read = bz2f.read()
         assert text_read == self.TEXT
         bz2f.close()

From pypy.commits at gmail.com  Tue Sep 27 07:46:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 27 Sep 2016 04:46:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: tracking the rest of the buffer
 that has not been read
Message-ID: <57ea5c27.87adc20a.79a1b.b86f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87413:b2c8a6a569f3
Date: 2016-09-27 12:34 +0200
http://bitbucket.org/pypy/pypy/changeset/b2c8a6a569f3/

Log:	tracking the rest of the buffer that has not been read

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -200,7 +200,7 @@
         else:
             size = max_length
         self._allocate_chunk(size)
-        self.avail_in_real = 0
+        self.left = 0
 
     def get_data_size(self):
         return 0
@@ -225,7 +225,10 @@
     def prepare_next_chunk(self):
         size = self.current_size
         self.temp.append(self._get_chunk(size))
-        self._allocate_chunk(_new_buffer_size(size))
+        newsize = size
+        if self.max_length == -1:
+            newsize = _new_buffer_size(size)
+        self._allocate_chunk(newsize)
 
     def make_result_string(self):
         count_unoccupied = rffi.getintfield(self.bzs, 'c_avail_out')
@@ -382,8 +385,9 @@
         try:
             self.running = False
             self.unused_data = ""
-            self.needs_input = 1
+            self.needs_input = True
             self.input_buffer = None
+            self.left_to_process = 0
 
             self._init_bz2decomp()
         except:
@@ -422,29 +426,31 @@
 
     def _decompress_buf(self, data, max_length):
         in_bufsize = len(data)
-
         with rffi.scoped_nonmovingbuffer(data) as in_buf:
+            # setup the input and the size it can consume
             self.bzs.c_next_in = in_buf
             rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
+            self.left_to_process -= in_bufsize
 
             with OutBuffer(self.bzs, max_length=max_length) as out:
                 while True:
-                    bzerror = BZ2_bzDecompress(self.bzs)
-                    if bzerror == BZ_STREAM_END:
+                    bzreturn = BZ2_bzDecompress(self.bzs)
+                    # add up the size that has not been processed
+                    avail_in = rffi.getintfield(self.bzs, 'c_avail_in')
+                    self.left_to_process += avail_in
+                    if bzreturn == BZ_STREAM_END:
                         self.running = False
                         break
-                    if bzerror != BZ_OK:
-                        _catch_bz2_error(self.space, bzerror)
+                    if bzreturn != BZ_OK:
+                        _catch_bz2_error(self.space, bzreturn)
 
-                    if rffi.getintfield(self.bzs, 'c_avail_in') == 0:
+                    if self.left_to_process == 0:
                         break
                     elif rffi.getintfield(self.bzs, 'c_avail_out') == 0:
                         if out.get_data_size() == max_length:
                             break
                         out.prepare_next_chunk()
                 res = out.make_result_string()
-                # might be non zero if max_length has been specified
-                self.left_to_process = out.left
                 return self.space.newbytes(res)
 
     @unwrap_spec(data='bufferstr', max_length=int)
@@ -462,22 +468,22 @@
                         "end of stream was already found")
         if data == '':
             return self.space.newbytes('')
-
-        bzs = self.bzs
-        if not self.input_buffer:
+        datalen = len(data)
+        import pdb; pdb.set_trace()
+        if self.input_buffer:
             input_buffer_in_use = True
             result = self._decompress_buf(self.input_buffer, max_length)
         else:
             input_buffer_in_use = False
+            self.left_to_process = datalen
             result = self._decompress_buf(data, max_length)
 
         if self.left_to_process == 0:
             self.input_buffer = None
-            self.need_input = 1
+            self.needs_input = True
         else:
-            self.need_input = 0
+            self.needs_input = False
             if not input_buffer_in_use:
-                datalen = len(data)
                 self.input_buffer = data[datalen-self.left_to_process-1:]
 
         return result
diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -34,11 +34,11 @@
     mod.decompress = decompress
     #
     # For tests, patch the value of SMALLCHUNK
-    mod.OLD_SMALLCHUNK = interp_bz2.SMALLCHUNK
-    interp_bz2.SMALLCHUNK = 32
+    mod.OLD_SMALLCHUNK = interp_bz2.INITIAL_BUFFER_SIZE
+    interp_bz2.INITIAL_BUFFER_SIZE = 32
 
 def teardown_module(mod):
-    interp_bz2.SMALLCHUNK = mod.OLD_SMALLCHUNK
+    interp_bz2.INITIAL_BUFFER_SIZE = mod.OLD_SMALLCHUNK
 
 class AppTestBZ2Compressor(CheckAllocation):
     spaceconfig = dict(usemodules=('bz2', 'time', 'struct'))
@@ -200,6 +200,31 @@
         exc = raises(TypeError, pickle.dumps, BZ2Decompressor())
         assert exc.value.args[0] == "cannot serialize '_bz2.BZ2Decompressor' object"
 
+    def test_decompress_max_length(self):
+        from bz2 import BZ2Decompressor
+
+        bz2d = BZ2Decompressor()
+        decomp= []
+
+        length = len(self.DATA)
+        decomp.append(bz2d.decompress(self.DATA[:length-64]))
+        assert bz2d.needs_input == False
+        assert len(decomp[-1]) == 100
+
+        decomp.append(bz2d.decompress(b"", max_length=50))
+        assert bz2d.needs_input == False
+        assert len(decomp[-1]) == 50
+
+        decomp.append(bz2d.decompress(self.DATA[length-64:], max_length=50))
+        assert bz2d.needs_input == False
+        assert len(decomp[-1]) == 50
+
+        while not bz2d.eof:
+            decomp.append(bz2d.decompress(b"", max_length=50))
+            assert len(decomp[-1]) <= 50
+
+        assert ''.join(decomp) == self.TEXT
+
 
 class AppTestBZ2ModuleFunctions(CheckAllocation):
     spaceconfig = dict(usemodules=('bz2', 'time'))

From pypy.commits at gmail.com  Tue Sep 27 12:42:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 27 Sep 2016 09:42:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Test and fix for allowing __len__ to
 return any index-like
Message-ID: <57eaa185.a6ecc20a.41325.3652@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87416:494a05343a22
Date: 2016-09-27 17:42 +0100
http://bitbucket.org/pypy/pypy/changeset/494a05343a22/

Log:	Test and fix for allowing __len__ to return any index-like

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -277,7 +277,7 @@
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj, allow_conversion=False)
+        result = space.getindex_w(w_obj, space.w_OverflowError)
         if result < 0:
             raise oefmt(space.w_ValueError, "__len__() should return >= 0")
         return result
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -293,7 +293,7 @@
             x = operate(A())
             assert x == '&#224;&#232;&#236;'
             assert type(x) is str
-            
+
 
     def test_byte_results_unicode(self):
         class A(object):
@@ -692,6 +692,34 @@
         raises(TypeError, bool, X(X(2)))
         raises(OverflowError, len, X(sys.maxsize + 1))
 
+    def test_len_index(self):
+        class Index(object):
+            def __index__(self):
+                return 42
+        class X(object):
+            def __len__(self):
+                return Index()
+        n = len(X())
+        assert type(n) is int
+        assert n == 42
+
+        class BadIndex(object):
+            def __index__(self):
+                return 'foo'
+        class Y(object):
+            def __len__(self):
+                return BadIndex()
+        excinfo = raises(TypeError, len, Y())
+        assert excinfo.value.args[0].startswith("__index__ returned non-")
+
+        class BadIndex2(object):
+            def __index__(self):
+                return 2**100
+        class Z(object):
+            def __len__(self):
+                return BadIndex2()
+        excinfo = raises(OverflowError, len, Z())
+
     def test_sane_len(self):
         # this test just tests our assumptions about __len__
         # this will start failing if __len__ changes assertions

From pypy.commits at gmail.com  Tue Sep 27 14:01:10 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 27 Sep 2016 11:01:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Skip PyPy-specific test on CPython
Message-ID: <57eab3e6.948e1c0a.686bf.eb26@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87418:101d0434327d
Date: 2016-09-27 19:00 +0100
http://bitbucket.org/pypy/pypy/changeset/101d0434327d/

Log:	Skip PyPy-specific test on CPython

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -1,7 +1,9 @@
 from __future__ import with_statement
+import pytest
 from rpython.tool.udir import udir
 
 
+
 class AppTestImpModule:
     # cpyext is required for _imp.load_dynamic()
     spaceconfig = {
@@ -289,4 +291,6 @@
     def test_get_tag(self):
         import imp
         import sys
+        if not hasattr(sys, 'pypy_version_info'):
+            skip('This test is PyPy-only')
         assert imp.get_tag() == 'pypy3-%d%d' % sys.pypy_version_info[0:2]

From pypy.commits at gmail.com  Tue Sep 27 14:13:27 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 27 Sep 2016 11:13:27 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Get simple test working
Message-ID: <57eab6c7.d8941c0a.4c1d9.eea2@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87419:afd1c22c477e
Date: 2016-09-27 14:12 -0400
http://bitbucket.org/pypy/pypy/changeset/afd1c22c477e/

Log:	Get simple test working

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -337,7 +337,7 @@
             try:
                 args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
                     args, self.optimizer, force_boxes=force_boxes)
-            except VirtualStatesCantMatch as e:
+            except VirtualStatesCantMatch:
                 assert force_boxes
                 virtual_state = self.get_virtual_state(args)
                 continue
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -15,7 +15,7 @@
     _last_guard_not_forced_2 = None
 
     def setup(self):
-        self.optimizer.optheap = self
+        self.optimizer.optvirtualize = self
 
     def make_virtual(self, known_class, source_op, descr):
         opinfo = info.InstancePtrInfo(descr, known_class, is_virtual=True)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -222,14 +222,12 @@
                     self, boxes, box, optimizer, force_boxes)
 
         optvirtualize = optimizer.optimizer.optvirtualize
-        if optvirtualize is None:
-            raise VirtualStatesCantMatch()
-
+        assert optvirtualize is not None
         assert isinstance(info, AbstractStructPtrInfo)
 
         newop = optvirtualize.make_virtual(self.known_class, box, self.typedescr)
         newop._fields = info._fields[:]
-        for i in range(min(len(self.fielddescrs, len(info._fields)))):
+        for i in range(min(len(self.fielddescrs), len(info._fields))):
             state = self.fieldstate[i]
             if state is None:
                 continue
@@ -250,7 +248,7 @@
         if not known_class.is_value_class():
             raise VirtualStatesCantMatch("different kinds of structs")
 
-        raise VirtualStatesCantMatch("different kinds of structs")
+        # raise VirtualStatesCantMatch("different kinds of structs")
         # TODO: Probably should rename state.extra_guards to extra_ops
         extra_guards = state.extra_guards
         cpu = state.cpu
@@ -260,7 +258,7 @@
         if other.level == LEVEL_UNKNOWN:
             op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class])
             extra_guards.append(op)
-        elif other.level == LEVEL_NONULL:
+        elif other.level == LEVEL_NONNULL:
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class])
             extra_guards.append(op)
         elif other.level == LEVEL_KNOWNCLASS:

From pypy.commits at gmail.com  Tue Sep 27 14:59:44 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 27 Sep 2016 11:59:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Replace marshal with _md5 in test,
 so that it works on CPython (no idea why this is needed, but whatevs)
Message-ID: <57eac1a0.a6ecc20a.41325.6dd3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87420:885295b9e9ea
Date: 2016-09-27 19:59 +0100
http://bitbucket.org/pypy/pypy/changeset/885295b9e9ea/

Log:	Replace marshal with _md5 in test, so that it works on CPython (no
	idea why this is needed, but whatevs)

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -265,28 +265,28 @@
         # taken from https://bugs.pypy.org/issue1514, with extra cases
         # that show a difference with CPython: we can get on CPython
         # several module objects for the same built-in module :-(
-        import sys, marshal
+        import sys, _md5
 
-        old = marshal.loads
-        marshal.loads = 42
+        old = _md5.md5
+        _md5.md5 = 42
 
         # save, re-import, restore.
-        saved = sys.modules.pop('marshal')
-        marshal2 = __import__('marshal')
-        assert marshal2 is not marshal
-        assert marshal2.loads is old
-        assert marshal2 is sys.modules['marshal']
-        assert marshal is saved
-        assert marshal.loads == 42
+        saved = sys.modules.pop('_md5')
+        _md52 = __import__('_md5')
+        assert _md52 is not _md5
+        assert _md52.md5 is old
+        assert _md52 is sys.modules['_md5']
+        assert _md5 is saved
+        assert _md5.md5 == 42
 
-        import marshal
-        assert marshal.loads is old
+        import _md5
+        assert _md5.md5 is old
 
-        sys.modules['marshal'] = saved
-        import marshal
-        assert marshal.loads == 42
+        sys.modules['_md5'] = saved
+        import _md5
+        assert _md5.md5 == 42
 
-        marshal.loads = old
+        _md5.md5 = old
 
     def test_get_tag(self):
         import imp

From pypy.commits at gmail.com  Tue Sep 27 15:07:50 2016
From: pypy.commits at gmail.com (remarkablerocket)
Date: Tue, 27 Sep 2016 12:07:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Make int() called with invalid argument
 type raise error message consistent with CPython
Message-ID: <57eac386.0b3b1c0a.dac6f.0f5b@mx.google.com>

Author: Daniel Patrick <danieljudepatrick at gmail.com>
Branch: py3.5
Changeset: r87421:8bfd0fa78952
Date: 2016-09-27 19:21 +0100
http://bitbucket.org/pypy/pypy/changeset/8bfd0fa78952/

Log:	Make int() called with invalid argument type raise error message
	consistent with CPython Existing test was failing when run with
	--python=python3

diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -886,8 +886,8 @@
                 if not e.match(space, space.w_TypeError):
                     raise
                 raise oefmt(space.w_TypeError,
-                            "int() argument must be a string or a number, "
-                            "not '%T'", w_value)
+                            "int() argument must be a string, a bytes-like "
+                            "object or a number, not '%T'", w_value)
             else:
                 return _string_to_int_or_long(space, w_inttype, w_value, buf)
     else:
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -501,8 +501,8 @@
 
     def test_int_error_msg(self):
         e = raises(TypeError, int, [])
-        assert str(e.value) == (
-            "int() argument must be a string or a number, not 'list'")
+        assert str(e.value) == ("int() argument must be a string, a bytes-"
+                                "like object or a number, not 'list'")
 
     def test_invalid_literal_message(self):
         import sys

From pypy.commits at gmail.com  Tue Sep 27 15:33:25 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 27 Sep 2016 12:33:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Fix NullImporter test and revert
 unnecessary stdlib change
Message-ID: <57eac985.94071c0a.88158.945b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87422:ea9cd722574d
Date: 2016-09-27 20:32 +0100
http://bitbucket.org/pypy/pypy/changeset/ea9cd722574d/

Log:	Fix NullImporter test and revert unnecessary stdlib change

diff --git a/lib-python/3/imp.py b/lib-python/3/imp.py
--- a/lib-python/3/imp.py
+++ b/lib-python/3/imp.py
@@ -73,7 +73,7 @@
         elif os.path.isdir(path):
             raise ImportError('existing directory', path=path)
 
-    def find_module(self, fullname, path=None):
+    def find_module(self, fullname):
         """Always returns None."""
         return None
 
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -169,7 +169,7 @@
     def test_nullimporter(self):
         import os, imp
         importer = imp.NullImporter("path")
-        assert importer.find_module(1, [2, 3, 4]) is None
+        assert importer.find_module(1) is None
         raises(ImportError, imp.NullImporter, os.getcwd())
 
     def test_path_importer_cache(self):

From pypy.commits at gmail.com  Tue Sep 27 16:12:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 27 Sep 2016 13:12:23 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: Next crasher
Message-ID: <57ead2a7.4319c20a.46ad4.870e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5717:3db40beefd28
Date: 2016-09-27 22:12 +0200
http://bitbucket.org/pypy/extradoc/changeset/3db40beefd28/

Log:	Next crasher

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -36,6 +36,11 @@
   this byte has the value 72, which is YIELD_FROM.  Total nonsense and
   crashes follow.
 
+* faulthandler: register(): the signal handler, faulthandler_user(),
+  changes errno in faulthandler_dump_traceback() but fails to restore it
+  if chain=False.  This can rarely cause random nonsense in the main
+  program.
+
 
 Other bugs
 ----------

From pypy.commits at gmail.com  Tue Sep 27 16:22:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 27 Sep 2016 13:22:52 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: tweaks
Message-ID: <57ead51c.ca11c30a.8a70a.8c9a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5718:bbe524770392
Date: 2016-09-27 22:22 +0200
http://bitbucket.org/pypy/extradoc/changeset/bbe524770392/

Log:	tweaks

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -54,11 +54,12 @@
   of buf being a read-write buffer, which is then mutated in-place.
   But if we call with a read-only buffer, mutate_flag is ignored (instead
   of rejecting a True value)---ioctl(x, y, "foo", True) will not actually
-  mutate the string "foo", but the True is completely ignored.
+  mutate the string "foo", but the True is completely ignored.  (I think
+  this is a bug introduced during the Argument Clinic refactoring.)
 
 * re.sub(b'y', bytearray(b'a'), bytearray(b'xyz')) -> b'xaz'
   re.sub(b'y', bytearray(b'\\n'), bytearray(b'xyz')) -> internal TypeError
- 
+
 * not a bug: argument clinic turns the "bool" specifier into
   PyObject_IsTrue(), accepting any argument whatsoever.  This can easily
   get very confusing for the user, e.g. after messing up the number of
@@ -84,8 +85,8 @@
   issue tracker: ``class C: __new__=int.__new__`` and ``class C(int):
   __new__=object.__new__`` can each be instantiated, even though they
   shouldn't.  This is because ``__new__`` is completely ignored if it is
-  set to a built-in function that uses ``tp_new_wrapper`` as its C code.
-  This is true for most or all the built-in types' ``__new__``.
+  set to any built-in function that uses ``tp_new_wrapper`` as its C code
+  (many of the built-in types' ``__new__`` are like that).
   http://bugs.python.org/issue1694663#msg75957,
   http://bugs.python.org/issue5322#msg84112.  In (at least) CPython 3.5,
   a few classes work only thanks to abuse of this bug: for example,

From pypy.commits at gmail.com  Tue Sep 27 19:49:14 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 27 Sep 2016 16:49:14 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Catch edge case
Message-ID: <57eb057a.4524c20a.5a2b9.ba2d@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87424:8b4d2fcf1975
Date: 2016-09-27 16:42 -0400
http://bitbucket.org/pypy/pypy/changeset/8b4d2fcf1975/

Log:	Catch edge case

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -237,7 +237,8 @@
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
 
-        if info is None or info.is_virtual():
+        if (not self.typedescr.is_value_class() or
+            info is None or info.is_virtual()):
             return AbstractVirtualStructStateInfo.enum_forced_boxes(
                     self, boxes, box, optimizer, force_boxes)
 

From pypy.commits at gmail.com  Tue Sep 27 19:49:16 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 27 Sep 2016 16:49:16 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Fix tests
Message-ID: <57eb057c.0cce1c0a.37d46.4fce@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87425:84b5711bcbb1
Date: 2016-09-27 19:48 -0400
http://bitbucket.org/pypy/pypy/changeset/84b5711bcbb1/

Log:	Fix tests

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -7,6 +7,7 @@
 from rpython.jit.metainterp.resoperation import rop, ResOperation, \
      InputArgInt, InputArgRef, InputArgFloat
 from rpython.rlib.debug import debug_print
+from rpython.rlib.objectmodel import specialize
 
 LEVEL_UNKNOWN = '\x00'
 LEVEL_NONNULL = '\x01'
@@ -213,6 +214,7 @@
         return (isinstance(other, VirtualStateInfo) and
                 self.typedescr is other.typedescr)
 
+    @specialize.argtype(4)
     def getfield(self, box, info, descr, optimizer):
         assert isinstance(info, AbstractStructPtrInfo)
         field = info._fields[descr.get_index()]
@@ -220,17 +222,23 @@
             return field
         opnum = rop.getfield_for_descr(descr)
         getfield = ResOperation(opnum, [box], descr=descr)
-        optimizer.emit_operation(getfield)
+        if isinstance(optimizer, list):
+            optimizer.append(getfield)
+        else:
+            from rpython.jit.metainterp.optimizeopt import Optimizer
+            assert isinstance(optimizer, Optimizer)
+            optimizer.emit_operation(getfield)
+        info._fields[descr.get_index()] = getfield
         return getfield
 
     def make_virtual_copy(self, box, info, optimizer):
         optvirtualize = optimizer.optvirtualize
         # newop = ResOperation(rop.NEW_WITH_VTABLE, [], descr=self.typedescr)
         opinfo = optvirtualize.make_virtual(self.known_class, box, self.typedescr)
-        # optimizer.emit_operation(newop)
         for i in range(len(info._fields)):
             descr = self.fielddescrs[i]
             opinfo._fields[i] = self.getfield(box, info, descr, optimizer)
+        # optimizer.emit_operation(newop)
         return opinfo
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
@@ -265,8 +273,7 @@
         """
         assert isinstance(other, NotVirtualStateInfoPtr)
 
-        known_class = self.typedescr
-        if not known_class.is_value_class():
+        if not self.typedescr.is_value_class():
             raise VirtualStatesCantMatch("different kinds of structs")
 
         # raise VirtualStatesCantMatch("different kinds of structs")
@@ -312,20 +319,18 @@
         # virtual object.
         # This will probably look a lot like
         # AbstractVirtualStateInfo._generate_guards
-        fields = []
         for i, descr in enumerate(self.fielddescrs):
             if runtime_box is not None and opinfo is not None:
-                fieldbox = opinfo._fields[descr.get_index()]
+                fieldbox = self.getfield(box, opinfo, descr, extra_guards)
                 fieldbox_runtime = state.get_runtime_field(runtime_box, descr)
             else:
                 opnum = rop.getfield_for_descr(descr)
                 fieldbox = ResOperation(opnum, [box], descr=descr)
                 extra_guards.append(fieldbox)
                 fieldbox_runtime = None
-            fields.append(fieldbox)
-            faux_field_state = not_virtual(cpu, fieldbox.type, opinfo)
+            other_field_state = not_virtual(cpu, fieldbox.type, opinfo)
             fieldstate = self.fieldstate[i]
-            fieldstate.generate_guards(faux_field_state, fieldbox,
+            fieldstate.generate_guards(other_field_state, fieldbox,
                                        fieldbox_runtime, state)
 
     def debug_header(self, indent):
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1043,6 +1043,7 @@
             return 0
 
         self.meta_interp(f, [])
+        import pdb; pdb.set_trace()
 
     def test_aliased_virtual_states(self):
         # All cases are covered when forcing one component of the virtual state

From pypy.commits at gmail.com  Tue Sep 27 21:21:13 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 27 Sep 2016 18:21:13 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Guard recursion in generate_guards
Message-ID: <57eb1b09.02d31c0a.c19d0.5dfb@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87426:27384c89faf6
Date: 2016-09-27 20:11 -0400
http://bitbucket.org/pypy/pypy/changeset/27384c89faf6/

Log:	Guard recursion in generate_guards

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -21,6 +21,8 @@
 
 
 class GenerateGuardState(object):
+    _attrs_ = ('optimizer', 'cpu', 'extra_guards', 'renum', 'bad', 'force_boxes')
+
     def __init__(self, optimizer=None, guards=None, renum=None, bad=None, force_boxes=False):
         self.optimizer = optimizer
         self.cpu = optimizer.cpu
@@ -233,12 +235,10 @@
 
     def make_virtual_copy(self, box, info, optimizer):
         optvirtualize = optimizer.optvirtualize
-        # newop = ResOperation(rop.NEW_WITH_VTABLE, [], descr=self.typedescr)
         opinfo = optvirtualize.make_virtual(self.known_class, box, self.typedescr)
         for i in range(len(info._fields)):
             descr = self.fielddescrs[i]
             opinfo._fields[i] = self.getfield(box, info, descr, optimizer)
-        # optimizer.emit_operation(newop)
         return opinfo
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
@@ -250,13 +250,12 @@
             return AbstractVirtualStructStateInfo.enum_forced_boxes(
                     self, boxes, box, optimizer, force_boxes)
 
-        optvirtualize = optimizer.optimizer
         assert isinstance(info, AbstractStructPtrInfo)
 
         # TODO: Do we need to create a new object via NEW_WITH_VTABLE, or will the
         # allocation be handled properly?
         opinfo = self.make_virtual_copy(box, info, optimizer.optimizer)
-        for i in range(min(len(self.fielddescrs), len(info._fields))):
+        for i in range(min(len(self.fielddescrs), len(opinfo._fields))):
             state = self.fieldstate[i]
             if state is None:
                 continue
@@ -328,10 +327,11 @@
                 fieldbox = ResOperation(opnum, [box], descr=descr)
                 extra_guards.append(fieldbox)
                 fieldbox_runtime = None
-            other_field_state = not_virtual(cpu, fieldbox.type, opinfo)
             fieldstate = self.fieldstate[i]
-            fieldstate.generate_guards(other_field_state, fieldbox,
-                                       fieldbox_runtime, state)
+            if fieldstate is not None:
+                other_field_state = not_virtual(cpu, fieldbox.type, opinfo)
+                fieldstate.generate_guards(other_field_state, fieldbox,
+                                           fieldbox_runtime, state)
 
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1043,7 +1043,6 @@
             return 0
 
         self.meta_interp(f, [])
-        import pdb; pdb.set_trace()
 
     def test_aliased_virtual_states(self):
         # All cases are covered when forcing one component of the virtual state

From pypy.commits at gmail.com  Tue Sep 27 21:21:15 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 27 Sep 2016 18:21:15 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Add test for case of
 Virtual/ConstPtr (failing currently)
Message-ID: <57eb1b0b.e9a8c20a.5f0c1.fff4@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87427:b4954bdf7ddc
Date: 2016-09-27 21:20 -0400
http://bitbucket.org/pypy/pypy/changeset/b4954bdf7ddc/

Log:	Add test for case of Virtual/ConstPtr (failing currently)

diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1044,6 +1044,35 @@
 
         self.meta_interp(f, [])
 
+    def test_avoid_preamble_3(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+        class X(object):
+            _value_class_ = True
+            def __init__(self, v):
+                self.v = v
+
+        const = X(5)
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            val   = X(0)
+            i     = 0
+            while i < 17:
+                driver.can_enter_jit(i=i, val=val)
+                driver.jit_merge_point(i=i, val=val)
+                # Logical & rather than comparison to confuse range analysis.
+                # Test only succeeds on the first 2 iterations
+                if i & -2 == 0:
+                    val = X(i)
+                else:
+                    val = const
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
+
     def test_aliased_virtual_states(self):
         # All cases are covered when forcing one component of the virtual state
         # also forces an as yet unseen component.

From pypy.commits at gmail.com  Tue Sep 27 21:38:50 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 27 Sep 2016 18:38:50 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Stop gap for dealing with
 Virtual/ConstPtr case
Message-ID: <57eb1f2a.d7ba1c0a.14c16.398e@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87428:eb82006f754f
Date: 2016-09-27 21:34 -0400
http://bitbucket.org/pypy/pypy/changeset/eb82006f754f/

Log:	Stop gap for dealing with Virtual/ConstPtr case

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,7 +1,7 @@
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.info import ArrayPtrInfo,\
-     ArrayStructInfo, AbstractStructPtrInfo
+     ArrayStructInfo, AbstractStructPtrInfo, ConstPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound, IntLowerBound, IntUnbounded
 from rpython.jit.metainterp.resoperation import rop, ResOperation, \
@@ -310,7 +310,9 @@
 
         if runtime_box is not None:
             opinfo = state.optimizer.getptrinfo(box)
-            assert opinfo is None or isinstance(opinfo, AbstractStructPtrInfo)
+            if opinfo is not None and not isinstance(opinfo, AbstractStructPtrInfo):
+                assert isinstance(opinfo, ConstPtrInfo)
+                raise VirtualStatesCantMatch("cannot yet handle const pointers")
         else:
             opinfo = None
 

From pypy.commits at gmail.com  Tue Sep 27 23:37:18 2016
From: pypy.commits at gmail.com (sbauman)
Date: Tue, 27 Sep 2016 20:37:18 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Add test case where a virtualized
 value class is forced
Message-ID: <57eb3aee.d8941c0a.4c1d9.7b8f@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87429:1994dc5dc367
Date: 2016-09-27 23:36 -0400
http://bitbucket.org/pypy/pypy/changeset/1994dc5dc367/

Log:	Add test case where a virtualized value class is forced

diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1022,6 +1022,7 @@
                 self.unbox = v
 
         const = Box(X(5))
+        array = [None, None]
         def f():
             # Prevent all retracing of side exits. Ensures that the unroll
             # optimizer will attempt to jump to either the preamble or loop.
@@ -1035,6 +1036,7 @@
                 driver.jit_merge_point(i=i, val=val)
                 # Logical & rather than comparison to confuse range analysis.
                 # Test only succeeds on the first 2 iterations
+                array[i % 2] = val
                 if i & -2 == 0:
                     val = X(i)
                 else:
@@ -1051,6 +1053,46 @@
             def __init__(self, v):
                 self.v = v
 
+        # A mutable cell to prevent the JIT from inferring that the non-virtual
+        # object is a constant.
+        class Box(object):
+            def __init__(self, v):
+                self.unbox = v
+
+        const = Box(X(5))
+        array = [None, None]
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            val   = X(0)
+            i     = 0
+            const.unbox = X(5)
+            while i < 17:
+                driver.can_enter_jit(i=i, val=val)
+                driver.jit_merge_point(i=i, val=val)
+                # Store into a constant array to force the virtual object created
+                # by the previous iteration.
+                array[i % 2] = val
+                # Logical & rather than comparison to confuse range analysis.
+                # Test only succeeds on the first 2 iterations
+                if i & -2 == 0:
+                    val = X(i)
+                else:
+                    val = const.unbox
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
+
+    def test_avoid_preamble_4(self):
+        driver = JitDriver(greens=[], reds=['i', 'val'])
+        class X(object):
+            _value_class_ = True
+            def __init__(self, v):
+                self.v = v
+
         const = X(5)
         def f():
             # Prevent all retracing of side exits. Ensures that the unroll

From pypy.commits at gmail.com  Wed Sep 28 04:42:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 28 Sep 2016 01:42:53 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: "import stackless" fails
Message-ID: <57eb828d.262ec20a.b3763.4b44@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5719:103c91845692
Date: 2016-09-28 10:42 +0200
http://bitbucket.org/pypy/extradoc/changeset/103c91845692/

Log:	"import stackless" fails

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -39,6 +39,9 @@
   'array' is typically an extension module; at least that's the case on
   Linux with default compilation settings).
 
+* 'import stackless' fails
+
+
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------
 

From pypy.commits at gmail.com  Wed Sep 28 04:56:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 28 Sep 2016 01:56:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: quick untested fix
Message-ID: <57eb85d3.516d1c0a.cdf89.e9a7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r87431:fcb7b33d25c5
Date: 2016-09-28 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/fcb7b33d25c5/

Log:	quick untested fix

diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -535,7 +535,7 @@
     _main_tasklet = coroutine.getcurrent()
     _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
-    tasklet._init.__func__(_main_tasklet, label='main')
+    tasklet._init(_main_tasklet, label='main')
     _squeue = deque()
     _scheduler_append(_main_tasklet)
 

From pypy.commits at gmail.com  Wed Sep 28 09:28:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 28 Sep 2016 06:28:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: progress on the lzma changes to
 allow max_length parameter
Message-ID: <57ebc59b.0c331c0a.68e0b.6d6f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87432:7e400cfdf59e
Date: 2016-09-28 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/7e400cfdf59e/

Log:	progress on the lzma changes to allow max_length parameter

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -445,6 +445,7 @@
         self.eof = False
         self.lzs = _new_lzma_stream()
         self._bufsiz = max(8192, io.DEFAULT_BUFFER_SIZE)
+        self.needs_input = True
 
         if format == FORMAT_AUTO:
             catch_lzma_error(m.lzma_auto_decoder, self.lzs, memlimit, decoder_flags)
@@ -473,9 +474,9 @@
         else:
             raise ValueError("invalid...")
 
-    def decompress(self, data):
+    def decompress(self, data, max_length=-1):
         """
-        decompress(data) -> bytes
+        decompress(data, max_length=-1) -> bytes
 
         Provide data to the decompressor object. Returns a chunk of
         decompressed data if possible, or b"" otherwise.
@@ -487,18 +488,71 @@
         with self.lock:
             if self.eof:
                 raise EOFError("Already...")
-            return self._decompress(data)
+            lzs = self.lzs
+            data = to_bytes(data)
+            buf = ffi.new('char[]', data)
+            buf_size = len(data)
+            if lzs.next_in:
+                # in this case there is data left that needs to be processed before the first
+                # argument can be processed
+                addr_input_buffer = int(ffi.cast('uintptr_t', self.input_buffer))
+                addr_next_in = int(ffi.cast('uintptr_t', lzs.next_in))
+                avail_now = (addr_input_buffer + self.input_buffer_size) - \
+                            (addr_next_in + lzs.avail_in)
+                avail_total = self.input_buffer_size - lzs.avail_in
+                if avail_total < len:
+                    # resize the buffer, it is too small!
+                    pass
+                elif avail_now < len:
+                    # move all data to the front
+                    ffi.memmove(self.input_buffer, lzs.next_in, lzs.avail_in)
+                ffi.memmove(lzs.next_in+lzs.avail_in, buf, buf_size)
+                lzn.avail_in += buf_size
+                used_input_buffer = True
+            else:
+                lzn.avail_in = buf_size
+                lzn.next_in = buf
+                used_input_buffer = False
 
-    def _decompress(self, data):
+            result = self._decompress(buf, buf_len, max_length)
+
+            if self.eof:
+                self.needs_input = False
+                if lzs.avail_in > 0:
+                    self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:]
+                    return result
+            elif lzs.avail_in == 0:
+                # completed successfully!
+                self.needs_input = True
+                lzs.next_in = None
+            else:
+                self.needs_input = False
+                if not used_input_buffer:
+                    # free buffer it is to small
+                    if self.input_buffer is ffi.NULL and \
+                       self.input_buffer_size < lzs.avail_in:
+                        m.free(self.input_buffer)
+                        self.input_buffer = None
+
+                    # allocate if necessary
+                    if self.input_buffer is None:
+                        self.input_buffer = m.malloc(lzs.avail_in)
+                        self.input_buffer_size = lzs.avail_in
+
+                    ffi.memmove(self.input_buffer, lzs.next_in, lzs.avail_in)
+                    lzs.next_in = self.input_buffer
+
+            return result
+
+    def _decompress(self, buf, buf_len, max_length):
         lzs = self.lzs
 
-        # we need in_ so that lzs.next_in doesn't get garbage collected until
-        # in_ goes out of scope
-        data = to_bytes(data)
-        lzs.next_in = in_ = ffi.new('char[]', data)
-        lzs.avail_in = len(data)
+        lzs.next_in = buf
+        lzs.avail_in = buf_len
 
         bufsiz = self._bufsiz
+        if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
+            bufsiz = max_length
 
         lzs.next_out = orig_out = m.malloc(bufsiz)
         if orig_out == ffi.NULL:
@@ -519,13 +573,13 @@
 
                 if ret == m.LZMA_STREAM_END:
                     self.eof = True
-                    if lzs.avail_in > 0:
-                        self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:]
                     break
                 elif lzs.avail_in == 0:
                     # it ate everything
                     break
                 elif lzs.avail_out == 0:
+                    if data_size == max_length:
+                        break
                     # ran out of space in the output buffer, let's grow it
                     bufsiz += (bufsiz >> 3) + 6
                     next_out = m.realloc(orig_out, bufsiz)
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -386,7 +386,7 @@
             self.running = False
             self.unused_data = ""
             self.needs_input = True
-            self.input_buffer = None
+            self.input_buffer = ""
             self.left_to_process = 0
 
             self._init_bz2decomp()
@@ -456,6 +456,7 @@
                     if self.left_to_process != 0:
                         end = len(data)
                         start = end - self.left_to_process
+                        assert start > 0
                         self.unused_data = data[start:]
                 res = out.make_result_string()
                 return self.space.newbytes(res)
@@ -476,7 +477,7 @@
         if data == '':
             return self.space.newbytes('')
         datalen = len(data)
-        if self.input_buffer:
+        if len(self.input_buffer) > 0:
             input_buffer_in_use = True
             result = self._decompress_buf(self.input_buffer, max_length)
         else:
@@ -485,12 +486,14 @@
             result = self._decompress_buf(data, max_length)
 
         if self.left_to_process == 0:
-            self.input_buffer = None
+            self.input_buffer = ""
             self.needs_input = True
         else:
             self.needs_input = False
             if not input_buffer_in_use:
-                self.input_buffer = data[datalen-self.left_to_process-1:]
+                start = datalen-self.left_to_process-1
+                assert start > 0
+                self.input_buffer = data[start:]
 
         return result
 

From pypy.commits at gmail.com  Wed Sep 28 09:29:04 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 28 Sep 2016 06:29:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: pass all lzma tests
Message-ID: <57ebc5a0.ca11c30a.8a70a.d339@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87433:34f0c5aa2e96
Date: 2016-09-28 15:28 +0200
http://bitbucket.org/pypy/pypy/changeset/34f0c5aa2e96/

Log:	pass all lzma tests

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -446,6 +446,8 @@
         self.lzs = _new_lzma_stream()
         self._bufsiz = max(8192, io.DEFAULT_BUFFER_SIZE)
         self.needs_input = True
+        self._input_buffer = ffi.NULL
+        self._input_buffer_size = 0
 
         if format == FORMAT_AUTO:
             catch_lzma_error(m.lzma_auto_decoder, self.lzs, memlimit, decoder_flags)
@@ -474,6 +476,64 @@
         else:
             raise ValueError("invalid...")
 
+    def pre_decompress_left_data(self, buf, buf_size):
+        # in this case there is data left that needs to be processed before the first
+        # argument can be processed
+
+        lzs = self.lzs
+
+        addr_input_buffer = int(ffi.cast('uintptr_t', self._input_buffer))
+        addr_next_in = int(ffi.cast('uintptr_t', lzs.next_in))
+        avail_now = (addr_input_buffer + self._input_buffer_size) - \
+                    (addr_next_in + lzs.avail_in)
+        avail_total = self._input_buffer_size - lzs.avail_in
+        if avail_total < buf_size:
+            # resize the buffer, it is too small!
+            offset = addr_next_in - addr_input_buffer
+            new_size = self._input_buffer_size + buf_size - avail_now
+            # there is no realloc?
+            tmp = ffi.cast("uint8_t*",m.malloc(new_size))
+            if tmp == ffi.NULL:
+                raise MemoryError
+            ffi.memmove(tmp, lzs.next_in, lzs.avail_in)
+            lzs.next_in = tmp
+            m.free(self._input_buffer)
+            self._input_buffer = tmp
+            self._input_buffer_size = new_size
+        elif avail_now < buf_size:
+            # the buffer is not too small, but we cannot append it!
+            # move all data to the front
+            ffi.memmove(self._input_buffer, lzs.next_in, lzs.avail_in)
+            lzs.next_in = self._input_buffer
+        ffi.memmove(lzs.next_in+lzs.avail_in, buf, buf_size)
+        lzs.avail_in += buf_size
+        return lzs.next_in, lzs.avail_in
+
+    def post_decompress_avail_data(self):
+        lzs = self.lzs
+        # free buffer it is to small
+        if self._input_buffer is not ffi.NULL and \
+           self._input_buffer_size < lzs.avail_in:
+            m.free(self._input_buffer)
+            self._input_buffer = ffi.NONE
+
+        # allocate if necessary
+        if self._input_buffer is ffi.NULL:
+            self._input_buffer = ffi.cast("uint8_t*",m.malloc(lzs.avail_in))
+            if self._input_buffer == ffi.NULL:
+                raise MemoryError
+            self._input_buffer_size = lzs.avail_in
+
+        ffi.memmove(self._input_buffer, lzs.next_in, lzs.avail_in)
+        lzs.next_in = self._input_buffer
+
+    def clear_input_buffer(self):
+        # clean the buffer
+        if self._input_buffer is not ffi.NULL:
+            m.free(self._input_buffer)
+            self._input_buffer = ffi.NULL
+            self._input_buffer_size = 0
+
     def decompress(self, data, max_length=-1):
         """
         decompress(data, max_length=-1) -> bytes
@@ -485,62 +545,41 @@
         reached raises an EOFError. Any data found after the end of the
         stream is ignored, and saved in the unused_data attribute.
         """
+        if not isinstance(max_length, int):
+            raise TypeError("max_length parameter object cannot be interpreted as an integer")
         with self.lock:
             if self.eof:
                 raise EOFError("Already...")
             lzs = self.lzs
             data = to_bytes(data)
-            buf = ffi.new('char[]', data)
+            buf = ffi.new('uint8_t[]', data)
             buf_size = len(data)
+
             if lzs.next_in:
-                # in this case there is data left that needs to be processed before the first
-                # argument can be processed
-                addr_input_buffer = int(ffi.cast('uintptr_t', self.input_buffer))
-                addr_next_in = int(ffi.cast('uintptr_t', lzs.next_in))
-                avail_now = (addr_input_buffer + self.input_buffer_size) - \
-                            (addr_next_in + lzs.avail_in)
-                avail_total = self.input_buffer_size - lzs.avail_in
-                if avail_total < len:
-                    # resize the buffer, it is too small!
-                    pass
-                elif avail_now < len:
-                    # move all data to the front
-                    ffi.memmove(self.input_buffer, lzs.next_in, lzs.avail_in)
-                ffi.memmove(lzs.next_in+lzs.avail_in, buf, buf_size)
-                lzn.avail_in += buf_size
-                used_input_buffer = True
+                buf, buf_size = self.pre_decompress_left_data(buf, buf_size)
+                used__input_buffer = True
             else:
-                lzn.avail_in = buf_size
-                lzn.next_in = buf
-                used_input_buffer = False
+                lzs.avail_in = buf_size
+                lzs.next_in = ffi.cast("uint8_t*",buf)
+                used__input_buffer = False
 
-            result = self._decompress(buf, buf_len, max_length)
+            # actual decompression
+            result = self._decompress(buf, buf_size, max_length)
 
             if self.eof:
                 self.needs_input = False
                 if lzs.avail_in > 0:
                     self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:]
-                    return result
+                self.clear_input_buffer()
             elif lzs.avail_in == 0:
                 # completed successfully!
                 self.needs_input = True
-                lzs.next_in = None
+                lzs.next_in = ffi.NULL
+                self.clear_input_buffer()
             else:
                 self.needs_input = False
-                if not used_input_buffer:
-                    # free buffer it is to small
-                    if self.input_buffer is ffi.NULL and \
-                       self.input_buffer_size < lzs.avail_in:
-                        m.free(self.input_buffer)
-                        self.input_buffer = None
-
-                    # allocate if necessary
-                    if self.input_buffer is None:
-                        self.input_buffer = m.malloc(lzs.avail_in)
-                        self.input_buffer_size = lzs.avail_in
-
-                    ffi.memmove(self.input_buffer, lzs.next_in, lzs.avail_in)
-                    lzs.next_in = self.input_buffer
+                if not used__input_buffer:
+                    self.post_decompress_avail_data()
 
             return result
 
@@ -693,9 +732,9 @@
 
         lzs = self.lzs
 
-        lzs.next_in = input_ = ffi.new('char[]', to_bytes(data))
+        lzs.next_in = input_ = ffi.new('uint8_t[]', to_bytes(data))
         lzs.avail_in = len(data)
-        outs = [ffi.new('char[]', BUFSIZ)]
+        outs = [ffi.new('uint8_t[]', BUFSIZ)]
         lzs.next_out, = outs
         lzs.avail_out = BUFSIZ
 
@@ -712,7 +751,7 @@
                 # ran out of space in the output buffer
                 #siz = (BUFSIZ << 1) + 6
                 siz = 512
-                outs.append(ffi.new('char[]', siz))
+                outs.append(ffi.new('uint8_t[]', siz))
                 lzs.next_out = outs[-1]
                 lzs.avail_out = siz
         last_out = outs.pop()
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -44,7 +44,10 @@
     return failures
 
 if __name__ == '__main__':
-    import py, os
+    # NOTE: it does not work to execute this file to rebuild the cffi backends
+    # for pypy3. This script is python 2! Thus you can specify
+    # exefile as an argument to still be able to run this script with a pypy2 vm
+    import py, os, argparse
     if '__pypy__' not in sys.builtin_module_names:
         print 'Call with a pypy interpreter'
         sys.exit(-1)
@@ -52,8 +55,15 @@
     class Options(object):
         pass
 
-    exename = py.path.local(sys.executable) 
+    parser = argparse.ArgumentParser(description='Build all cffi backends in lib_pypy')
+    parser.add_argument('--exefile', dest='exefile', default=sys.executable,
+                        help='instead of executing sys.executable' \
+                             ' you can specify an alternative pypy vm here')
+    args = parser.parse_args()
+
+    exename = py.path.local(args.exefile)
     basedir = exename
+
     while not basedir.join('include').exists():
         _basedir = basedir.dirpath()
         if _basedir == basedir:

From pypy.commits at gmail.com  Wed Sep 28 10:15:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 28 Sep 2016 07:15:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: issue in bz2, data not appended
Message-ID: <57ebd095.c4371c0a.2fb1.0b05@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87434:87e7a5ea092e
Date: 2016-09-28 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/87e7a5ea092e/

Log:	issue in bz2, data not appended

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -479,7 +479,9 @@
         datalen = len(data)
         if len(self.input_buffer) > 0:
             input_buffer_in_use = True
-            result = self._decompress_buf(self.input_buffer, max_length)
+            data = self.input_buffer + data
+            datalen = len(data)
+            result = self._decompress_buf(data, max_length)
         else:
             input_buffer_in_use = False
             self.left_to_process = datalen

From pypy.commits at gmail.com  Wed Sep 28 17:54:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 28 Sep 2016 14:54:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: tweaks, in-progress
Message-ID: <57ec3c07.9102c20a.baa3.919e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87435:4753b6674b9f
Date: 2016-09-28 23:39 +0200
http://bitbucket.org/pypy/pypy/changeset/4753b6674b9f/

Log:	tweaks, in-progress

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -40,13 +40,14 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
-    "faulthandler", "_jitlog",
+    "_jitlog",
 ])
 
 from rpython.jit.backend import detect_cpu
 try:
     if detect_cpu.autodetect().startswith('x86'):
         working_modules.add('_vmprof')
+        working_modules.add('faulthandler')
 except detect_cpu.ProcessorAutodetectError:
     pass
 
@@ -118,7 +119,8 @@
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
-    "_vmprof" : ["pypy.module._vmprof.interp_vmprof"],
+    "_vmprof"      : ["pypy.module._vmprof.interp_vmprof"],
+    "faulthandler" : ["pypy.module._vmprof.interp_vmprof"],
     "_lzma"     : ["pypy.module._lzma.interp_lzma"],
     }
 
diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py
--- a/pypy/module/faulthandler/dumper.py
+++ b/pypy/module/faulthandler/dumper.py
@@ -51,5 +51,4 @@
     but hopefully close enough).  In particular, this is written as
     RPython but shouldn't allocate anything.
     """
-    _dump(fd, "Current stack (most recent call first):\n")
     traceback.walk_traceback(PyCode, dump_code, fd, array_p, array_length)
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -9,7 +9,12 @@
 #include <sys/resource.h>
 #include <math.h>
 
-#include "rvmprof.h"
+#ifdef RPYTHON_LL2CTYPES
+#  include "../../../rpython/rlib/rvmprof/src/rvmprof.h"
+#else
+#  include "rvmprof.h"
+#endif
+#include "src/threadlocal.h"
 
 #define MAX_FRAME_DEPTH   100
 #define FRAME_DEPTH_N     RVMPROF_TRACEBACK_ESTIMATE_N(MAX_FRAME_DEPTH)
@@ -59,7 +64,7 @@
 RPY_EXTERN
 void pypy_faulthandler_write_int(int fd, long value)
 {
-    char buf[32];
+    char buf[48];
     sprintf(buf, "%ld", value);
     pypy_faulthandler_write(fd, buf);
 }
@@ -72,13 +77,51 @@
     pypy_faulthandler_cb_t fn;
     intptr_t array_p[FRAME_DEPTH_N], array_length;
 
-    /* XXX 'all_threads' ignored */
     fn = fatal_error.dump_traceback;
-    if (fn) {
+    if (!fn)
+        return;
+
+#ifndef RPYTHON_LL2CTYPES
+    if (all_threads && _RPython_ThreadLocals_AcquireTimeout(10000) == 0) {
+        /* This is known not to be perfectly safe against segfaults if we
+           don't hold the GIL ourselves.  Too bad.  I suspect that CPython
+           has issues there too.
+        */
+        struct pypy_threadlocal_s *my, *p;
+        int blankline = 0;
+        char buf[40];
+
+        my = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get();
+        p = _RPython_ThreadLocals_Head();
+        p = _RPython_ThreadLocals_Enum(p);
+        while (p != NULL) {
+            if (blankline)
+                pypy_faulthandler_write(fd, "\n");
+            blankline = 1;
+
+            pypy_faulthandler_write(fd, my == p ? "Current thread" : "Thread");
+            sprintf(buf, " 0x%lx", (unsigned long)p->thread_ident);
+            pypy_faulthandler_write(fd, buf);
+            pypy_faulthandler_write(fd, " (most recent call first):\n");
+
+            array_length = vmprof_get_traceback(p->vmprof_tl_stack,
+                                                my == p ? ucontext : NULL,
+                                                array_p, FRAME_DEPTH_N);
+            fn(fd, array_p, array_length);
+
+            p = _RPython_ThreadLocals_Enum(p);
+        }
+        _RPython_ThreadLocals_Release();
+    }
+    else {
+        pypy_faulthandler_write(fd, "Stack (most recent call first):\n");
         array_length = vmprof_get_traceback(NULL, ucontext,
                                             array_p, FRAME_DEPTH_N);
         fn(fd, array_p, array_length);
     }
+#else
+    pypy_faulthandler_write(fd, "(no traceback when untranslated)\n");
+#endif
 }
 
 static void
diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py
--- a/pypy/module/faulthandler/test/test_faulthander.py
+++ b/pypy/module/faulthandler/test/test_faulthander.py
@@ -1,8 +1,3 @@
-from pypy.module.faulthandler import interp_faulthandler
-
-class TestFaultHandler:
-    def test_fatal_error(self, space):
-        raises(RuntimeError, interp_faulthandler.fatal_error, space, "Message")
 
 class AppTestFaultHandler:
     spaceconfig = {
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -14,11 +14,19 @@
 
 static int check_valid(void);
 
-void _RPython_ThreadLocals_Acquire(void) {
-    while (!pypy_lock_test_and_set(&pypy_threadlocal_lock, 1)) {
+int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations) {
+    while (1) {
+        long old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1);
+        if (old_value == 0)
+            break;
         /* busy loop */
+        if (max_wait_iterations == 0)
+            return -1;
+        if (max_wait_iterations > 0)
+            --max_wait_iterations;
     }
     assert(check_valid());
+    return 0;
 }
 void _RPython_ThreadLocals_Release(void) {
     assert(check_valid());
@@ -59,11 +67,7 @@
 {
     /* assume that at most one pypy_threadlocal_s survived, the current one */
     struct pypy_threadlocal_s *cur;
-#ifdef USE___THREAD
-    cur = &pypy_threadlocal;
-#else
     cur = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get();
-#endif
     if (cur && cur->ready == 42) {
         cur->next = cur->prev = &linkedlist_head;
         linkedlist_head.next = linkedlist_head.prev = cur;
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -19,8 +19,10 @@
    current thread, and if not, calls the following helper. */
 RPY_EXTERN char *_RPython_ThreadLocals_Build(void);
 
-RPY_EXTERN void _RPython_ThreadLocals_Acquire(void);
+RPY_EXTERN int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations);
 RPY_EXTERN void _RPython_ThreadLocals_Release(void);
+#define _RPython_ThreadLocals_Acquire()  \
+    (void)_RPython_ThreadLocals_AcquireTimeout(-1)
 
 /* Must acquire/release the thread-local lock around a series of calls
    to the following function */
@@ -66,6 +68,8 @@
 
 #define RPY_THREADLOCALREF_GET(FIELD)   pypy_threadlocal.FIELD
 
+#define _RPy_ThreadLocals_Get()  (&pypy_threadlocal)
+
 
 /* ------------------------------------------------------------ */
 #else

From pypy.commits at gmail.com  Wed Sep 28 18:19:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 28 Sep 2016 15:19:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: fixes
Message-ID: <57ec4205.d7bb1c0a.91d23.2da2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87436:bfe89f6b4c68
Date: 2016-09-28 23:28 +0100
http://bitbucket.org/pypy/pypy/changeset/bfe89f6b4c68/

Log:	fixes

diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -12,6 +12,8 @@
 #ifdef RPYTHON_LL2CTYPES
 #  include "../../../rpython/rlib/rvmprof/src/rvmprof.h"
 #else
+#  include "common_header.h"
+#  include "structdef.h"
 #  include "rvmprof.h"
 #endif
 #include "src/threadlocal.h"
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -28,6 +28,9 @@
     assert(check_valid());
     return 0;
 }
+void _RPython_ThreadLocals_Acquire(void) {
+    _RPython_ThreadLocals_AcquireTimeout(-1);
+}
 void _RPython_ThreadLocals_Release(void) {
     assert(check_valid());
     pypy_lock_release(&pypy_threadlocal_lock);
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -19,10 +19,9 @@
    current thread, and if not, calls the following helper. */
 RPY_EXTERN char *_RPython_ThreadLocals_Build(void);
 
+RPY_EXTERN void _RPython_ThreadLocals_Acquire(void);
+RPY_EXTERN void _RPython_ThreadLocals_Release(void);
 RPY_EXTERN int _RPython_ThreadLocals_AcquireTimeout(int max_wait_iterations);
-RPY_EXTERN void _RPython_ThreadLocals_Release(void);
-#define _RPython_ThreadLocals_Acquire()  \
-    (void)_RPython_ThreadLocals_AcquireTimeout(-1)
 
 /* Must acquire/release the thread-local lock around a series of calls
    to the following function */

From pypy.commits at gmail.com  Thu Sep 29 04:13:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 29 Sep 2016 01:13:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: De-duplicate traceback
 entries after a LOC_JITTED
Message-ID: <57eccd43.a3adc20a.a0ebe.3d21@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87437:cabd383ef26c
Date: 2016-09-29 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/cabd383ef26c/

Log:	De-duplicate traceback entries after a LOC_JITTED

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -18,6 +18,7 @@
 VMPROF_JITTED_TAG = 3
 VMPROF_JITTING_TAG = 4
 VMPROF_GC_TAG = 5
+VMPROF_ASSEMBLER_TAG = 6
 
 class VMProfError(Exception):
     def __init__(self, msg):
diff --git a/rpython/rlib/rvmprof/traceback.py b/rpython/rlib/rvmprof/traceback.py
--- a/rpython/rlib/rvmprof/traceback.py
+++ b/rpython/rlib/rvmprof/traceback.py
@@ -47,19 +47,40 @@
     'code_obj' may be None if it can't be determined.  'loc' is one
     of the LOC_xxx constants.
     """
+    dedup = 0
     i = 0
     while i < array_length - 1:
         tag = array_p[i]
         tagged_value = array_p[i + 1]
+
         if tag == rvmprof.VMPROF_CODE_TAG:
+            # VMPROF_CODE_TAG is from an interpreted frame.
+            # If the previously-printed frame was a LOC_JITTED, then it
+            # should be a duplicate of the very same frame here.  In
+            # that case, after checking that it is indeed the same frame
+            # for sanity, we don't print it again.
             loc = LOC_INTERPRETED
-            _traceback_one(CodeClass, callback, arg, tagged_value, loc)
+            if tagged_value != dedup or tagged_value == 0:
+                _traceback_one(CodeClass, callback, arg, tagged_value, loc)
+            dedup = 0
+
         elif tag == rvmprof.VMPROF_JITTED_TAG:
             if i + 2 >= array_length:  # skip last entry, can't determine if
                 break                  # it's LOC_JITTED_INLINED or LOC_JITTED
+
+            # A bunch of VMPROF_JITTED_TAG entries consecutively in the
+            # stack trace correspond to one piece of machine code with
+            # inlined frames.  There is a VMPROF_ASSEMBLER_TAG before,
+            # so we are sure that there is at least one non-JITTED_TAG
+            # between one such bunch and the next one.  De-duplicate
+            # like we do for LOC_INTERPRETED, and record in 'dedup' the
+            # code id if we're printing LOC_JITTED (i.e. the last frame
+            # of this bunch).
             if array_p[i + 2] == rvmprof.VMPROF_JITTED_TAG:
                 loc = LOC_JITTED_INLINED
             else:
                 loc = LOC_JITTED
-            _traceback_one(CodeClass, callback, arg, tagged_value, loc)
+            if tagged_value != dedup or tagged_value == 0:
+                _traceback_one(CodeClass, callback, arg, tagged_value, loc)
+            dedup = tagged_value if loc == LOC_JITTED else 0
         i += 2

From pypy.commits at gmail.com  Thu Sep 29 04:27:34 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 29 Sep 2016 01:27:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: both bz2 and lzma seem to be in
 shape for py3.5
Message-ID: <57ecd076.05d71c0a.7cc31.5ca7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87439:2f6b9156a6be
Date: 2016-09-29 10:19 +0200
http://bitbucket.org/pypy/pypy/changeset/2f6b9156a6be/

Log:	both bz2 and lzma seem to be in shape for py3.5

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -203,7 +203,7 @@
         self.left = 0
 
     def get_data_size(self):
-        return 0
+        return self.current_size - rffi.getintfield(self.bzs, 'c_avail_out')
 
     def _allocate_chunk(self, size):
         self.raw_buf, self.gc_buf, self.case_num = rffi.alloc_buffer(size)
@@ -430,14 +430,13 @@
             # setup the input and the size it can consume
             self.bzs.c_next_in = in_buf
             rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
-            self.left_to_process -= in_bufsize
 
             with OutBuffer(self.bzs, max_length=max_length) as out:
                 while True:
                     bzreturn = BZ2_bzDecompress(self.bzs)
                     # add up the size that has not been processed
                     avail_in = rffi.getintfield(self.bzs, 'c_avail_in')
-                    self.left_to_process += avail_in
+                    self.left_to_process = avail_in
                     if bzreturn == BZ_STREAM_END:
                         self.running = False
                         break
@@ -474,8 +473,6 @@
         if not self.running:
             raise oefmt(self.space.w_EOFError,
                         "end of stream was already found")
-        if data == '':
-            return self.space.newbytes('')
         datalen = len(data)
         if len(self.input_buffer) > 0:
             input_buffer_in_use = True
@@ -484,7 +481,6 @@
             result = self._decompress_buf(data, max_length)
         else:
             input_buffer_in_use = False
-            self.left_to_process = datalen
             result = self._decompress_buf(data, max_length)
 
         if self.left_to_process == 0:
@@ -493,7 +489,7 @@
         else:
             self.needs_input = False
             if not input_buffer_in_use:
-                start = datalen-self.left_to_process-1
+                start = datalen-self.left_to_process
                 assert start > 0
                 self.input_buffer = data[start:]
 
diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -39,15 +39,6 @@
     mod.OLD_SMALLCHUNK = interp_bz2.INITIAL_BUFFER_SIZE
     interp_bz2.INITIAL_BUFFER_SIZE = 32
 
-    test_size = 0
-    mod.BIG_TEXT = bytearray(128*1024)
-    for fname in glob.glob(os.path.join(os.path.dirname(__file__), '*.py')):
-        with open(fname, 'rb') as fh:
-            test_size += fh.readinto(memoryview(BIG_TEXT)[test_size:])
-        if test_size > 128*1024:
-            break
-    mod.BIG_DATA = bz2.compress(BIG_TEXT, compresslevel=1)
-
 def teardown_module(mod):
     interp_bz2.INITIAL_BUFFER_SIZE = mod.OLD_SMALLCHUNK
 
@@ -133,8 +124,6 @@
     def setup_class(cls):
         cls.w_TEXT = cls.space.newbytes(TEXT)
         cls.w_DATA = cls.space.newbytes(DATA)
-        cls.w_BIG_DATA = cls.space.newbytes(BIG_DATA)
-        cls.w_BIG_TEXT = cls.space.newbytes(BIG_TEXT)
         cls.w_BUGGY_DATA = cls.space.newbytes(BUGGY_DATA)
 
         cls.space.appexec([], """(): import warnings""")  # Work around a recursion limit
@@ -219,24 +208,15 @@
         bz2d = BZ2Decompressor()
         decomp= []
 
-        length = len(self.BIG_DATA)
-        decomp.append(bz2d.decompress(self.BIG_DATA[:length-64]), max_length=100)
-        assert bz2d.needs_input == False
+        length = len(self.DATA)
+        decomp.append(bz2d.decompress(self.DATA, max_length=100))
         assert len(decomp[-1]) == 100
 
-        decomp.append(bz2d.decompress(b"", max_length=50))
-        assert bz2d.needs_input == False
-        assert len(decomp[-1]) == 50
-
-        decomp.append(bz2d.decompress(self.BIG_DATA[length-64:], max_length=50))
-        assert bz2d.needs_input == False
-        assert len(decomp[-1]) == 50
-
         while not bz2d.eof:
             decomp.append(bz2d.decompress(b"", max_length=50))
             assert len(decomp[-1]) <= 50
 
-        assert ''.join(decomp) == self.BIG_TEXT
+        assert b''.join(decomp) == self.TEXT
 
 
 class AppTestBZ2ModuleFunctions(CheckAllocation):
diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py
--- a/rpython/tool/runsubprocess.py
+++ b/rpython/tool/runsubprocess.py
@@ -8,10 +8,15 @@
 import os
 from subprocess import PIPE, Popen
 
+PY3  = sys.version_info[0] >= 3
+
 def run_subprocess(executable, args, env=None, cwd=None):
     if isinstance(args, list):
-        args = [a.encode('latin1') if isinstance(a, unicode) else a
-                for a in args]
+        if PY3:
+            args = [a for a in args]
+        else:
+            args = [a.encode('latin1') if isinstance(a, unicode) else a
+                    for a in args]
     return _run(executable, args, env, cwd)
 
 shell_default = False
@@ -83,7 +88,7 @@
 
     def _run(*args):
         try:
-            _child.stdin.write('%r\n' % (args,))
+            _child.stdin.write(b'%r\n' % (args,))
         except (OSError, IOError):
             # lost the child.  Try again...
             spawn_subprocess()

From pypy.commits at gmail.com  Thu Sep 29 05:01:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 29 Sep 2016 02:01:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge bz2 and lzma changes
Message-ID: <57ecd84e.02c11c0a.5b222.e87c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r87440:43ca4b14e86e
Date: 2016-09-29 11:00 +0200
http://bitbucket.org/pypy/pypy/changeset/43ca4b14e86e/

Log:	merge bz2 and lzma changes

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -445,6 +445,9 @@
         self.eof = False
         self.lzs = _new_lzma_stream()
         self._bufsiz = max(8192, io.DEFAULT_BUFFER_SIZE)
+        self.needs_input = True
+        self._input_buffer = ffi.NULL
+        self._input_buffer_size = 0
 
         if format == FORMAT_AUTO:
             catch_lzma_error(m.lzma_auto_decoder, self.lzs, memlimit, decoder_flags)
@@ -473,9 +476,67 @@
         else:
             raise ValueError("invalid...")
 
-    def decompress(self, data):
+    def pre_decompress_left_data(self, buf, buf_size):
+        # in this case there is data left that needs to be processed before the first
+        # argument can be processed
+
+        lzs = self.lzs
+
+        addr_input_buffer = int(ffi.cast('uintptr_t', self._input_buffer))
+        addr_next_in = int(ffi.cast('uintptr_t', lzs.next_in))
+        avail_now = (addr_input_buffer + self._input_buffer_size) - \
+                    (addr_next_in + lzs.avail_in)
+        avail_total = self._input_buffer_size - lzs.avail_in
+        if avail_total < buf_size:
+            # resize the buffer, it is too small!
+            offset = addr_next_in - addr_input_buffer
+            new_size = self._input_buffer_size + buf_size - avail_now
+            # there is no realloc?
+            tmp = ffi.cast("uint8_t*",m.malloc(new_size))
+            if tmp == ffi.NULL:
+                raise MemoryError
+            ffi.memmove(tmp, lzs.next_in, lzs.avail_in)
+            lzs.next_in = tmp
+            m.free(self._input_buffer)
+            self._input_buffer = tmp
+            self._input_buffer_size = new_size
+        elif avail_now < buf_size:
+            # the buffer is not too small, but we cannot append it!
+            # move all data to the front
+            ffi.memmove(self._input_buffer, lzs.next_in, lzs.avail_in)
+            lzs.next_in = self._input_buffer
+        ffi.memmove(lzs.next_in+lzs.avail_in, buf, buf_size)
+        lzs.avail_in += buf_size
+        return lzs.next_in, lzs.avail_in
+
+    def post_decompress_avail_data(self):
+        lzs = self.lzs
+        # free buffer it is to small
+        if self._input_buffer is not ffi.NULL and \
+           self._input_buffer_size < lzs.avail_in:
+            m.free(self._input_buffer)
+            self._input_buffer = ffi.NONE
+
+        # allocate if necessary
+        if self._input_buffer is ffi.NULL:
+            self._input_buffer = ffi.cast("uint8_t*",m.malloc(lzs.avail_in))
+            if self._input_buffer == ffi.NULL:
+                raise MemoryError
+            self._input_buffer_size = lzs.avail_in
+
+        ffi.memmove(self._input_buffer, lzs.next_in, lzs.avail_in)
+        lzs.next_in = self._input_buffer
+
+    def clear_input_buffer(self):
+        # clean the buffer
+        if self._input_buffer is not ffi.NULL:
+            m.free(self._input_buffer)
+            self._input_buffer = ffi.NULL
+            self._input_buffer_size = 0
+
+    def decompress(self, data, max_length=-1):
         """
-        decompress(data) -> bytes
+        decompress(data, max_length=-1) -> bytes
 
         Provide data to the decompressor object. Returns a chunk of
         decompressed data if possible, or b"" otherwise.
@@ -484,21 +545,53 @@
         reached raises an EOFError. Any data found after the end of the
         stream is ignored, and saved in the unused_data attribute.
         """
+        if not isinstance(max_length, int):
+            raise TypeError("max_length parameter object cannot be interpreted as an integer")
         with self.lock:
             if self.eof:
                 raise EOFError("Already...")
-            return self._decompress(data)
+            lzs = self.lzs
+            data = to_bytes(data)
+            buf = ffi.new('uint8_t[]', data)
+            buf_size = len(data)
 
-    def _decompress(self, data):
+            if lzs.next_in:
+                buf, buf_size = self.pre_decompress_left_data(buf, buf_size)
+                used__input_buffer = True
+            else:
+                lzs.avail_in = buf_size
+                lzs.next_in = ffi.cast("uint8_t*",buf)
+                used__input_buffer = False
+
+            # actual decompression
+            result = self._decompress(buf, buf_size, max_length)
+
+            if self.eof:
+                self.needs_input = False
+                if lzs.avail_in > 0:
+                    self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:]
+                self.clear_input_buffer()
+            elif lzs.avail_in == 0:
+                # completed successfully!
+                self.needs_input = True
+                lzs.next_in = ffi.NULL
+                self.clear_input_buffer()
+            else:
+                self.needs_input = False
+                if not used__input_buffer:
+                    self.post_decompress_avail_data()
+
+            return result
+
+    def _decompress(self, buf, buf_len, max_length):
         lzs = self.lzs
 
-        # we need in_ so that lzs.next_in doesn't get garbage collected until
-        # in_ goes out of scope
-        data = to_bytes(data)
-        lzs.next_in = in_ = ffi.new('char[]', data)
-        lzs.avail_in = len(data)
+        lzs.next_in = buf
+        lzs.avail_in = buf_len
 
         bufsiz = self._bufsiz
+        if not (max_length < 0 or max_length > io.DEFAULT_BUFFER_SIZE):
+            bufsiz = max_length
 
         lzs.next_out = orig_out = m.malloc(bufsiz)
         if orig_out == ffi.NULL:
@@ -519,13 +612,13 @@
 
                 if ret == m.LZMA_STREAM_END:
                     self.eof = True
-                    if lzs.avail_in > 0:
-                        self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:]
                     break
                 elif lzs.avail_in == 0:
                     # it ate everything
                     break
                 elif lzs.avail_out == 0:
+                    if data_size == max_length:
+                        break
                     # ran out of space in the output buffer, let's grow it
                     bufsiz += (bufsiz >> 3) + 6
                     next_out = m.realloc(orig_out, bufsiz)
@@ -639,9 +732,9 @@
 
         lzs = self.lzs
 
-        lzs.next_in = input_ = ffi.new('char[]', to_bytes(data))
+        lzs.next_in = input_ = ffi.new('uint8_t[]', to_bytes(data))
         lzs.avail_in = len(data)
-        outs = [ffi.new('char[]', BUFSIZ)]
+        outs = [ffi.new('uint8_t[]', BUFSIZ)]
         lzs.next_out, = outs
         lzs.avail_out = BUFSIZ
 
@@ -658,7 +751,7 @@
                 # ran out of space in the output buffer
                 #siz = (BUFSIZ << 1) + 6
                 siz = 512
-                outs.append(ffi.new('char[]', siz))
+                outs.append(ffi.new('uint8_t[]', siz))
                 lzs.next_out = outs[-1]
                 lzs.avail_out = siz
         last_out = outs.pop()
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -96,9 +96,11 @@
 BZ_SEQUENCE_ERROR = cConfig.BZ_SEQUENCE_ERROR
 
 if BUFSIZ < 8192:
-    SMALLCHUNK = 8192
+    INITIAL_BUFFER_SIZE = 8192
 else:
-    SMALLCHUNK = BUFSIZ
+    INITIAL_BUFFER_SIZE = 8192
+
+UINT_MAX = 2**32-1
 
 if rffi.sizeof(rffi.INT) > 4:
     BIGCHUNK = 512 * 32
@@ -187,12 +189,21 @@
     encapsulate the logic of setting up the fields of 'bzs' and
     allocating raw memory as needed.
     """
-    def __init__(self, bzs, initial_size=SMALLCHUNK):
+    def __init__(self, bzs, initial_size=INITIAL_BUFFER_SIZE, max_length=-1):
         # when the constructor is called, allocate a piece of memory
         # of length 'piece_size' and make bzs ready to dump there.
         self.temp = []
         self.bzs = bzs
-        self._allocate_chunk(initial_size)
+        self.max_length = max_length
+        if max_length < 0 or max_length >= initial_size:
+            size = initial_size
+        else:
+            size = max_length
+        self._allocate_chunk(size)
+        self.left = 0
+
+    def get_data_size(self):
+        return self.current_size - rffi.getintfield(self.bzs, 'c_avail_out')
 
     def _allocate_chunk(self, size):
         self.raw_buf, self.gc_buf, self.case_num = rffi.alloc_buffer(size)
@@ -214,7 +225,10 @@
     def prepare_next_chunk(self):
         size = self.current_size
         self.temp.append(self._get_chunk(size))
-        self._allocate_chunk(_new_buffer_size(size))
+        newsize = size
+        if self.max_length == -1:
+            newsize = _new_buffer_size(size)
+        self._allocate_chunk(newsize)
 
     def make_result_string(self):
         count_unoccupied = rffi.getintfield(self.bzs, 'c_avail_out')
@@ -357,7 +371,6 @@
     W_BZ2Decompressor.__init__(x, space)
     return space.wrap(x)
 
-
 class W_BZ2Decompressor(W_Root):
     """BZ2Decompressor() -> decompressor object
 
@@ -372,6 +385,9 @@
         try:
             self.running = False
             self.unused_data = ""
+            self.needs_input = True
+            self.input_buffer = ""
+            self.left_to_process = 0
 
             self._init_bz2decomp()
         except:
@@ -397,15 +413,56 @@
     def descr_getstate(self):
         raise oefmt(self.space.w_TypeError, "cannot serialize '%T' object", self)
 
+    def needs_input_w(self, space):
+        """ True if more input is needed before more decompressed
+            data can be produced. """
+        return space.wrap(self.needs_input)
+
     def eof_w(self, space):
         if self.running:
             return space.w_False
         else:
             return space.w_True
 
-    @unwrap_spec(data='bufferstr')
-    def decompress(self, data):
-        """decompress(data) -> string
+    def _decompress_buf(self, data, max_length):
+        in_bufsize = len(data)
+        with rffi.scoped_nonmovingbuffer(data) as in_buf:
+            # setup the input and the size it can consume
+            self.bzs.c_next_in = in_buf
+            rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
+
+            with OutBuffer(self.bzs, max_length=max_length) as out:
+                while True:
+                    bzreturn = BZ2_bzDecompress(self.bzs)
+                    # add up the size that has not been processed
+                    avail_in = rffi.getintfield(self.bzs, 'c_avail_in')
+                    self.left_to_process = avail_in
+                    if bzreturn == BZ_STREAM_END:
+                        self.running = False
+                        break
+                    if bzreturn != BZ_OK:
+                        _catch_bz2_error(self.space, bzreturn)
+
+                    if self.left_to_process == 0:
+                        break
+                    elif rffi.getintfield(self.bzs, 'c_avail_out') == 0:
+                        if out.get_data_size() == max_length:
+                            break
+                        out.prepare_next_chunk()
+
+                if not self.running:
+                    self.needs_input = False
+                    if self.left_to_process != 0:
+                        end = len(data)
+                        start = end - self.left_to_process
+                        assert start > 0
+                        self.unused_data = data[start:]
+                res = out.make_result_string()
+                return self.space.newbytes(res)
+
+    @unwrap_spec(data='bufferstr', max_length=int)
+    def decompress(self, data, max_length=-1):
+        """decompress(data, max_length=-1) -> bytes
 
         Provide more data to the decompressor object. It will return chunks
         of decompressed data whenever possible. If you try to decompress data
@@ -416,37 +473,30 @@
         if not self.running:
             raise oefmt(self.space.w_EOFError,
                         "end of stream was already found")
-        if data == '':
-            return self.space.newbytes('')
+        datalen = len(data)
+        if len(self.input_buffer) > 0:
+            input_buffer_in_use = True
+            data = self.input_buffer + data
+            datalen = len(data)
+            result = self._decompress_buf(data, max_length)
+        else:
+            input_buffer_in_use = False
+            result = self._decompress_buf(data, max_length)
 
-        in_bufsize = len(data)
+        if self.left_to_process == 0:
+            self.input_buffer = ""
+            self.needs_input = True
+        else:
+            self.needs_input = False
+            if not input_buffer_in_use:
+                start = datalen-self.left_to_process
+                assert start > 0
+                self.input_buffer = data[start:]
 
-        with rffi.scoped_nonmovingbuffer(data) as in_buf:
-            self.bzs.c_next_in = in_buf
-            rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
+        return result
 
-            with OutBuffer(self.bzs) as out:
-                while True:
-                    bzerror = BZ2_bzDecompress(self.bzs)
-                    if bzerror == BZ_STREAM_END:
-                        if rffi.getintfield(self.bzs, 'c_avail_in') != 0:
-                            unused = [self.bzs.c_next_in[i]
-                                      for i in range(
-                                          rffi.getintfield(self.bzs,
-                                                           'c_avail_in'))]
-                            self.unused_data = "".join(unused)
-                        self.running = False
-                        break
-                    if bzerror != BZ_OK:
-                        _catch_bz2_error(self.space, bzerror)
 
-                    if rffi.getintfield(self.bzs, 'c_avail_in') == 0:
-                        break
-                    elif rffi.getintfield(self.bzs, 'c_avail_out') == 0:
-                        out.prepare_next_chunk()
 
-                res = out.make_result_string()
-                return self.space.newbytes(res)
 
 
 W_BZ2Decompressor.typedef = TypeDef("_bz2.BZ2Decompressor",
@@ -456,5 +506,6 @@
     unused_data = interp_attrproperty_bytes("unused_data", W_BZ2Decompressor),
     eof = GetSetProperty(W_BZ2Decompressor.eof_w),
     decompress = interp2app(W_BZ2Decompressor.decompress),
+    needs_input = GetSetProperty(W_BZ2Decompressor.needs_input_w),
 )
 W_BZ2Decompressor.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -1,6 +1,8 @@
 import os
 
 import py
+import glob
+import bz2
 
 from pypy.module.bz2.test.support import CheckAllocation
 from pypy.module.bz2 import interp_bz2
@@ -34,11 +36,11 @@
     mod.decompress = decompress
     #
     # For tests, patch the value of SMALLCHUNK
-    mod.OLD_SMALLCHUNK = interp_bz2.SMALLCHUNK
-    interp_bz2.SMALLCHUNK = 32
+    mod.OLD_SMALLCHUNK = interp_bz2.INITIAL_BUFFER_SIZE
+    interp_bz2.INITIAL_BUFFER_SIZE = 32
 
 def teardown_module(mod):
-    interp_bz2.SMALLCHUNK = mod.OLD_SMALLCHUNK
+    interp_bz2.INITIAL_BUFFER_SIZE = mod.OLD_SMALLCHUNK
 
 class AppTestBZ2Compressor(CheckAllocation):
     spaceconfig = dict(usemodules=('bz2', 'time', 'struct'))
@@ -200,6 +202,22 @@
         exc = raises(TypeError, pickle.dumps, BZ2Decompressor())
         assert exc.value.args[0] == "cannot serialize '_bz2.BZ2Decompressor' object"
 
+    def test_decompress_max_length(self):
+        from bz2 import BZ2Decompressor
+
+        bz2d = BZ2Decompressor()
+        decomp= []
+
+        length = len(self.DATA)
+        decomp.append(bz2d.decompress(self.DATA, max_length=100))
+        assert len(decomp[-1]) == 100
+
+        while not bz2d.eof:
+            decomp.append(bz2d.decompress(b"", max_length=50))
+            assert len(decomp[-1]) <= 50
+
+        assert b''.join(decomp) == self.TEXT
+
 
 class AppTestBZ2ModuleFunctions(CheckAllocation):
     spaceconfig = dict(usemodules=('bz2', 'time'))
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -256,7 +256,6 @@
         self.create_temp_file()
 
         bz2f = BZ2File(self.temppath)
-        raises(TypeError, bz2f.read, None)
         text_read = bz2f.read()
         assert text_read == self.TEXT
         bz2f.close()
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -44,7 +44,10 @@
     return failures
 
 if __name__ == '__main__':
-    import py, os
+    # NOTE: it does not work to execute this file to rebuild the cffi backends
+    # for pypy3. This script is python 2! Thus you can specify
+    # exefile as an argument to still be able to run this script with a pypy2 vm
+    import py, os, argparse
     if '__pypy__' not in sys.builtin_module_names:
         print 'Call with a pypy interpreter'
         sys.exit(-1)
@@ -52,8 +55,15 @@
     class Options(object):
         pass
 
-    exename = py.path.local(sys.executable) 
+    parser = argparse.ArgumentParser(description='Build all cffi backends in lib_pypy')
+    parser.add_argument('--exefile', dest='exefile', default=sys.executable,
+                        help='instead of executing sys.executable' \
+                             ' you can specify an alternative pypy vm here')
+    args = parser.parse_args()
+
+    exename = py.path.local(args.exefile)
     basedir = exename
+
     while not basedir.join('include').exists():
         _basedir = basedir.dirpath()
         if _basedir == basedir:
diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py
--- a/rpython/tool/runsubprocess.py
+++ b/rpython/tool/runsubprocess.py
@@ -8,10 +8,15 @@
 import os
 from subprocess import PIPE, Popen
 
+PY3  = sys.version_info[0] >= 3
+
 def run_subprocess(executable, args, env=None, cwd=None):
     if isinstance(args, list):
-        args = [a.encode('latin1') if isinstance(a, unicode) else a
-                for a in args]
+        if PY3:
+            args = [a for a in args]
+        else:
+            args = [a.encode('latin1') if isinstance(a, unicode) else a
+                    for a in args]
     return _run(executable, args, env, cwd)
 
 shell_default = False
@@ -83,7 +88,7 @@
 
     def _run(*args):
         try:
-            _child.stdin.write('%r\n' % (args,))
+            _child.stdin.write(b'%r\n' % (args,))
         except (OSError, IOError):
             # lost the child.  Try again...
             spawn_subprocess()

From pypy.commits at gmail.com  Thu Sep 29 06:12:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 29 Sep 2016 03:12:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: simplifications for bz2
 decompress
Message-ID: <57ece908.e2cdc20a.2c914.72a1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87442:17c2b19b643b
Date: 2016-09-29 12:07 +0200
http://bitbucket.org/pypy/pypy/changeset/17c2b19b643b/

Log:	simplifications for bz2 decompress

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -425,18 +425,22 @@
             return space.w_True
 
     def _decompress_buf(self, data, max_length):
-        in_bufsize = len(data)
+        total_in = len(data)
+        in_bufsize = min(total_in, UINT_MAX)
+        total_in -= in_bufsize
         with rffi.scoped_nonmovingbuffer(data) as in_buf:
             # setup the input and the size it can consume
             self.bzs.c_next_in = in_buf
             rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
+            self.left_to_process = in_bufsize
 
             with OutBuffer(self.bzs, max_length=max_length) as out:
                 while True:
                     bzreturn = BZ2_bzDecompress(self.bzs)
                     # add up the size that has not been processed
                     avail_in = rffi.getintfield(self.bzs, 'c_avail_in')
-                    self.left_to_process = avail_in
+                    total_in += avail_in
+                    self.left_to_process = total_in
                     if bzreturn == BZ_STREAM_END:
                         self.running = False
                         break
@@ -450,13 +454,6 @@
                             break
                         out.prepare_next_chunk()
 
-                if not self.running:
-                    self.needs_input = False
-                    if self.left_to_process != 0:
-                        end = len(data)
-                        start = end - self.left_to_process
-                        assert start > 0
-                        self.unused_data = data[start:]
                 res = out.make_result_string()
                 return self.space.newbytes(res)
 
@@ -475,24 +472,28 @@
                         "end of stream was already found")
         datalen = len(data)
         if len(self.input_buffer) > 0:
-            input_buffer_in_use = True
             data = self.input_buffer + data
             datalen = len(data)
-            result = self._decompress_buf(data, max_length)
-        else:
-            input_buffer_in_use = False
-            result = self._decompress_buf(data, max_length)
+            self.input_buffer = ""
 
-        if self.left_to_process == 0:
+        result = self._decompress_buf(data, max_length)
+
+        if not self.running: # eq. with eof == Ture
+            self.needs_input = False
+            if self.left_to_process != 0:
+                start = datalen - self.left_to_process
+                assert start > 0
+                self.unused_data = data[start:]
+                self.left_to_process = 0
+        elif self.left_to_process == 0:
             self.input_buffer = ""
             self.needs_input = True
         else:
             self.needs_input = False
-            if not input_buffer_in_use:
+            if self.left_to_process > 0:
                 start = datalen-self.left_to_process
-                assert start > 0
+                assert start >= 0
                 self.input_buffer = data[start:]
-
         return result
 
 
diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -213,8 +213,8 @@
         assert len(decomp[-1]) == 100
 
         while not bz2d.eof:
-            decomp.append(bz2d.decompress(b"", max_length=50))
-            assert len(decomp[-1]) <= 50
+            decomp.append(bz2d.decompress(b"", max_length=200))
+            assert len(decomp[-1]) <= 200
 
         assert b''.join(decomp) == self.TEXT
 

From pypy.commits at gmail.com  Thu Sep 29 06:14:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 29 Sep 2016 03:14:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: translation fixes
Message-ID: <57ece969.a22fc20a.f806c.790f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87443:cc3c09d40bf8
Date: 2016-09-29 12:13 +0200
http://bitbucket.org/pypy/pypy/changeset/cc3c09d40bf8/

Log:	translation fixes

diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -139,7 +139,7 @@
 }
 
 #ifdef PYPY_FAULTHANDLER_LATER
-#include "src/thead.h"
+#include "src/thread.h"
 static struct {
     int fd;
     long long microseconds;
@@ -199,7 +199,7 @@
 
         if (thread_later.exit)
             _exit(1);
-    } while (thread.repeat);
+    } while (thread_later.repeat);
 
     /* The only way out */
     RPyThreadReleaseLock(&thread_later.running);
@@ -231,7 +231,7 @@
 {
 #ifdef PYPY_FAULTHANDLER_LATER
     /* Notify cancellation */
-    RRyThreadReleaseLock(&thread_later.cancel_event);
+    RPyThreadReleaseLock(&thread_later.cancel_event);
 
     /* Wait for thread to join (or does nothing if no thread is running) */
     RPyThreadAcquireLock(&thread_later.running, 1);
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -77,6 +77,7 @@
         keepalive_until_here(w_file)
 
     def dump_traceback_later(self, timeout, repeat, w_file, exit):
+        space = self.space
         timeout *= 1e6
         try:
             microseconds = ovfcheck_float_to_longlong(timeout)
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -49,14 +49,14 @@
 
 #define OP_THREADLOCALREF_ADDR(r)               \
     do {                                        \
-        r = (char *)&pypy_threadlocal;          \
+        r = (void *)&pypy_threadlocal;          \
         if (pypy_threadlocal.ready != 42)       \
             r = _RPython_ThreadLocals_Build();  \
     } while (0)
 
 #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r)   \
     do {                                        \
-        r = (char *)&pypy_threadlocal;          \
+        r = (void *)&pypy_threadlocal;          \
         if (pypy_threadlocal.ready != 42)       \
             r = NULL;                           \
     } while (0)
@@ -92,14 +92,14 @@
 
 #define OP_THREADLOCALREF_ADDR(r)               \
     do {                                        \
-        r = (char *)_RPy_ThreadLocals_Get();    \
+        r = (void *)_RPy_ThreadLocals_Get();    \
         if (!r)                                 \
             r = _RPython_ThreadLocals_Build();  \
     } while (0)
 
 #define _OP_THREADLOCALREF_ADDR_SIGHANDLER(r)   \
     do {                                        \
-        r = (char *)_RPy_ThreadLocals_Get();    \
+        r = (void *)_RPy_ThreadLocals_Get();    \
     } while (0)
 
 #define RPY_THREADLOCALREF_ENSURE()             \

From pypy.commits at gmail.com  Thu Sep 29 10:19:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 29 Sep 2016 07:19:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: count diff too often
Message-ID: <57ed230a.948e1c0a.686bf.8365@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r87445:c1062888fa88
Date: 2016-09-29 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/c1062888fa88/

Log:	count diff too often

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -468,8 +468,7 @@
                     bzreturn = BZ2_bzDecompress(self.bzs)
                     # add up the size that has not been processed
                     avail_in = rffi.getintfield(self.bzs, 'c_avail_in')
-                    total_in += avail_in
-                    self.left_to_process = total_in
+                    self.left_to_process = avail_in
                     if bzreturn == BZ_STREAM_END:
                         self.running = False
                         break
@@ -483,6 +482,7 @@
                             break
                         out.prepare_next_chunk()
 
+                self.left_to_process += total_in
                 res = out.make_result_string()
                 return self.space.newbytes(res)
 

From pypy.commits at gmail.com  Thu Sep 29 10:58:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 29 Sep 2016 07:58:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: in-progress
Message-ID: <57ed2c31.a255c20a.29be8.f78b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87446:f17896d64f65
Date: 2016-09-29 16:58 +0200
http://bitbucket.org/pypy/pypy/changeset/f17896d64f65/

Log:	in-progress

diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py
--- a/pypy/module/faulthandler/__init__.py
+++ b/pypy/module/faulthandler/__init__.py
@@ -1,5 +1,7 @@
+import sys
 from pypy.interpreter.mixedmodule import MixedModule
 
+
 class Module(MixedModule):
     appleveldefs = {
     }
@@ -26,6 +28,9 @@
                                  'handler.dump_traceback_later')
             self.extra_interpdef('cancel_dump_traceback_later',
                                  'handler.cancel_dump_traceback_later')
+        if sys.platform != 'win32':
+            self.extra_interpdef('register', 'handler.register')
+            self.extra_interpdef('unregister', 'handler.unregister')
 
     def shutdown(self, space):
         from pypy.module.faulthandler import handler
diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py
--- a/pypy/module/faulthandler/cintf.py
+++ b/pypy/module/faulthandler/cintf.py
@@ -12,6 +12,8 @@
 
 eci_later = eci.merge(ExternalCompilationInfo(
     pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n']))
+eci_user = eci.merge(ExternalCompilationInfo(
+    pre_include_bits=['#define PYPY_FAULTHANDLER_USER\n']))
 
 def direct_llexternal(*args, **kwargs):
     kwargs.setdefault('_nowrapper', True)
@@ -55,6 +57,21 @@
 pypy_faulthandler_cancel_dump_traceback_later = direct_llexternal(
     'pypy_faulthandler_cancel_dump_traceback_later', [], lltype.Void)
 
+pypy_faulthandler_check_signum = direct_llexternal(
+    'pypy_faulthandler_check_signum',
+    [rffi.LONG], rffi.INT,
+    compilation_info=eci_user)
+
+pypy_faulthandler_register = direct_llexternal(
+    'pypy_faulthandler_register',
+    [rffi.INT, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP,
+    compilation_info=eci_user)
+
+pypy_faulthandler_unregister = direct_llexternal(
+    'pypy_faulthandler_unregister',
+    [rffi.INT], rffi.INT,
+    compilation_info=eci_user)
+
 
 # for tests...
 
diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c
--- a/pypy/module/faulthandler/faulthandler.c
+++ b/pypy/module/faulthandler/faulthandler.c
@@ -138,6 +138,10 @@
     reentrant = 0;
 }
 
+
+/************************************************************/
+
+
 #ifdef PYPY_FAULTHANDLER_LATER
 #include "src/thread.h"
 static struct {
@@ -243,6 +247,152 @@
 }
 
 
+/************************************************************/
+
+
+#ifdef PYPY_FAULTHANDLER_USER
+typedef struct {
+    int enabled;
+    int fd;
+    int all_threads;
+    int chain;
+    _Py_sighandler_t previous;
+} user_signal_t;
+
+static user_signal_t *user_signals;
+
+#ifndef NSIG
+# if defined(_NSIG)
+#  define NSIG _NSIG            /* For BSD/SysV */
+# elif defined(_SIGMAX)
+#  define NSIG (_SIGMAX + 1)    /* For QNX */
+# elif defined(SIGMAX)
+#  define NSIG (SIGMAX + 1)     /* For djgpp */
+# else
+#  define NSIG 64               /* Use a reasonable default value */
+# endif
+#endif
+
+static void faulthandler_user(int signum, siginfo_t *info, void *ucontext);
+
+static int
+faulthandler_register(int signum, int chain, _Py_sighandler_t *p_previous)
+{
+    struct sigaction action;
+    action.sa_handler = faulthandler_user;
+    sigemptyset(&action.sa_mask);
+    /* if the signal is received while the kernel is executing a system
+       call, try to restart the system call instead of interrupting it and
+       return EINTR. */
+    action.sa_flags = SA_RESTART | SA_SIGINFO;
+    if (chain) {
+        /* do not prevent the signal from being received from within its
+           own signal handler */
+        action.sa_flags = SA_NODEFER;
+    }
+    if (stack.ss_sp != NULL) {
+        /* Call the signal handler on an alternate signal stack
+           provided by sigaltstack() */
+        action.sa_flags |= SA_ONSTACK;
+    }
+    return sigaction(signum, &action, p_previous);
+}
+
+static void faulthandler_user(int signum, siginfo_t *info, void *ucontext)
+{
+    int save_errno;
+    user_signal_t *user = &user_signals[signum];
+
+    if (!user->enabled)
+        return;
+
+    save_errno = errno;
+    faulthandler_dump_traceback(user->fd, user->all_threads, ucontext);
+
+    if (user->chain) {
+        (void)sigaction(signum, &user->previous, NULL);
+        errno = save_errno;
+
+        /* call the previous signal handler */
+        raise(signum);
+
+        save_errno = errno;
+        (void)faulthandler_register(signum, user->chain, NULL);
+    }
+
+    errno = save_errno;
+}
+
+RPY_EXTERN
+int pypy_faulthandler_check_signum(long signum)
+{
+    unsigned int i;
+
+    for (i = 0; i < faulthandler_nsignals; i++) {
+        if (faulthandler_handlers[i].signum == signum) {
+            return -1;
+        }
+    }
+    if (signum < 1 || NSIG <= signum) {
+        return -2;
+    }
+    return 0;
+}
+
+RPY_EXTERN
+char *pypy_faulthandler_register(int signum, int fd, int all_threads, int chain)
+{
+    user_signal_t *user;
+    _Py_sighandler_t previous;
+    int err;
+
+    if (user_signals == NULL) {
+        user_signals = malloc(NSIG * sizeof(user_signal_t));
+        if (user_signals == NULL)
+            return "out of memory";
+        memset(user_signals, 0, NSIG * sizeof(user_signal_t));
+    }
+
+    user = &user_signals[signum];
+    user->fd = fd;
+    user->all_threads = all_threads;
+    user->chain = chain;
+
+    if (!user->enabled) {
+        err = faulthandler_register(signum, chain, &previous);
+        if (err)
+            return strerror(errno);
+
+        user->previous = previous;
+        user->enabled = 1;
+    }
+    return NULL;
+}
+
+RPY_EXTERN
+int pypy_faulthandler_unregister(int signum)
+{
+    user_signal_t *user;
+
+    if (user_signals == NULL)
+        return 0;
+
+    user = &user_signals[signum];
+    if (user->enabled) {
+        user->enabled = 0;
+        (void)sigaction(signum, &user->previous, NULL);
+        user->fd = -1;
+        return 1;
+    }
+    else
+        return 0;
+}
+#endif   /* PYPY_FAULTHANDLER_USER */
+
+
+/************************************************************/
+
+
 /* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals.
 
    Display the current Python traceback, restore the previous handler and call
@@ -344,6 +494,14 @@
         RPyOpaqueDealloc_ThreadLock(&thread_later.cancel_event);
 #endif
 
+#ifdef PYPY_FAULTHANDLER_USER
+        int signum;
+        for (signum = 0; signum < NSIG; signum++)
+            pypy_faulthandler_unregister(signum);
+        /* don't free 'user_signals', the gain is very minor and it can
+           lead to rare crashes if another thread is still busy */
+#endif
+
         pypy_faulthandler_disable();
         fatal_error.initialized = 0;
         if (stack.ss_sp) {
@@ -415,6 +573,9 @@
 }
 
 
+/************************************************************/
+
+
 /* for tests... */
 
 static void
diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -25,6 +25,10 @@
     long long microseconds, int repeat, int fd, int exit);
 RPY_EXTERN void pypy_faulthandler_cancel_dump_traceback_later(void);
 
+RPY_EXTERN int pypy_faulthandler_check_signum(long signum);
+RPY_EXTERN char *pypy_faulthandler_register(int, int, int, int);
+RPY_EXTERN int pypy_faulthandler_unregister(int signum);
+
 
 RPY_EXTERN int pypy_faulthandler_read_null(void);
 RPY_EXTERN void pypy_faulthandler_sigsegv(void);
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -1,5 +1,5 @@
 import os
-from rpython.rtyper.lltypesystem import llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.rposix import is_valid_fd
 from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong
 from rpython.rlib.objectmodel import keepalive_until_here
@@ -20,6 +20,7 @@
     def _cleanup_(self):
         self.fatal_error_w_file = None
         self.dump_traceback_later_w_file = None
+        self.user_w_files = None
 
     def check_err(self, p_err):
         if p_err:
@@ -98,6 +99,37 @@
         cintf.pypy_faulthandler_cancel_dump_traceback_later()
         self.dump_traceback_later_w_file = None
 
+    def check_signum(self, signum):
+        err = rffi.cast(lltype.Signed,
+                        cintf.pypy_faulthandler_check_signum(signum))
+        if err < 0:
+            space = self.space
+            if err == -1:
+                raise oefmt(space.w_RuntimeError,
+                            "signal %d cannot be registered, "
+                            "use enable() instead", signum)
+            else:
+                raise oefmt(space.w_ValueError, "signal number out of range")
+
+    def register(self, signum, w_file, all_threads, chain):
+        self.check_signum(signum)
+        fileno, w_file = self.get_fileno_and_file(w_file)
+        self.setup()
+        self.check_err(cintf.pypy_faulthandler_register(
+            rffi.cast(rffi.INT, signum),
+            rffi.cast(rffi.INT, fileno),
+            rffi.cast(rffi.INT, all_threads),
+            rffi.cast(rffi.INT, chain)))
+        if self.user_w_files is None:
+            self.user_w_files = {}
+        self.user_w_files[signum] = w_file
+
+    def unregister(self, signum):
+        self.check_signum(signum)
+        change = cintf.pypy_faulthandler_unregister(
+            rffi.cast(rffi.INT, signum))
+        return rffi.cast(lltype.Signed, change) == 1
+
     def finish(self):
         cintf.pypy_faulthandler_teardown()
         self._cleanup_()
@@ -138,6 +170,14 @@
     """cancel the previous call to dump_traceback_later()."""
     space.fromcache(Handler).cancel_dump_traceback_later()
 
+ at unwrap_spec(signum=int, all_threads=int, chain=int)
+def register(space, signum, w_file=None, all_threads=1, chain=0):
+    space.fromcache(Handler).register(signum, w_file, all_threads, chain)
+
+ at unwrap_spec(signum=int)
+def unregister(space, signum):
+    return space.wrap(space.fromcache(Handler).unregister(signum))
+
 
 # for tests...
 

From pypy.commits at gmail.com  Thu Sep 29 11:20:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 29 Sep 2016 08:20:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: add explicit dependency
Message-ID: <57ed3122.4d081c0a.e105b.215f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87447:8aae306f582f
Date: 2016-09-29 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/8aae306f582f/

Log:	add explicit dependency

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -94,6 +94,7 @@
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
+    'faulthandler': [('objspace.usemodules._vmprof', True)],
     }
 module_suggests = {
     # the reason you want _rawffi is for ctypes, which

From pypy.commits at gmail.com  Thu Sep 29 11:44:27 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 29 Sep 2016 08:44:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merged py3.5-bz2-lzma
Message-ID: <57ed36db.0145c20a.1aa82.13a8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r87448:865e96c7c129
Date: 2016-09-29 17:43 +0200
http://bitbucket.org/pypy/pypy/changeset/865e96c7c129/

Log:	merged py3.5-bz2-lzma

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -10,6 +10,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform as compiler
 from rpython.rlib.rarithmetic import intmask, r_longlong
+from rpython.rlib import rgil
 import sys
 
 
@@ -266,6 +267,7 @@
     must be a number between 1 and 9."""
     def __init__(self, space, compresslevel):
         self.space = space
+        self._lock = space.allocate_lock()
         self.bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True)
         try:
             self.running = False
@@ -276,6 +278,15 @@
             raise
         self.register_finalizer(space)
 
+    def lock(self):
+        if not self._lock.acquire(False):
+            rgil.release()
+            self._lock.acquire(True)
+            rgil.acquire()
+
+    def unlock(self):
+        self._lock.release()
+
     def _init_bz2comp(self, compresslevel):
         if compresslevel < 1 or compresslevel > 9:
             raise oefmt(self.space.w_ValueError,
@@ -306,35 +317,39 @@
         to compress, call the flush() method to finish the compression process,
         and return what is left in the internal buffers."""
 
-        datasize = len(data)
+        try:
+            self.lock()
+            datasize = len(data)
 
-        if datasize == 0:
-            return self.space.newbytes("")
+            if datasize == 0:
+                return self.space.newbytes("")
 
-        if not self.running:
-            raise oefmt(self.space.w_ValueError,
-                        "this object was already flushed")
+            if not self.running:
+                raise oefmt(self.space.w_ValueError,
+                            "this object was already flushed")
 
-        in_bufsize = datasize
+            in_bufsize = datasize
 
-        with OutBuffer(self.bzs) as out:
-            with rffi.scoped_nonmovingbuffer(data) as in_buf:
+            with OutBuffer(self.bzs) as out:
+                with rffi.scoped_nonmovingbuffer(data) as in_buf:
 
-                self.bzs.c_next_in = in_buf
-                rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
+                    self.bzs.c_next_in = in_buf
+                    rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
 
-                while True:
-                    bzerror = BZ2_bzCompress(self.bzs, BZ_RUN)
-                    if bzerror != BZ_RUN_OK:
-                        _catch_bz2_error(self.space, bzerror)
+                    while True:
+                        bzerror = BZ2_bzCompress(self.bzs, BZ_RUN)
+                        if bzerror != BZ_RUN_OK:
+                            _catch_bz2_error(self.space, bzerror)
 
-                    if rffi.getintfield(self.bzs, 'c_avail_in') == 0:
-                        break
-                    elif rffi.getintfield(self.bzs, 'c_avail_out') == 0:
-                        out.prepare_next_chunk()
+                        if rffi.getintfield(self.bzs, 'c_avail_in') == 0:
+                            break
+                        elif rffi.getintfield(self.bzs, 'c_avail_out') == 0:
+                            out.prepare_next_chunk()
 
-                res = out.make_result_string()
-                return self.space.newbytes(res)
+                    res = out.make_result_string()
+                    return self.space.newbytes(res)
+        finally:
+            self.unlock()
 
     def flush(self):
         if not self.running:
@@ -342,19 +357,23 @@
                         "this object was already flushed")
         self.running = False
 
-        with OutBuffer(self.bzs) as out:
-            while True:
-                bzerror = BZ2_bzCompress(self.bzs, BZ_FINISH)
-                if bzerror == BZ_STREAM_END:
-                    break
-                elif bzerror != BZ_FINISH_OK:
-                    _catch_bz2_error(self.space, bzerror)
+        try:
+            self.lock()
+            with OutBuffer(self.bzs) as out:
+                while True:
+                    bzerror = BZ2_bzCompress(self.bzs, BZ_FINISH)
+                    if bzerror == BZ_STREAM_END:
+                        break
+                    elif bzerror != BZ_FINISH_OK:
+                        _catch_bz2_error(self.space, bzerror)
 
-                if rffi.getintfield(self.bzs, 'c_avail_out') == 0:
-                    out.prepare_next_chunk()
+                    if rffi.getintfield(self.bzs, 'c_avail_out') == 0:
+                        out.prepare_next_chunk()
 
-            res = out.make_result_string()
-            return self.space.newbytes(res)
+                res = out.make_result_string()
+                return self.space.newbytes(res)
+        finally:
+            self.unlock()
 
 W_BZ2Compressor.typedef = TypeDef("_bz2.BZ2Compressor",
     __doc__ = W_BZ2Compressor.__doc__,
@@ -380,6 +399,7 @@
 
     def __init__(self, space):
         self.space = space
+        self._lock = space.allocate_lock()
 
         self.bzs = lltype.malloc(bz_stream.TO, flavor='raw', zero=True)
         try:
@@ -396,6 +416,15 @@
             raise
         self.register_finalizer(space)
 
+    def lock(self):
+        if not self._lock.acquire(False):
+            rgil.release()
+            self._lock.acquire(True)
+            rgil.acquire()
+
+    def unlock(self):
+        self._lock.release()
+
     def _init_bz2decomp(self):
         bzerror = BZ2_bzDecompressInit(self.bzs, 0, 0)
         if bzerror != BZ_OK:
@@ -425,11 +454,14 @@
             return space.w_True
 
     def _decompress_buf(self, data, max_length):
-        in_bufsize = len(data)
+        total_in = len(data)
+        in_bufsize = min(total_in, UINT_MAX)
+        total_in -= in_bufsize
         with rffi.scoped_nonmovingbuffer(data) as in_buf:
             # setup the input and the size it can consume
             self.bzs.c_next_in = in_buf
             rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize)
+            self.left_to_process = in_bufsize
 
             with OutBuffer(self.bzs, max_length=max_length) as out:
                 while True:
@@ -450,13 +482,7 @@
                             break
                         out.prepare_next_chunk()
 
-                if not self.running:
-                    self.needs_input = False
-                    if self.left_to_process != 0:
-                        end = len(data)
-                        start = end - self.left_to_process
-                        assert start > 0
-                        self.unused_data = data[start:]
+                self.left_to_process += total_in
                 res = out.make_result_string()
                 return self.space.newbytes(res)
 
@@ -470,30 +496,38 @@
         was found after the end of stream, it'll be ignored and saved in
         unused_data attribute."""
 
-        if not self.running:
-            raise oefmt(self.space.w_EOFError,
-                        "end of stream was already found")
-        datalen = len(data)
-        if len(self.input_buffer) > 0:
-            input_buffer_in_use = True
-            data = self.input_buffer + data
+        try:
+            self.lock()
+            if not self.running:
+                raise oefmt(self.space.w_EOFError,
+                            "end of stream was already found")
             datalen = len(data)
-            result = self._decompress_buf(data, max_length)
-        else:
-            input_buffer_in_use = False
+            if len(self.input_buffer) > 0:
+                data = self.input_buffer + data
+                datalen = len(data)
+                self.input_buffer = ""
+
             result = self._decompress_buf(data, max_length)
 
-        if self.left_to_process == 0:
-            self.input_buffer = ""
-            self.needs_input = True
-        else:
-            self.needs_input = False
-            if not input_buffer_in_use:
-                start = datalen-self.left_to_process
-                assert start > 0
-                self.input_buffer = data[start:]
-
-        return result
+            if not self.running: # eq. with eof == Ture
+                self.needs_input = False
+                if self.left_to_process != 0:
+                    start = datalen - self.left_to_process
+                    assert start > 0
+                    self.unused_data = data[start:]
+                    self.left_to_process = 0
+            elif self.left_to_process == 0:
+                self.input_buffer = ""
+                self.needs_input = True
+            else:
+                self.needs_input = False
+                if self.left_to_process > 0:
+                    start = datalen-self.left_to_process
+                    assert start >= 0
+                    self.input_buffer = data[start:]
+            return result
+        finally:
+            self.unlock()
 
 
 
diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -213,8 +213,8 @@
         assert len(decomp[-1]) == 100
 
         while not bz2d.eof:
-            decomp.append(bz2d.decompress(b"", max_length=50))
-            assert len(decomp[-1]) <= 50
+            decomp.append(bz2d.decompress(b"", max_length=200))
+            assert len(decomp[-1]) <= 200
 
         assert b''.join(decomp) == self.TEXT
 

From pypy.commits at gmail.com  Thu Sep 29 11:45:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 29 Sep 2016 08:45:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3k-faulthandler: fix
Message-ID: <57ed372d.02c11c0a.5b222.a5a6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k-faulthandler
Changeset: r87449:acb5b449d753
Date: 2016-09-29 17:45 +0200
http://bitbucket.org/pypy/pypy/changeset/acb5b449d753/

Log:	fix

diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -128,6 +128,8 @@
         self.check_signum(signum)
         change = cintf.pypy_faulthandler_unregister(
             rffi.cast(rffi.INT, signum))
+        if self.user_w_files is not None:
+            self.user_w_files.pop(signum, None)
         return rffi.cast(lltype.Signed, change) == 1
 
     def finish(self):

From pypy.commits at gmail.com  Thu Sep 29 12:47:24 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 29 Sep 2016 09:47:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57ed459c.4756c20a.c7ddf.272e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87450:db0da9ef9eca
Date: 2016-09-29 17:46 +0100
http://bitbucket.org/pypy/pypy/changeset/db0da9ef9eca/

Log:	hg merge default

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py
+	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -34,9 +34,11 @@
    This function searches the PyPy standard library starting from the given
    "PyPy home directory".  The arguments are:
 
-   * ``home``: NULL terminated path to an executable inside the pypy directory
+   * ``home``: path to an executable inside the pypy directory
      (can be a .so name, can be made up).  Used to look up the standard
-     library, and is also set as ``sys.executable``.
+     library, and is also set as ``sys.executable``.  From PyPy 5.5, you can
+     just say NULL here, as long as the ``libpypy-c.so/dylib/dll`` is itself
+     inside this directory.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
@@ -82,18 +84,14 @@
 
 Note that this API is a lot more minimal than say CPython C API, so at first
 it's obvious to think that you can't do much. However, the trick is to do
-all the logic in Python and expose it via `cffi`_ callbacks. Let's assume
-we're on linux and pypy is installed in ``/opt/pypy`` (with
-subdirectories like ``lib-python`` and ``lib_pypy``), and with the
-library in ``/opt/pypy/bin/libpypy-c.so``.  (It doesn't need to be
-installed; you can also replace these paths with a local extract of the
-installation tarballs, or with your local checkout of pypy.) We write a
-little C program:
+all the logic in Python and expose it via `cffi`_ callbacks.
+We write a little C program:
 
 .. code-block:: c
 
     #include "PyPy.h"
     #include <stdio.h>
+    #include <stdlib.h>
 
     static char source[] = "print 'hello from pypy'";
 
@@ -102,9 +100,9 @@
         int res;
 
         rpython_startup_code();
-        /* note: in the path /opt/pypy/x, the final x is ignored and
-           replaced with lib-python and lib_pypy. */
-        res = pypy_setup_home("/opt/pypy/x", 1);
+        /* Before PyPy 5.5, you may need to say e.g. "/opt/pypy/bin" instead
+         * of NULL. */
+        res = pypy_setup_home(NULL, 1);
         if (res) {
             printf("Error setting pypy home!\n");
             return 1;
@@ -123,11 +121,6 @@
     $ LD_LIBRARY_PATH=/opt/pypy/bin ./x
     hello from pypy
 
-.. note:: If the compilation fails because of missing PyPy.h header file,
-          you are running PyPy <= 2.2.1.  Get it here__.
-
-.. __: https://bitbucket.org/pypy/pypy/raw/c4cd6eca9358066571500ac82aaacfdaa3889e8c/include/PyPy.h
-
 On OSX it is necessary to set the rpath of the binary if one wants to link to it,
 with a command like::
 
@@ -181,6 +174,7 @@
     /* C example */
     #include "PyPy.h"
     #include <stdio.h>
+    #include <stdlib.h>
 
     struct API {
         double (*add_numbers)(double x, double y);
@@ -196,7 +190,7 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/x", 1);
+        res = pypy_setup_home(NULL, 1);
         if (res) {
             fprintf(stderr, "Error setting pypy home!\n");
             return -1;
@@ -237,6 +231,8 @@
 Finding pypy_home
 -----------------
 
+**You can usually skip this section if you are running PyPy >= 5.5.**
+
 The function pypy_setup_home() takes as first parameter the path to a
 file from which it can deduce the location of the standard library.
 More precisely, it tries to remove final components until it finds
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -27,3 +27,14 @@
 JIT residual calls: if the called function starts with a fast-path
 like "if x.foo != 0: return x.foo", then inline the check before
 doing the CALL.  For now, string hashing is about the only case.
+
+.. branch: search-path-from-libpypy
+
+The compiled pypy now looks for its lib-python/lib_pypy path starting
+from the location of the *libpypy-c* instead of the executable. This is
+arguably more consistent, and also it is what occurs anyway if you're
+embedding pypy.  Linux distribution packagers, take note!  At a minimum,
+the ``libpypy-c.so`` must really be inside the path containing
+``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
+from somewhere else.  You no longer have to do the same with the
+``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -96,17 +96,18 @@
     def pypy_setup_home(ll_home, verbose):
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
-        if ll_home:
+        if ll_home and ord(ll_home[0]):
             home1 = rffi.charp2str(ll_home)
             home = os.path.join(home1, 'x') # <- so that 'll_home' can be
                                             # directly the root directory
         else:
-            home = home1 = pypydir
+            home1 = "pypy's shared library location"
+            home = '*'
         w_path = pypy_find_stdlib(space, home)
         if space.is_none(w_path):
             if verbose:
                 debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
-                      " not found in '%s' or in any parent directory" % home1)
+                      " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.appexec([w_path], """(path):
@@ -306,37 +307,20 @@
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_* to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.tool.runsubprocess import run_subprocess
         from rpython.translator.driver import taskdef
         import types
 
-        class Options(object):
-            pass
-
-
-        def mkexename(name):
-            if sys.platform == 'win32':
-                name = name.new(ext='exe')
-            return name
-
         compile_goal, = driver.backend_select_goals(['compile'])
         @taskdef([compile_goal], "Create cffi bindings for modules")
         def task_build_cffi_imports(self):
-            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
             ''' Use cffi to compile cffi interfaces to modules'''
-            exename = mkexename(driver.compute_exe_name())
-            basedir = exename
-            while not basedir.join('include').exists():
-                _basedir = basedir.dirpath()
-                if _basedir == basedir:
-                    raise ValueError('interpreter %s not inside pypy repo',
-                                     str(exename))
-                basedir = _basedir
-            modules = self.config.objspace.usemodules.getpaths()
-            options = Options()
-            # XXX possibly adapt options using modules
-            failures = create_cffi_import_libraries(exename, options, basedir)
-            # if failures, they were already printed
-            print  >> sys.stderr, str(exename),'successfully built (errors, if any, while building the above modules are ignored)'
+            filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py')
+            status, out, err = run_subprocess(str(driver.compute_exe_name()),
+                                              [filename])
+            sys.stdout.write(out)
+            sys.stderr.write(err)
+            # otherwise, ignore errors
         driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
         driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal]
         driver.default_goal = 'build_cffi_imports'
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -379,12 +379,12 @@
 class BufferInterfaceNotFound(Exception):
     pass
 
+ at specialize.memo()
 def wrappable_class_name(Class):
     try:
         return Class.typedef.name
     except AttributeError:
         return 'internal subclass of %s' % (Class.__name__,)
-wrappable_class_name._annspecialcase_ = 'specialize:memo'
 
 class CannotHaveLock(Exception):
     """Raised by space.allocate_lock() if we're translating."""
@@ -836,12 +836,13 @@
         u = s.decode('utf-8')
         return self.interned_strings.get(u) is not None
 
+    @specialize.arg(1)
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
             raise DescrMismatch()
         return w_obj
-    descr_self_interp_w._annspecialcase_ = 'specialize:arg(1)'
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         """
         Unwrap w_obj, checking that it is an instance of the required internal
@@ -856,7 +857,6 @@
                         wrappable_class_name(RequiredClass),
                         w_obj.getclass(self))
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def unpackiterable(self, w_iterable, expected_length=-1):
         """Unpack an iterable into a real (interpreter-level) list.
@@ -1270,6 +1270,7 @@
             self.setitem(w_globals, w_key, self.wrap(self.builtin))
         return statement.exec_code(self, w_globals, w_locals)
 
+    @specialize.arg(2)
     def appexec(self, posargs_w, source):
         """ return value from executing given source at applevel.
             EXPERIMENTAL. The source must look like
@@ -1281,7 +1282,6 @@
         w_func = self.fromcache(AppExecCache).getorbuild(source)
         args = Arguments(self, list(posargs_w))
         return self.call_args(w_func, args)
-    appexec._annspecialcase_ = 'specialize:arg(2)'
 
     def _next_or_none(self, w_it):
         try:
@@ -1291,6 +1291,7 @@
                 raise
             return None
 
+    @specialize.arg(3)
     def compare_by_iteration(self, w_iterable1, w_iterable2, op):
         w_it1 = self.iter(w_iterable1)
         w_it2 = self.iter(w_iterable2)
@@ -1313,7 +1314,6 @@
                 if op == 'gt': return self.gt(w_x1, w_x2)
                 if op == 'ge': return self.ge(w_x1, w_x2)
                 assert False, "bad value for op"
-    compare_by_iteration._annspecialcase_ = 'specialize:arg(3)'
 
     def decode_index(self, w_index_or_slice, seqlength):
         """Helper for custom sequence implementations
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,7 +1,7 @@
 import sys
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib import jit, rgc
+from rpython.rlib import jit, rgc, objectmodel
 
 TICK_COUNTER_STEP = 100
 
@@ -130,6 +130,7 @@
         if self.gettrace() is not None:
             self._trace(frame, 'return', w_retval)
 
+    @objectmodel.always_inline
     def bytecode_trace(self, frame, decr_by=TICK_COUNTER_STEP):
         "Trace function called before each bytecode."
         # this is split into a fast path and a slower path that is
@@ -138,7 +139,6 @@
         actionflag = self.space.actionflag
         if actionflag.decrement_ticker(decr_by) < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
-    bytecode_trace._always_inline_ = True
 
     def _run_finalizers_now(self):
         # Tests only: run the actions now, to ensure that the
@@ -146,6 +146,7 @@
         # pypy.tool.pytest.apptest.
         self.space.actionflag.action_dispatcher(self, None)
 
+    @objectmodel.always_inline
     def bytecode_only_trace(self, frame):
         """
         Like bytecode_trace() but doesn't invoke any other events besides the
@@ -155,7 +156,6 @@
             self.gettrace() is None):
             return
         self.run_trace_func(frame)
-    bytecode_only_trace._always_inline_ = True
 
     @jit.unroll_safe
     def run_trace_func(self, frame):
@@ -202,13 +202,13 @@
 
         d.instr_prev_plus_one = frame.last_instr + 1
 
+    @objectmodel.try_inline
     def bytecode_trace_after_exception(self, frame):
         "Like bytecode_trace(), but without increasing the ticker."
         actionflag = self.space.actionflag
         self.bytecode_only_trace(frame)
         if actionflag.get_ticker() < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
-    bytecode_trace_after_exception._always_inline_ = 'try'
     # NB. this function is not inlined right now.  backendopt.inline would
     # need some improvements to handle this case, but it's not really an
     # issue
@@ -452,6 +452,7 @@
         periodic_actions = unrolling_iterable(self._periodic_actions)
 
         @jit.unroll_safe
+        @objectmodel.dont_inline
         def action_dispatcher(ec, frame):
             # periodic actions (first reset the bytecode counter)
             self.reset_ticker(self.checkinterval_scaled)
@@ -473,7 +474,6 @@
                     action._fired = False
                     action.perform(ec, frame)
 
-        action_dispatcher._dont_inline_ = True
         self.action_dispatcher = action_dispatcher
 
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -6,7 +6,8 @@
 
 from rpython.rlib import jit, rstackovf, rstring
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import (we_are_translated, always_inline,
+        dont_inline)
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -453,20 +454,20 @@
         # of oparg failed to produce an integer which is annotated as non-neg
         check_nonneg(oparg)
 
+    @always_inline
     def LOAD_FAST(self, varindex, next_instr):
         # access a local variable directly
         w_value = self.locals_cells_stack_w[varindex]
         if w_value is None:
             self._load_fast_failed(varindex)
         self.pushvalue(w_value)
-    LOAD_FAST._always_inline_ = True
 
+    @dont_inline
     def _load_fast_failed(self, varindex):
         varname = self.getlocalvarname(varindex)
         raise oefmt(self.space.w_UnboundLocalError,
                     "local variable '%s' referenced before assignment",
                     varname)
-    _load_fast_failed._dont_inline_ = True
 
     def LOAD_CONST(self, constindex, next_instr):
         w_const = self.getconstant_w(constindex)
@@ -847,26 +848,26 @@
                         "name %R is not defined", w_varname)
         self.pushvalue(w_value)
 
+    @always_inline
     def _load_global(self, varname):
         w_value = self.space.finditem_str(self.get_w_globals(), varname)
         if w_value is None:
             # not in the globals, now look in the built-ins
             w_value = self.get_builtin().getdictvalue(self.space, varname)
         return w_value
-    _load_global._always_inline_ = True
 
+    @dont_inline
     def _load_global_failed(self, w_varname):
         raise oefmt(self.space.w_NameError,
                     "global name %R is not defined", w_varname)
-    _load_global_failed._dont_inline_ = True
 
+    @always_inline
     def LOAD_GLOBAL(self, nameindex, next_instr):
         w_varname = self.getname_w(nameindex)
         w_value = self._load_global(self.space.identifier_w(w_varname))
         if w_value is None:
             self._load_global_failed(w_varname)
         self.pushvalue(w_value)
-    LOAD_GLOBAL._always_inline_ = True
 
     def DELETE_FAST(self, varindex, next_instr):
         if self.locals_cells_stack_w[varindex] is None:
@@ -900,6 +901,7 @@
         self.pushvalue(space.newlist([], sizehint=length_hint))
         self.pushvalue(last_val)
 
+    @always_inline
     def LOAD_ATTR(self, nameindex, next_instr):
         "obj.attributename"
         w_obj = self.popvalue()
@@ -910,7 +912,6 @@
             w_attributename = self.getname_w(nameindex)
             w_value = self.space.getattr(w_obj, w_attributename)
         self.pushvalue(w_value)
-    LOAD_ATTR._always_inline_ = True
 
     @jit.unroll_safe
     def cmp_exc_match(self, w_1, w_2):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1089,23 +1089,32 @@
         old_sys_path = sys.path[:]
         old_cwd = os.getcwd()
 
-        sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
         if self.tmp_dir.startswith(self.trunkdir):
             skip('TMPDIR is inside the PyPy source')
-        os.chdir(self.tmp_dir)
+        sys.path.append(self.goal_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
+            os.chdir(self.tmp_dir)
+
+            # If we are running PyPy with a libpypy-c, the following
+            # lines find the stdlib anyway.  Otherwise, it is not found.
+            expected_found = (
+                getattr(sys, 'pypy_translation_info', {})
+                .get('translation.shared'))
+
             import app_main
-            app_main.setup_bootstrap_path(tmp_pypy_c)  # stdlib not found
+            app_main.setup_bootstrap_path(tmp_pypy_c)
             assert sys.executable == ''
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not expected_found:
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
             if not sys.platform == 'win32':
                 # an existing file is always 'executable' on windows
                 assert sys.executable == ''      # not executable!
-                assert sys.path == old_sys_path + [self.goal_dir]
+                if not expected_found:
+                    assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0o755)
             app_main.setup_bootstrap_path(self.fake_exe)
@@ -1116,7 +1125,8 @@
             if newpath[0].endswith('__extensions__'):
                 newpath = newpath[1:]
             # we get at least 'expected_path', and maybe more (e.g.plat-linux2)
-            assert newpath[:len(self.expected_path)] == self.expected_path
+            if not expected_found:
+                assert newpath[:len(self.expected_path)] == self.expected_path
         finally:
             sys.path[:] = old_sys_path
             os.chdir(old_cwd)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -109,6 +109,7 @@
 # we need two subclasses of the app-level type, one to add mapdict, and then one
 # to add del to not slow down the GC.
 
+ at specialize.memo()
 def get_unique_interplevel_subclass(space, cls):
     "NOT_RPYTHON: initialization-time only"
     assert cls.typedef.acceptable_as_base_class
@@ -119,7 +120,6 @@
         assert cls not in _unique_subclass_cache
         _unique_subclass_cache[cls] = subcls
         return subcls
-get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo"
 _unique_subclass_cache = {}
 
 def _getusercls(cls, reallywantdict=False):
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -112,29 +112,7 @@
 #define _WIN32_WINNT 0x0501
 #include <windows.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  _MAX_PATH
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    HMODULE hModule = 0;
-    DWORD res;
-
-    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | 
-                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-                       (LPCTSTR)&_cffi_init, &hModule);
-
-    if (hModule == 0 ) {
-        _cffi_init_error("GetModuleHandleEx() failed", "");
-        return -1;
-    }
-    res = GetModuleFileName(hModule, output_home_path, CFFI_INIT_HOME_PATH_MAX);
-    if (res >= CFFI_INIT_HOME_PATH_MAX) {
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -155,28 +133,9 @@
 else:
 
     do_includes = r"""
-#include <dlfcn.h>
 #include <pthread.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  PATH_MAX
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    Dl_info info;
-    dlerror();   /* reset */
-    if (dladdr(&_cffi_init, &info) == 0) {
-        _cffi_init_error("dladdr() failed: ", dlerror());
-        return -1;
-    }
-    if (realpath(info.dli_fname, output_home_path) == NULL) {
-        perror("realpath() failed");
-        _cffi_init_error("realpath() failed", "");
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -201,14 +160,10 @@
 
 static void _cffi_init(void)
 {
-    char home[CFFI_INIT_HOME_PATH_MAX + 1];
-
     rpython_startup_code();
     RPyGilAllocate();
 
-    if (_cffi_init_home(home) != 0)
-        return;
-    if (pypy_setup_home(home, 1) != 0) {
+    if (pypy_setup_home(NULL, 1) != 0) {
         _cffi_init_error("pypy_setup_home() failed", "");
         return;
     }
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.debug import check_nonneg
+from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
@@ -316,12 +317,12 @@
             w_currently_in_repr = ec._py_repr = space.newdict()
         return dequerepr(space, w_currently_in_repr, space.wrap(self))
 
+    @specialize.arg(2)
     def compare(self, w_other, op):
         space = self.space
         if not isinstance(w_other, W_Deque):
             return space.w_NotImplemented
         return space.compare_by_iteration(space.wrap(self), w_other, op)
-    compare._annspecialcase_ = 'specialize:arg(2)'
 
     def lt(self, w_other):
         return self.compare(w_other, 'lt')
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
@@ -24,13 +25,13 @@
     def iter_w(self):
         return self.space.wrap(self)
 
+    @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         msg = u'line %d: %s' % (self.line_num, msg)
         w_module = space.getbuiltinmodule('_csv')
         w_error = space.getattr(w_module, space.wrap('Error'))
         raise OperationError(w_error, space.wrap(msg))
-    error._dont_inline_ = True
 
     def add_char(self, field_builder, c):
         assert field_builder is not None
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.typedef import TypeDef, interp2app
@@ -21,12 +22,12 @@
             special += dialect.quotechar
         self.special_characters = special
 
+    @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
         w_error = space.getattr(w_module, space.wrap('Error'))
         raise OperationError(w_error, space.wrap(msg))
-    error._dont_inline_ = True
 
     def writerow(self, w_fields):
         """Construct and write a CSV record from a sequence of fields.
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -7,7 +7,7 @@
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
                                       interp_attrproperty)
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, always_inline
 from rpython.rlib.rtimer import read_timestamp, _is_64_bit
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -258,7 +258,7 @@
         return w_frame.wrap_string(space)
     return w_frame    # actually a PyCode object
 
-
+ at always_inline
 def prepare_spec(space, w_arg):
     if isinstance(w_arg, Method):
         return (w_arg.w_function, space.type(w_arg.w_instance))
@@ -266,8 +266,6 @@
         return (w_arg, None)
     else:
         return (None, space.type(w_arg))
-prepare_spec._always_inline_ = True
-
 
 def lsprof_call(space, w_self, frame, event, w_arg):
     assert isinstance(w_self, W_Profiler)
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1,6 +1,6 @@
 import sys
 from rpython.rlib.rstring import StringBuilder
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, always_inline
 from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import oefmt
@@ -189,6 +189,7 @@
         self.pos = i
         return self.space.call_function(self.space.w_int, self.space.wrap(s))
 
+    @always_inline
     def parse_integer(self, i):
         "Parse a decimal number with an optional minus sign"
         sign = 1
@@ -219,7 +220,6 @@
         # overflowed
         ovf_maybe = (count >= OVF_DIGITS)
         return i, ovf_maybe, sign * intval
-    parse_integer._always_inline_ = True
 
     def decode_array(self, i):
         w_list = self.space.newlist([])
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -6,6 +6,7 @@
 import time
 from pypy.interpreter.error import OperationError
 from pypy.module._pypyjson.interp_decoder import loads
+from rpython.rlib.objectmodel import specialize, dont_inline
 
 
 ## MSG = open('msg.json').read()
@@ -68,11 +69,11 @@
         assert isinstance(w_x, W_String)
         return w_x.strval
 
+    @dont_inline
     def call_method(self, obj, name, arg):
         assert name == 'append'
         assert isinstance(obj, W_List)
         obj.listval.append(arg)
-    call_method._dont_inline_ = True
 
     def call_function(self, w_func, *args_w):
         return self.w_None # XXX
@@ -91,6 +92,7 @@
     def wrapfloat(self, x):
         return W_Float(x)
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if isinstance(x, int):
             return W_Int(x)
@@ -100,7 +102,6 @@
         ##     assert False
         else:
             return W_Unicode(unicode(x))
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
 
 fakespace = FakeSpace()
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -11,6 +11,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 import rpython.rlib.rposix as rposix
 
 _MS_WINDOWS = os.name == "nt"
@@ -262,6 +263,7 @@
 
 _ARM = rffi_platform.getdefined('__arm__', '')
 
+ at specialize.arg(2)
 def read_ptr(ptr, ofs, TP):
     T = lltype.Ptr(rffi.CArray(TP))
     for c in unroll_letters_for_floats:
@@ -281,8 +283,8 @@
                 return ptr_val
     else:
         return rffi.cast(T, ptr)[ofs]
-read_ptr._annspecialcase_ = 'specialize:arg(2)'
 
+ at specialize.argtype(2)
 def write_ptr(ptr, ofs, value):
     TP = lltype.typeOf(value)
     T = lltype.Ptr(rffi.CArray(TP))
@@ -303,7 +305,6 @@
                 return
     else:
         rffi.cast(T, ptr)[ofs] = value
-write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
@@ -375,11 +376,12 @@
     def getrawsize(self):
         raise NotImplementedError("abstract base class")
 
+ at specialize.arg(0)
 def unwrap_truncate_int(TP, space, w_arg):
     return rffi.cast(TP, space.bigint_w(w_arg).ulonglongmask())
-unwrap_truncate_int._annspecialcase_ = 'specialize:arg(0)'
 
 
+ at specialize.arg(1)
 def unwrap_value(space, push_func, add_arg, argdesc, letter, w_arg):
     if letter in TYPEMAP_PTR_LETTERS:
         # check for NULL ptr
@@ -423,10 +425,10 @@
                 return
         else:
             raise oefmt(space.w_TypeError, "cannot directly write value")
-unwrap_value._annspecialcase_ = 'specialize:arg(1)'
 
 ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items())
 
+ at specialize.arg(1)
 def wrap_value(space, func, add_arg, argdesc, letter):
     for c, ll_type in ll_typemap_iter:
         if letter == c:
@@ -440,7 +442,6 @@
             else:
                 return space.wrap(func(add_arg, argdesc, ll_type))
     raise oefmt(space.w_TypeError, "cannot directly read value")
-wrap_value._annspecialcase_ = 'specialize:arg(1)'
 
 NARROW_INTEGER_TYPES = 'cbhiBIH?'
 
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, signedtype, r_uint, \
     r_ulonglong
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import specialize
 import sys
 
 IS_BIG_ENDIAN = sys.byteorder == 'big'
@@ -284,6 +285,7 @@
 def NUM_BITS(x):
     return x >> 16
 
+ at specialize.arg(1)
 def BIT_MASK(x, ll_t):
     if ll_t is lltype.SignedLongLong or ll_t is lltype.UnsignedLongLong:
         one = r_ulonglong(1)
@@ -291,8 +293,8 @@
         one = r_uint(1)
     # to avoid left shift by x == sizeof(ll_t)
     return (((one << (x - 1)) - 1) << 1) + 1
-BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.argtype(2)
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
     TP = lltype.typeOf(value)
@@ -313,8 +315,8 @@
                 value = rffi.cast(TP, current)
             break
     write_ptr(ptr, 0, value)
-push_field._annspecialcase_ = 'specialize:argtype(2)'
 
+ at specialize.arg(2)
 def cast_pos(self, i, ll_t):
     pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
     value = read_ptr(pos, 0, ll_t)
@@ -337,7 +339,6 @@
                 value = rffi.cast(ll_t, value)
             break
     return value
-cast_pos._annspecialcase_ = 'specialize:arg(2)'
 
 class W_StructureInstance(W_DataInstance):
     def __init__(self, space, shape, address):
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -2,6 +2,7 @@
 import sys
 
 from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError, oefmt
 
 class State:
@@ -17,6 +18,7 @@
     else:
         return space.float_w(space.float(w_x))
 
+ at specialize.arg(1)
 def math1(space, f, w_x):
     x = _get_double(space, w_x)
     try:
@@ -26,8 +28,8 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return space.wrap(y)
-math1._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.arg(1)
 def math1_w(space, f, w_x):
     x = _get_double(space, w_x)
     try:
@@ -37,8 +39,8 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return r
-math1_w._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.arg(1)
 def math2(space, f, w_x, w_snd):
     x = _get_double(space, w_x)
     snd = _get_double(space, w_snd)
@@ -49,7 +51,6 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return space.wrap(r)
-math2._annspecialcase_ = 'specialize:arg(1)'
 
 def trunc(space, w_x):
     """Truncate x."""
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rlib import rmmap, rarithmetic
+from rpython.rlib import rmmap, rarithmetic, objectmodel
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
@@ -294,6 +294,7 @@
 ACCESS_COPY  = rmmap.ACCESS_COPY
 
 
+ at objectmodel.dont_inline
 def mmap_error(space, e):
     if isinstance(e, RValueError):
         return OperationError(space.w_ValueError, space.wrap(e.message))
@@ -304,7 +305,6 @@
     else:
         # bogus 'e'?
         return OperationError(space.w_SystemError, space.wrap('%s' % e))
-mmap_error._dont_inline_ = True
 
 
 class MMapBuffer(Buffer):
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -8,6 +8,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
+from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
@@ -570,6 +571,7 @@
             return self.w_character_data_handler or space.w_None
         return self.handlers[index]
 
+    @specialize.arg(2)
     def sethandler(self, space, name, w_handler, index, setter, handler):
         if name == 'CharacterDataHandler':
             self.flush_character_buffer(space)
@@ -581,8 +583,6 @@
         self.handlers[index] = w_handler
         setter(self.itself, handler)
 
-    sethandler._annspecialcase_ = 'specialize:arg(2)'
-
     all_chars = ''.join(chr(i) for i in range(256))
 
     def UnknownEncodingHandler(self, space, name, info):
diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py
--- a/pypy/module/select/interp_select.py
+++ b/pypy/module/select/interp_select.py
@@ -3,6 +3,7 @@
 from rpython.rlib import _rsocket_rffi as _c, rpoll
 from rpython.rlib.rarithmetic import USHRT_MAX
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib import objectmodel
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
@@ -87,7 +88,7 @@
 
 # ____________________________________________________________
 
-
+ at objectmodel.always_inline  # get rid of the tuple result
 def _build_fd_set(space, list_w, ll_list, nfds):
     _c.FD_ZERO(ll_list)
     fdlist = []
@@ -101,7 +102,6 @@
         _c.FD_SET(fd, ll_list)
         fdlist.append(fd)
     return fdlist, nfds
-_build_fd_set._always_inline_ = True    # get rid of the tuple result
 
 
 def _unbuild_fd_set(space, list_w, fdlist, ll_list, reslist_w):
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -7,8 +7,10 @@
 import stat
 import sys
 
-from rpython.rlib import rpath
+from rpython.rlib import rpath, rdynload
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.sys.state import get as get_state
@@ -157,9 +159,18 @@
 
 @unwrap_spec(executable='fsencode')
 def pypy_find_stdlib(space, executable):
-    path, prefix = find_stdlib(get_state(space), executable)
+    path, prefix = None, None
+    if executable != '*':
+        path, prefix = find_stdlib(get_state(space), executable)
     if path is None:
-        return space.w_None
+        if space.config.translation.shared:
+            dynamic_location = pypy_init_home()
+            if dynamic_location:
+                dyn_path = rffi.charp2str(dynamic_location)
+                pypy_init_free(dynamic_location)
+                path, prefix = find_stdlib(get_state(space), dyn_path)
+        if path is None:
+            return space.w_None
     w_prefix = space.wrap_fsdecoded(prefix)
     space.setitem(space.sys.w_dict, space.wrap('prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), w_prefix)
@@ -169,3 +180,77 @@
 
 def pypy_initfsencoding(space):
     space.sys.filesystemencoding = _getfilesystemencoding(space)
+
+
+# ____________________________________________________________
+
+
+if os.name == 'nt':
+
+    _source_code = r"""
+#define _WIN32_WINNT 0x0501
+#include <windows.h>
+#include <stdio.h>
+
+RPY_EXPORTED
+char *_pypy_init_home(void)
+{
+    HMODULE hModule = 0;
+    DWORD res;
+    char *p;
+
+    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                       (LPCTSTR)&_pypy_init_home, &hModule);
+
+    if (hModule == 0 ) {
+        fprintf(stderr, "PyPy initialization: GetModuleHandleEx() failed\n");
+        return NULL;
+    }
+    p = malloc(_MAX_PATH);
+    if (p == NULL)
+        return NULL;
+    res = GetModuleFileName(hModule, p, _MAX_PATH);
+    if (res >= _MAX_PATH || res <= 0) {
+        free(p);
+        fprintf(stderr, "PyPy initialization: GetModuleFileName() failed\n");
+        return NULL;
+    }
+    return p;
+}
+"""
+
+else:
+
+    _source_code = r"""
+#include <dlfcn.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+RPY_EXPORTED
+char *_pypy_init_home(void)
+{
+    Dl_info info;
+    dlerror();   /* reset */
+    if (dladdr(&_pypy_init_home, &info) == 0) {
+        fprintf(stderr, "PyPy initialization: dladdr() failed: %s\n",
+                dlerror());
+        return NULL;
+    }
+    char *p = realpath(info.dli_fname, NULL);
+    if (p == NULL) {
+        p = strdup(info.dli_fname);
+    }
+    return p;
+}
+"""
+
+_eci = ExternalCompilationInfo(separate_module_sources=[_source_code],
+    post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);'])
+_eci = _eci.merge(rdynload.eci)
+
+pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP,
+                                 _nowrapper=True, compilation_info=_eci)
+pypy_init_free = rffi.llexternal("free", [rffi.CCHARP], lltype.Void,
+                                 _nowrapper=True, compilation_info=_eci)
diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py
--- a/pypy/module/sys/test/test_initpath.py
+++ b/pypy/module/sys/test/test_initpath.py
@@ -1,8 +1,10 @@
 import py
 import os.path
-from pypy.module.sys.initpath import (compute_stdlib_path, find_executable, find_stdlib,
-                                      resolvedirof)
+from pypy.module.sys.initpath import (compute_stdlib_path, find_executable,
+                                      find_stdlib, resolvedirof,
+                                      pypy_init_home, pypy_init_free)
 from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
+from rpython.rtyper.lltypesystem import rffi
 
 def build_hierarchy(prefix):
     dirname = '%d' % CPYTHON_VERSION[0]
@@ -10,7 +12,7 @@
     b = prefix.join('lib-python', dirname).ensure(dir=1)
     return a, b
 
-def test_find_stdlib(tmpdir, monkeypatch):
+def test_find_stdlib(tmpdir):
     bin_dir = tmpdir.join('bin').ensure(dir=True)
     pypy = bin_dir.join('pypy').ensure(file=True)
     build_hierarchy(tmpdir)
@@ -33,6 +35,14 @@
     path, prefix = find_stdlib(None, str(pypy_sym))
     assert prefix == pypydir
 
+def test_pypy_init_home():
+    p = pypy_init_home()
+    assert p
+    s = rffi.charp2str(p)
+    pypy_init_free(p)
+    print s
+    assert os.path.exists(s)
+
 def test_compute_stdlib_path(tmpdir):
     dirs = build_hierarchy(tmpdir)
     path = compute_stdlib_path(None, str(tmpdir))
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -8,61 +8,61 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 
+ at specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
     w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
                                                        '__getattribute__')
     return w_getattribute
-object_getattribute._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
     w_src, w_setattr = space.lookup_in_type_where(space.w_object,
                                                   '__setattr__')
     return w_setattr
-object_setattr._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
     w_src, w_delattr = space.lookup_in_type_where(space.w_object,
                                                   '__delattr__')
     return w_delattr
-object_delattr._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
     w_src, w_hash = space.lookup_in_type_where(space.w_object,
                                                   '__hash__')
     return w_hash
-object_hash._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
     w_src, w_eq = space.lookup_in_type_where(space.w_type,
                                              '__eq__')
     return w_eq
-type_eq._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
     w_src, w_iter = space.lookup_in_type_where(space.w_list,
                                                '__iter__')
     return w_iter
-list_iter._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
     w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
                                                '__iter__')
     return w_iter
-tuple_iter._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def unicode_iter(space):
     "Utility that returns the app-level descriptor str.__iter__."
     w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
                                                '__iter__')
     return w_iter
-unicode_iter._annspecialcase_ = 'specialize:memo'
 
 def raiseattrerror(space, w_obj, w_name, w_descr=None):
     # space.repr always returns an encodable string.
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -7,6 +7,7 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
+from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 
@@ -329,6 +330,7 @@
                         "unsupported format character %R (%s) at index %d",
                         w_s, hex(ord(c)), self.fmtpos - 1)
 
+        @specialize.argtype(1)
         def std_wp(self, r):
             length = len(r)
             if do_unicode and isinstance(r, str):
@@ -354,7 +356,6 @@
             if padding > 0:
                 result.append_multiple_char(const(' '), padding)
             # add any remaining padding at the right
-        std_wp._annspecialcase_ = 'specialize:argtype(1)'
 
         def std_wp_number(self, r, prefix=''):
             result = self.result
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -369,6 +369,7 @@
         assert isinstance(w_starttype, W_TypeObject)
         return w_type.lookup_starting_at(w_starttype, name)
 
+    @specialize.arg(1)
     def allocate_instance(self, cls, w_subtype):
         """Allocate the memory needed for an instance of an internal or
         user-defined type, without actually __init__ializing the instance."""
@@ -393,7 +394,6 @@
                         "%N.__new__(%N): only for the type %N",
                         w_type, w_subtype, w_type)
         return instance
-    allocate_instance._annspecialcase_ = "specialize:arg(1)"
 
     # two following functions are almost identical, but in fact they
     # have different return type. First one is a resizable list, second
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -46,8 +46,8 @@
 if __name__ == '__main__':
     import py, os
     if '__pypy__' not in sys.builtin_module_names:
-        print 'Call with a pypy interpreter'
-        sys.exit(-1)
+        print >> sys.stderr, 'Call with a pypy interpreter'
+        sys.exit(1)
 
     class Options(object):
         pass
@@ -61,18 +61,26 @@
                                  str(exename))
         basedir = _basedir
     options = Options()
-    print >> sys.stderr, "There should be no failures here"
     failures = create_cffi_import_libraries(exename, options, basedir)
     if len(failures) > 0:
-        print 'failed to build', [f[1] for f in failures]
-        assert False
+        print >> sys.stderr, '*** failed to build the CFFI modules %r' % (
+            [f[1] for f in failures],)
+        print >> sys.stderr, '''
+PyPy can still be used as long as you don't need the corresponding
+modules.  If you do need them, please install the missing headers and
+libraries (see error messages just above) and then re-run the command:
 
-    # monkey patch a failure, just to test
-    print >> sys.stderr, 'This line should be followed by a traceback'
-    for k in cffi_build_scripts:
-        setattr(options, 'no_' + k, True)
-    must_fail = '_missing_build_script.py'
-    assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
-    cffi_build_scripts['should_fail'] = must_fail
-    failures = create_cffi_import_libraries(exename, options, basedir)
-    assert len(failures) == 1
+    %s %s
+''' % (sys.executable, ' '.join(sys.argv))
+        sys.exit(1)
+
+    if len(sys.argv) > 1 and sys.argv[1] == '--test':
+        # monkey patch a failure, just to test
+        print >> sys.stderr, 'This line should be followed by a traceback'
+        for k in cffi_build_scripts:
+            setattr(options, 'no_' + k, True)
+        must_fail = '_missing_build_script.py'
+        assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+        cffi_build_scripts['should_fail'] = must_fail
+        failures = create_cffi_import_libraries(exename, options, basedir)
+        assert len(failures) == 1
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -714,7 +714,7 @@
     def filter(self, builder):
         if not builder.cpu.supports_cond_call_value and \
            self.opnum == rop.COND_CALL_VALUE_I:
-            raise CannotProduceOperation
+            raise test_random.CannotProduceOperation
 
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -138,6 +138,7 @@
             # shadowstack: change 'rpy_fastgil' to 0 (it should be
             # non-zero right now).
             self.change_extra_stack_depth = False
+            # ^^ note that set_extra_stack_depth() in this case is a no-op
             css_value = imm(0)
         else:
             from rpython.memory.gctransform import asmgcroot
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -217,6 +217,12 @@
     func._dont_inline_ = True
     return func
 
+def try_inline(func):
+    """ tell the RPython inline (not the JIT!), to try to inline this function,
+    no matter its size."""
+    func._always_inline_ = 'try'
+    return func
+
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -34,6 +34,7 @@
 else:
     libraries = ['dl']
 
+# this 'eci' is also used in pypy/module/sys/initpath.py
 eci = ExternalCompilationInfo(
     pre_include_bits = pre_include_bits,
     includes = includes,
diff --git a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
@@ -4,17 +4,18 @@
                                 long *current_pos_addr);
 
 
+#define MAX_INLINE_DEPTH  384
+
+
 static long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
-                                             intptr_t ip, int max_depth)
+                                             intptr_t addr, int max_depth)
 {
 #ifdef PYPY_JIT_CODEMAP
     void *codemap;
     long current_pos = 0;
-    intptr_t ident;
+    intptr_t ident, local_stack[MAX_INLINE_DEPTH];
+    long m;
     long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    intptr_t tmp;
 
     codemap = pypy_find_codemap_at_addr(addr, &start_addr);
     if (codemap == NULL || n >= max_depth - 2)
@@ -25,24 +26,29 @@
     // in the middle
     result[n++] = VMPROF_ASSEMBLER_TAG;
     result[n++] = start_addr;
-    start = n;
-    while (n < max_depth) {
+
+    // build the list of code idents corresponding to the current
+    // position inside this particular piece of assembler.  If (very
+    // unlikely) we get more than MAX_INLINE_DEPTH recursion levels
+    // all inlined inside this single piece of assembler, then stop:
+    // there will be some missing frames then.  Otherwise, we need to
+    // first collect 'local_stack' and then write it to 'result' in the
+    // opposite order, stopping at 'max_depth'.  Previous versions of
+    // the code would write the oldest calls and then stop---whereas
+    // what we really need it to write the newest calls and then stop.
+    m = 0;
+    while (m < MAX_INLINE_DEPTH) {
         ident = pypy_yield_codemap_at_addr(codemap, addr, &current_pos);
         if (ident == -1)
             // finish
             break;
         if (ident == 0)
             continue; // not main codemap
+        local_stack[m++] = ident;
+    }
+    while (m > 0 && n < max_depth) {
         result[n++] = VMPROF_JITTED_TAG;
-        result[n++] = ident;
-    }
-    k = 1;
-
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k];
-        result[n - k] = tmp;
-        k += 2;
+        result[n++] = local_stack[--m];
     }
 #endif
     return n;
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config)
+    fetch_translated_config, try_inline)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -483,6 +483,13 @@
         return a, b, c
     assert f._always_inline_ is True
 
+def test_try_inline():
+    @try_inline
+    def f(a, b, c):
+        return a, b, c
+    assert f._always_inline_ == "try"
+
+
 def test_enforceargs_defaults():
     @enforceargs(int, int)
     def f(a, b=40):
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -15,9 +15,10 @@
 static int check_valid(void);
 
 void _RPython_ThreadLocals_Acquire(void) {
-    while (!pypy_lock_test_and_set(&pypy_threadlocal_lock, 1)) {
-        /* busy loop */
-    }
+    long old_value;
+    do {
+        old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1);
+    } while (old_value != 0);   /* busy loop */
     assert(check_valid());
 }
 void _RPython_ThreadLocals_Release(void) {

From pypy.commits at gmail.com  Thu Sep 29 13:33:21 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 10:33:21 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: test,
 fix whoops for not assigning the buf when readonly
Message-ID: <57ed5061.0b3b1c0a.dac6f.e044@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87451:64cae72e4923
Date: 2016-09-29 16:32 +0300
http://bitbucket.org/pypy/pypy/changeset/64cae72e4923/

Log:	test, fix whoops for not assigning the buf when readonly

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -147,6 +147,7 @@
     except ValueError:
         w_s = w_obj.descr_tobytes(space)
         view.c_obj = make_ref(space, w_s)
+        view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
         rffi.setintfield(view, 'c_readonly', 1)
         isstr = True
     fill_Py_buffer(space, w_obj.buf, view)     
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -23,6 +23,8 @@
         assert w_view.c_shape[0] == 5
         assert w_view.c_strides[0] == 1
         assert w_view.c_len == 5
+        o = rffi.charp2str(w_view.c_buf)
+        assert o == 'hello'
 
 class AppTestBufferProtocol(AppTestCpythonExtensionBase):
     def test_buffer_protocol(self):

From pypy.commits at gmail.com  Thu Sep 29 13:33:23 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 10:33:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: make sure exception is raised if call
 returns error value
Message-ID: <57ed5063.516d1c0a.cdf89.cc4b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87452:106a495914a5
Date: 2016-09-29 20:27 +0300
http://bitbucket.org/pypy/pypy/changeset/106a495914a5/

Log:	make sure exception is raised if call returns error value

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -214,7 +214,10 @@
     i = space.int_w(space.index(args_w[0]))
     j = space.int_w(space.index(args_w[1]))
     w_y = args_w[2]
-    return space.wrap(generic_cpy_call(space, func_target, w_self, i, j, w_y))
+    res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
+    if rffi.cast(lltype.Signed, res) == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return res
 
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
@@ -296,7 +299,10 @@
 def wrap_hashfunc(space, w_self, w_args, func):
     func_target = rffi.cast(hashfunc, func)
     check_num_args(space, w_args, 0)
-    return space.wrap(generic_cpy_call(space, func_target, w_self))
+    res = generic_cpy_call(space, func_target, w_self)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return space.wrap(res)
 
 class CPyBuffer(Buffer):
     # Similar to Py_buffer
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -49,6 +49,7 @@
         assert arr.tolist() == [1, 21, 22, 23, 4]
         del arr[slice(1, 3)]
         assert arr.tolist() == [1, 23, 4]
+        raises(TypeError, 'arr[slice(1, 3)] = "abc"')
 
     def test_buffer(self):
         import sys

From pypy.commits at gmail.com  Thu Sep 29 13:33:25 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 10:33:25 -0700 (PDT)
Subject: [pypy-commit] pypy rmod-radd-slots: test,
 add functioning __rmod__ and dummy __radd__ to W_AbstractBytesObject
Message-ID: <57ed5065.8a13c20a.1f2f0.3228@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: rmod-radd-slots
Changeset: r87453:f61471441ebd
Date: 2016-09-29 19:57 +0300
http://bitbucket.org/pypy/pypy/changeset/f61471441ebd/

Log:	test, add functioning __rmod__ and dummy __radd__ to
	W_AbstractBytesObject

	discover pypy cpython incompatibility with u'%s'.__rmod__('a') and
	'%s'.__rmod__(u'a')

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -60,6 +60,10 @@
     def descr_add(self, space, w_other):
         """x.__add__(y) <==> x+y"""
 
+    def descr_radd(self, space, w_other):
+        """x.__radd__(y) <==> y+x"""
+        return space.w_NotImplemented
+
     def descr_contains(self, space, w_sub):
         """x.__contains__(y) <==> y in x"""
 
@@ -602,6 +606,9 @@
     def descr_mod(self, space, w_values):
         return mod_format(space, self, w_values, do_unicode=False)
 
+    def descr_rmod(self, space, w_values):
+        return mod_format(space, w_values, self, do_unicode=False)
+
     def descr_eq(self, space, w_other):
         if space.config.objspace.std.withstrbuf:
             from pypy.objspace.std.strbufobject import W_StringBufferObject
@@ -886,6 +893,7 @@
     __contains__ = interpindirect2app(W_AbstractBytesObject.descr_contains),
 
     __add__ = interpindirect2app(W_AbstractBytesObject.descr_add),
+    __radd__ = interpindirect2app(W_AbstractBytesObject.descr_radd),
     __mul__ = interpindirect2app(W_AbstractBytesObject.descr_mul),
     __rmul__ = interpindirect2app(W_AbstractBytesObject.descr_rmul),
 
@@ -934,6 +942,7 @@
     format = interpindirect2app(W_BytesObject.descr_format),
     __format__ = interpindirect2app(W_BytesObject.descr__format__),
     __mod__ = interpindirect2app(W_BytesObject.descr_mod),
+    __rmod__ = interpindirect2app(W_BytesObject.descr_rmod),
     __getnewargs__ = interpindirect2app(
         W_AbstractBytesObject.descr_getnewargs),
     _formatter_parser = interp2app(W_BytesObject.descr_formatter_parser),
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -92,6 +92,7 @@
         raises(ValueError, 'a%Zb'.__mod__, ((23,),))
 
     def test_format(self):
+        import sys
         raises(TypeError, "foo".__mod__, "bar")
         raises(TypeError, u"foo".__mod__, "bar")
         raises(TypeError, "foo".__mod__, u"bar")
@@ -105,11 +106,28 @@
             assert result == "a foo b"
             assert isinstance(result, cls)
 
+        for format, arg, cls in [("a %s b", "foo", str),
+                                 (u"a %s b", u"foo", unicode)]:
+            raises(TypeError, arg.__rmod__, format[:2])
+            result = arg.__rmod__(format)
+            assert result == "a foo b"
+            assert isinstance(result, cls)
+        for format, arg, cls in [(u"a %s b", "foo", str),
+                                 ("a %s b", u"foo", unicode)]:
+            result = arg.__rmod__(format)
+            if '__pypy__' in sys.builtin_module_names:
+                raises(TypeError, arg.__rmod__, format[:2])
+                assert result == "a foo b"
+                assert isinstance(result, cls)
+            else:
+                assert result is NotImplemented
+
     def test_format_wrongtype(self):
         for int_format in '%d', '%o', '%x':
             exc_info = raises(TypeError, int_format.__mod__, '123')
             expected = int_format + ' format: a number is required, not str'
             assert str(exc_info.value) == expected
+        raises(TypeError, "None % 'abc'") # __rmod__
 
     def test_split(self):
         assert b"".split() == []
@@ -656,7 +674,7 @@
         table = maketrans(b'abc', b'xyz')
         assert b'xyzxyz' == b'xyzabcdef'.translate(table, b'def')
         exc = raises(TypeError, "'xyzabcdef'.translate(memoryview(table), 'def')")
-        assert str(exc.value) == 'expected a character buffer object'
+        assert 'expected a' in str(exc.value)
 
         table = maketrans(b'a', b'A')
         assert b'Abc' == b'abc'.translate(table)
@@ -799,8 +817,18 @@
         raises(TypeError, len, iter(iterable))
 
     def test___radd__(self):
+        import sys
+        raises(TypeError, "None + ''")
+        if '__pypy__' in sys.builtin_module_names:
+            assert 'abc'.__radd__('def') is NotImplemented
+        else:
+            # xxx CPython has no str.__radd__ and no unicode.__radd__
+            raises(AttributeError, "'abc'.__radd__('def')")
+
+
         class Foo(object):
             def __radd__(self, other):
                 return 42
+
         x = Foo()
         assert "hello" + x == 42
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -738,11 +738,21 @@
                             '\x00\x00\x00X\x00\x00\x00Y']
 
     def test_call_special_methods(self):
+        import sys
         # xxx not completely clear if these are implementation details or not
         assert 'abc'.__add__(u'def') == u'abcdef'
         assert u'abc'.__add__(u'def') == u'abcdef'
         assert u'abc'.__add__('def') == u'abcdef'
-        # xxx CPython has no str.__radd__ and no unicode.__radd__
+        assert u'abc'.__rmod__(u'%s') == u'abc'
+        ret = u'abc'.__rmod__('%s')
+        
+        if '__pypy__' in sys.builtin_module_names:
+            assert ret == u'abc'
+            assert u'abc'.__radd__(u'def') is NotImplemented
+        else:
+            assert ret is NotImplemented
+            # xxx CPython has no str.__radd__ and no unicode.__radd__
+            raises(AttributeError, "u'abc'.__radd__(u'def')")
 
     def test_str_unicode_concat_overrides(self):
         "Test from Jython about being bug-compatible with CPython."
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -301,6 +301,9 @@
             raise
         return space.newbool(res)
 
+    def descr_radd(self, space, w_other):
+        return space.w_NotImplemented
+
     def descr_format(self, space, __args__):
         return newformat.format_method(space, self, __args__, is_unicode=True)
 
@@ -316,6 +319,9 @@
     def descr_mod(self, space, w_values):
         return mod_format(space, self, w_values, do_unicode=True)
 
+    def descr_rmod(self, space, w_values):
+        return mod_format(space, w_values, self, do_unicode=True)
+
     def descr_translate(self, space, w_table):
         selfvalue = self._value
         w_sys = space.getbuiltinmodule('sys')
@@ -582,6 +588,9 @@
     def __add__():
         """x.__add__(y) <==> x+y"""
 
+    def __radd__():
+        """x.__radd__(y) <==> y+x"""
+
     def __contains__():
         """x.__contains__(y) <==> y in x"""
 
@@ -630,6 +639,9 @@
     def __mod__():
         """x.__mod__(y) <==> x%y"""
 
+    def __rmod__():
+        """x.__rmod__(y) <==> y%x"""
+
     def __mul__():
         """x.__mul__(n) <==> x*n"""
 
@@ -998,6 +1010,8 @@
 
     __add__ = interp2app(W_UnicodeObject.descr_add,
                          doc=UnicodeDocstrings.__add__.__doc__),
+    __radd__ = interp2app(W_UnicodeObject.descr_radd,
+                         doc=UnicodeDocstrings.__radd__.__doc__),
     __mul__ = interp2app(W_UnicodeObject.descr_mul,
                          doc=UnicodeDocstrings.__mul__.__doc__),
     __rmul__ = interp2app(W_UnicodeObject.descr_mul,
@@ -1093,6 +1107,8 @@
                             doc=UnicodeDocstrings.__format__.__doc__),
     __mod__ = interp2app(W_UnicodeObject.descr_mod,
                          doc=UnicodeDocstrings.__mod__.__doc__),
+    __rmod__ = interp2app(W_UnicodeObject.descr_rmod,
+                         doc=UnicodeDocstrings.__rmod__.__doc__),
     __getnewargs__ = interp2app(W_UnicodeObject.descr_getnewargs,
                                 doc=UnicodeDocstrings.__getnewargs__.__doc__),
     _formatter_parser = interp2app(W_UnicodeObject.descr_formatter_parser),

From pypy.commits at gmail.com  Thu Sep 29 14:19:36 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 11:19:36 -0700 (PDT)
Subject: [pypy-commit] pypy _warnings: add test that passes untranslated,
 fails w/pypy -A (TypeError: expected a readable buffer object)
Message-ID: <57ed5b38.9c4a1c0a.9aa57.01ac@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: _warnings
Changeset: r87454:f31d6c388d20
Date: 2016-09-29 21:17 +0300
http://bitbucket.org/pypy/pypy/changeset/f31d6c388d20/

Log:	add test that passes untranslated, fails w/pypy -A (TypeError:
	expected a readable buffer object)

diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -11,6 +11,7 @@
         import _warnings
         _warnings.warn("some message", DeprecationWarning)
         _warnings.warn("some message", Warning)
+        _warnings.warn(("some message",1), Warning)
 
     def test_lineno(self):
         import warnings, _warnings, sys
@@ -40,7 +41,10 @@
     def test_show_source_line(self):
         import warnings
         import sys, StringIO
-        from test.warning_tests import inner
+        try:
+            from test.warning_tests import inner
+        except ImportError:
+            skip('no test, -A on cpython?')
         # With showarning() missing, make sure that output is okay.
         del warnings.showwarning
 

From pypy.commits at gmail.com  Thu Sep 29 14:38:53 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 29 Sep 2016 11:38:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: Skip in test_app_main instead of failing
 if there is no python3.3
Message-ID: <57ed5fbd.07bd1c0a.b6264.050e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87455:a71ca3c65328
Date: 2016-09-29 19:38 +0100
http://bitbucket.org/pypy/pypy/changeset/a71ca3c65328/

Log:	Skip in test_app_main instead of failing if there is no python3.3

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -17,8 +17,7 @@
 def get_python3():
     if PYTHON3:
         return PYTHON3
-    import py.test
-    py.test.fail("Test requires %r (not found in PATH) or a PYTHON3 "
+    py.test.skip("Test requires %r (not found in PATH) or a PYTHON3 "
                  "environment variable set" % (LOOK_FOR_PYTHON3,))
 
 _counter = 0
@@ -51,7 +50,8 @@
     # return relative path for testing purposes
     return py.path.local().bestrelpath(pdir)
 
-def pytest_funcarg__demo_script(request):
+ at py.test.fixture
+def demo_script():
     return getscript("""
         print('hello')
         print('Name:', __name__)
@@ -64,7 +64,8 @@
         myvalue = 6*7
     """)
 
-def pytest_funcarg__crashing_demo_script(request):
+ at py.test.fixture
+def crashing_demo_script():
     return getscript("""
         print('Hello2')
         myvalue2 = 11

From pypy.commits at gmail.com  Thu Sep 29 17:05:09 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 29 Sep 2016 14:05:09 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Attempted support for unpacking
 constant pointers
Message-ID: <57ed8205.4398c20a.d6ca0.8e2d@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87456:48e89a5c1a07
Date: 2016-09-29 17:04 -0400
http://bitbucket.org/pypy/pypy/changeset/48e89a5c1a07/

Log:	Attempted support for unpacking constant pointers

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -328,7 +328,8 @@
                         guard.rd_resume_position = patchguardop.rd_resume_position
                         guard.setdescr(compile.ResumeAtPositionDescr())
                     self.send_extra_operation(guard)
-            except VirtualStatesCantMatch:
+            except VirtualStatesCantMatch as e:
+                print "*** failing with: ", e.msg
                 continue
 
             # When force_boxes == True, creating the virtual args can fail when
@@ -337,8 +338,9 @@
             try:
                 args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
                     args, self.optimizer, force_boxes=force_boxes)
-            except VirtualStatesCantMatch:
+            except VirtualStatesCantMatch as e:
                 assert force_boxes
+                print "*** failing with: ", e.msg
                 virtual_state = self.get_virtual_state(args)
                 continue
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -219,7 +219,7 @@
     @specialize.argtype(4)
     def getfield(self, box, info, descr, optimizer):
         assert isinstance(info, AbstractStructPtrInfo)
-        field = info._fields[descr.get_index()]
+        field = info.getfield(descr, optimizer)
         if field is not None:
             return field
         opnum = rop.getfield_for_descr(descr)
@@ -229,12 +229,18 @@
         else:
             from rpython.jit.metainterp.optimizeopt import Optimizer
             assert isinstance(optimizer, Optimizer)
-            optimizer.emit_operation(getfield)
-        info._fields[descr.get_index()] = getfield
+            optimizer.send_extra_operation(getfield)
+        info.setfield(descr, None, getfield)
         return getfield
 
     def make_virtual_copy(self, box, info, optimizer):
         optvirtualize = optimizer.optvirtualize
+
+        if isinstance(box, ConstPtr):
+            # Wrap const pointers in SAME_AS_R operation to cloning via
+            # make_virtual, which expects a ResOperation with an opnum.
+            box = ResOperation(rop.SAME_AS_R, [box])
+
         opinfo = optvirtualize.make_virtual(self.known_class, box, self.typedescr)
         for i in range(len(info._fields)):
             descr = self.fielddescrs[i]
@@ -250,6 +256,9 @@
             return AbstractVirtualStructStateInfo.enum_forced_boxes(
                     self, boxes, box, optimizer, force_boxes)
 
+        if isinstance(info, ConstPtrInfo):
+            optheap = optimizer.optimizer.optheap
+            info = info._get_info(self.typedescr, optheap)
         assert isinstance(info, AbstractStructPtrInfo)
 
         # TODO: Do we need to create a new object via NEW_WITH_VTABLE, or will the
@@ -273,7 +282,7 @@
         assert isinstance(other, NotVirtualStateInfoPtr)
 
         if not self.typedescr.is_value_class():
-            raise VirtualStatesCantMatch("different kinds of structs")
+            raise VirtualStatesCantMatch("_generate_guards_non_virtual: different kinds of structs")
 
         # raise VirtualStatesCantMatch("different kinds of structs")
         # TODO: Probably should rename state.extra_guards to extra_ops
@@ -290,7 +299,7 @@
             extra_guards.append(op)
         elif other.level == LEVEL_KNOWNCLASS:
             if not self.known_class.same_constant(other.known_class):
-                raise VirtualStatesCantMatch("classes don't match")
+                raise VirtualStatesCantMatch("_generate_guards_non_virtual: classes don't match")
         else:
             assert other.level == LEVEL_CONSTANT
             known = self.known_class
@@ -298,7 +307,7 @@
             if const.nonnull() and known.same_constant(cpu.ts.cls_of_box(const)):
                 pass
             else:
-                raise VirtualStatesCantMatch("classes don't match")
+                raise VirtualStatesCantMatch("_generate_guards_non_virtual: classes don't match")
 
         # Things to do...
         # 1. Generate new_with_vtable operation to allocate the new virtual object
@@ -308,11 +317,13 @@
         #     non-virtual object which we are promoting to a virtual. How do we
         #     generate this new virtual state so we can operate recursively)
 
+        optimizer = state.optimizer
         if runtime_box is not None:
-            opinfo = state.optimizer.getptrinfo(box)
-            if opinfo is not None and not isinstance(opinfo, AbstractStructPtrInfo):
-                assert isinstance(opinfo, ConstPtrInfo)
-                raise VirtualStatesCantMatch("cannot yet handle const pointers")
+            opinfo = optimizer.getptrinfo(box)
+            if isinstance(opinfo, ConstPtrInfo):
+                optheap = optimizer.optheap
+                opinfo = opinfo._get_info(self.typedescr, optheap)
+            assert opinfo is None or isinstance(opinfo, AbstractStructPtrInfo)
         else:
             opinfo = None
 
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1087,33 +1087,39 @@
         self.meta_interp(f, [])
 
     def test_avoid_preamble_4(self):
-        driver = JitDriver(greens=[], reds=['i', 'val'])
+        driver = JitDriver(greens=[], reds=['i', 'acc', 'val'])
         class X(object):
             _value_class_ = True
             def __init__(self, v):
                 self.v = v
 
         const = X(5)
+        arr = [None]
         def f():
             # Prevent all retracing of side exits. Ensures that the unroll
             # optimizer will attempt to jump to either the preamble or loop.
             set_param(driver, 'retrace_limit', -1)
             set_param(driver, 'threshold', 1)
-            val   = X(0)
-            i     = 0
+            val = X(0)
+            acc = 0
+            i = 0
             while i < 17:
-                driver.can_enter_jit(i=i, val=val)
-                driver.jit_merge_point(i=i, val=val)
+                driver.can_enter_jit(i=i, acc=acc, val=val)
+                driver.jit_merge_point(i=i, acc=acc, val=val)
                 # Logical & rather than comparison to confuse range analysis.
                 # Test only succeeds on the first 2 iterations
+                arr[0] = val
                 if i & -2 == 0:
                     val = X(i)
                 else:
                     val = const
+                acc += val.v
                 i += 1
             return 0
 
         self.meta_interp(f, [])
+        loop = get_stats().loops[0]
+        import pdb; pdb.set_trace()
 
     def test_aliased_virtual_states(self):
         # All cases are covered when forcing one component of the virtual state
@@ -1150,6 +1156,30 @@
         self.meta_interp(f, [])
         # assert did not crash
 
+    def test_double_allocation(self):
+        driver = JitDriver(greens=[], reds=['i'])
+
+        class X(object):
+            _value_class_ = True
+            def __init__(self, v):
+                self.v = v
+
+        array = [None, None]
+        def f():
+            # Prevent all retracing of side exits. Ensures that the unroll
+            # optimizer will attempt to jump to either the preamble or loop.
+            set_param(driver, 'retrace_limit', -1)
+            set_param(driver, 'threshold', 1)
+            i = 0
+            while i < 17:
+                driver.can_enter_jit(i=i)
+                driver.jit_merge_point(i=i)
+                array[0] = X(i)
+                array[1] = X(i)
+                i += 1
+            return 0
+
+        self.meta_interp(f, [])
 
 class VirtualMiscTests:
     def test_multiple_equal_virtuals(self):

From pypy.commits at gmail.com  Thu Sep 29 21:49:02 2016
From: pypy.commits at gmail.com (sbauman)
Date: Thu, 29 Sep 2016 18:49:02 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Fixes for unpacking const pointers
Message-ID: <57edc48e.4a301c0a.24b9e.73dc@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87457:d227639f3fa6
Date: 2016-09-29 21:48 -0400
http://bitbucket.org/pypy/pypy/changeset/d227639f3fa6/

Log:	Fixes for unpacking const pointers

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -329,7 +329,6 @@
                         guard.setdescr(compile.ResumeAtPositionDescr())
                     self.send_extra_operation(guard)
             except VirtualStatesCantMatch as e:
-                print "*** failing with: ", e.msg
                 continue
 
             # When force_boxes == True, creating the virtual args can fail when
@@ -340,7 +339,6 @@
                     args, self.optimizer, force_boxes=force_boxes)
             except VirtualStatesCantMatch as e:
                 assert force_boxes
-                print "*** failing with: ", e.msg
                 virtual_state = self.get_virtual_state(args)
                 continue
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,7 +1,7 @@
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.info import ArrayPtrInfo,\
-     ArrayStructInfo, AbstractStructPtrInfo, ConstPtrInfo
+     ArrayStructInfo, AbstractStructPtrInfo, ConstPtrInfo, InstancePtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound, IntLowerBound, IntUnbounded
 from rpython.jit.metainterp.resoperation import rop, ResOperation, \
@@ -216,32 +216,27 @@
         return (isinstance(other, VirtualStateInfo) and
                 self.typedescr is other.typedescr)
 
-    @specialize.argtype(4)
-    def getfield(self, box, info, descr, optimizer):
+    def getfield(self, box, info, descr, optimizer, send_to=None):
         assert isinstance(info, AbstractStructPtrInfo)
         field = info.getfield(descr, optimizer)
         if field is not None:
             return field
         opnum = rop.getfield_for_descr(descr)
         getfield = ResOperation(opnum, [box], descr=descr)
-        if isinstance(optimizer, list):
-            optimizer.append(getfield)
+        if send_to is not None:
+            send_to.append(getfield)
         else:
-            from rpython.jit.metainterp.optimizeopt import Optimizer
-            assert isinstance(optimizer, Optimizer)
             optimizer.send_extra_operation(getfield)
-        info.setfield(descr, None, getfield)
+        struct = optimizer.get_box_replacement(box)
+        info.setfield(descr, struct, getfield)
         return getfield
 
     def make_virtual_copy(self, box, info, optimizer):
-        optvirtualize = optimizer.optvirtualize
+        newop = ResOperation(rop.NEW_WITH_VTABLE, [], descr=self.typedescr)
+        optimizer.send_extra_operation(newop)
+        opinfo = optimizer.getptrinfo(newop)
+        assert isinstance(opinfo, InstancePtrInfo)
 
-        if isinstance(box, ConstPtr):
-            # Wrap const pointers in SAME_AS_R operation to cloning via
-            # make_virtual, which expects a ResOperation with an opnum.
-            box = ResOperation(rop.SAME_AS_R, [box])
-
-        opinfo = optvirtualize.make_virtual(self.known_class, box, self.typedescr)
         for i in range(len(info._fields)):
             descr = self.fielddescrs[i]
             opinfo._fields[i] = self.getfield(box, info, descr, optimizer)
@@ -333,7 +328,8 @@
         # AbstractVirtualStateInfo._generate_guards
         for i, descr in enumerate(self.fielddescrs):
             if runtime_box is not None and opinfo is not None:
-                fieldbox = self.getfield(box, opinfo, descr, extra_guards)
+                fieldbox = self.getfield(box, opinfo, descr, optimizer,
+                                         send_to=extra_guards)
                 fieldbox_runtime = state.get_runtime_field(runtime_box, descr)
             else:
                 opnum = rop.getfield_for_descr(descr)
diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1045,6 +1045,8 @@
             return 0
 
         self.meta_interp(f, [])
+        loop = get_stats().loops[0]
+        # import pdb; pdb.set_trace()
 
     def test_avoid_preamble_3(self):
         driver = JitDriver(greens=[], reds=['i', 'val'])
@@ -1085,6 +1087,8 @@
             return 0
 
         self.meta_interp(f, [])
+        loop = get_stats().loops[0]
+        # import pdb; pdb.set_trace()
 
     def test_avoid_preamble_4(self):
         driver = JitDriver(greens=[], reds=['i', 'acc', 'val'])
@@ -1119,7 +1123,7 @@
 
         self.meta_interp(f, [])
         loop = get_stats().loops[0]
-        import pdb; pdb.set_trace()
+        # import pdb; pdb.set_trace()
 
     def test_aliased_virtual_states(self):
         # All cases are covered when forcing one component of the virtual state

From pypy.commits at gmail.com  Thu Sep 29 23:52:14 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 20:52:14 -0700 (PDT)
Subject: [pypy-commit] pypy rmod-radd-slots: fix translation
Message-ID: <57ede16e.8a13c20a.1f2f0.d1d7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: rmod-radd-slots
Changeset: r87458:248a73d17a0e
Date: 2016-09-30 06:51 +0300
http://bitbucket.org/pypy/pypy/changeset/248a73d17a0e/

Log:	fix translation

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -217,7 +217,6 @@
     res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return res
 
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)

From pypy.commits at gmail.com  Thu Sep 29 23:54:07 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 20:54:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix translation
Message-ID: <57ede1df.6773c20a.64a1f.df70@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87459:aa7addfd4bd1
Date: 2016-09-30 06:53 +0300
http://bitbucket.org/pypy/pypy/changeset/aa7addfd4bd1/

Log:	fix translation

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -217,7 +217,6 @@
     res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return res
 
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)

From pypy.commits at gmail.com  Fri Sep 30 02:06:06 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 29 Sep 2016 23:06:06 -0700 (PDT)
Subject: [pypy-commit] pypy py35-getbuiltin: Experiment with
 space.getbuiltinmodule(): it should not interact with sys.modules.
Message-ID: <57ee00ce.c918c20a.3f6cf.f62e@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py35-getbuiltin
Changeset: r87460:5a730fdf1043
Date: 2016-09-26 21:10 +0200
http://bitbucket.org/pypy/pypy/changeset/5a730fdf1043/

Log:	Experiment with space.getbuiltinmodule(): it should not interact
	with sys.modules. Should fix test___all__ (which sets
	sys.modules['_frozen_importlib']=None)


From pypy.commits at gmail.com  Fri Sep 30 02:06:08 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 29 Sep 2016 23:06:08 -0700 (PDT)
Subject: [pypy-commit] pypy py35-getbuiltin: Progress:
 space.getbuiltinmodule() does not use or update sys.modules
Message-ID: <57ee00d0.516d1c0a.cc065.ad02@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py35-getbuiltin
Changeset: r87461:ecaeac990f09
Date: 2016-09-30 08:02 +0200
http://bitbucket.org/pypy/pypy/changeset/ecaeac990f09/

Log:	Progress: space.getbuiltinmodule() does not use or update
	sys.modules

diff --git a/lib-python/3/importlib/_bootstrap.py b/lib-python/3/importlib/_bootstrap.py
--- a/lib-python/3/importlib/_bootstrap.py
+++ b/lib-python/3/importlib/_bootstrap.py
@@ -1137,6 +1137,6 @@
     sys.meta_path.append(FrozenImporter)
 
     global _bootstrap_external
-    _frozen_importlib_external = sys.modules['_frozen_importlib_external']
+    import _frozen_importlib_external
     _bootstrap_external = _frozen_importlib_external
     _frozen_importlib_external._install(sys.modules[__name__])
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -491,14 +491,6 @@
 
     def getbuiltinmodule(self, name, force_init=False, reuse=True):
         w_name = self.wrap(name)
-        w_modules = self.sys.get('modules')
-        if not force_init:
-            assert reuse
-            try:
-                return self.getitem(w_modules, w_name)
-            except OperationError as e:
-                if not e.match(self, self.w_KeyError):
-                    raise
 
         # If the module is a builtin but not yet imported,
         # retrieve it and initialize it
@@ -509,23 +501,18 @@
                         "getbuiltinmodule() called with non-builtin module %s",
                         name)
 
-        # Add the module to sys.modules and initialize the module. The
-        # order is important to avoid recursions.
         from pypy.interpreter.module import Module
         if isinstance(w_mod, Module):
             if not reuse and w_mod.startup_called:
                 # create a copy of the module.  (see issue1514) eventlet
                 # patcher relies on this behaviour.
                 w_mod2 = self.wrap(Module(self, w_name))
-                self.setitem(w_modules, w_name, w_mod2)
                 w_mod.getdict(self)  # unlazy w_initialdict
                 self.call_method(w_mod2.getdict(self), 'update',
                                  w_mod.w_initialdict)
                 return w_mod2
-            self.setitem(w_modules, w_name, w_mod)
-            w_mod.init(self)
-        else:
-            self.setitem(w_modules, w_name, w_mod)
+            if not w_mod.startup_called or force_init:
+                w_mod.init(self)
         return w_mod
 
     def get_builtinmodule_to_install(self):
@@ -651,10 +638,10 @@
         if self.config.objspace.usemodules.cpyext:
             from pypy.module.cpyext.state import State
             self.fromcache(State).build_api(self)
-        self.getbuiltinmodule('sys')
-        self.getbuiltinmodule('_imp')
-        self.getbuiltinmodule('_frozen_importlib')
-        self.getbuiltinmodule('builtins')
+        w_modules = self.sys.get('modules')
+        for name in ('sys', '_imp', '_frozen_importlib', 'builtins'):
+            mod = self.getbuiltinmodule(name)
+            self.setitem(w_modules, mod.w_name, mod)
         for mod in self.builtin_modules.values():
             mod.setup_after_space_initialization()
 
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -77,6 +77,7 @@
 
     def startup(self, space):
         """Copy our __import__ to builtins."""
+        space.setitem(space.sys.get('modules'), self.w_name, self)
         w_install = self.getdictvalue(space, '_install')
         space.call_function(w_install,
                             space.getbuiltinmodule('sys'),
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -70,7 +70,6 @@
 @unwrap_spec(modulename='str0', level=int)
 def importhook(space, modulename, w_globals=None, w_locals=None, w_fromlist=None, level=0):
     # A minimal version, that can only import builtin and lib_pypy modules!
-    assert w_locals is w_globals
     assert level == 0
 
     w_mod = check_sys_modules_w(space, modulename)

From pypy.commits at gmail.com  Fri Sep 30 02:06:09 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 29 Sep 2016 23:06:09 -0700 (PDT)
Subject: [pypy-commit] pypy py35-getbuiltin: Don't 'inject' the .pyc magic
 number from the outside, but import it from __pypy__.
Message-ID: <57ee00d1.c62f1c0a.50285.abfe@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py35-getbuiltin
Changeset: r87462:e5b6f76186cf
Date: 2016-09-29 23:17 +0200
http://bitbucket.org/pypy/pypy/changeset/e5b6f76186cf/

Log:	Don't 'inject' the .pyc magic number from the outside, but import it
	from __pypy__. This fixes the reload of test_importlib.util.

diff --git a/lib-python/3/importlib/_bootstrap_external.py b/lib-python/3/importlib/_bootstrap_external.py
--- a/lib-python/3/importlib/_bootstrap_external.py
+++ b/lib-python/3/importlib/_bootstrap_external.py
@@ -232,11 +232,10 @@
 #
 # PyPy change: the MAGIC_NUMBER is defined in
 # pypy/interpreter/pycode.py, 'default_magic'.  It is based on a number
-# different than CPython's, always < 3000.  We get the 4-bytes string
-# here via a hack: MAGIC_NUMBER is set in the module from
-# module/_frozen_importlib/__init__.py before the module is executed.
+# different than CPython's, always < 3000.
+from __pypy__ import PYC_MAGIC as _RAW_MAGIC_NUMBER
 
-_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
+MAGIC_NUMBER = _RAW_MAGIC_NUMBER.to_bytes(4, 'little')
 
 _PYCACHE = '__pycache__'
 _OPT = 'opt-'
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -570,10 +570,16 @@
         w_builtin.install()
         self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin)
 
+        from pypy.module.__pypy__ import Module
+        w_name = self.wrap('__pypy__')
+        mod = Module(self, w_name)
+        mod.install()
+
         # exceptions was bootstrapped as '__exceptions__' but still
         # lives in pypy/module/exceptions, we rename it below for
         # sys.builtin_module_names
-        bootstrap_modules = set(('sys', 'imp', 'builtins', 'exceptions'))
+        bootstrap_modules = set(('sys', 'imp', 'builtins',
+                                 'exceptions', '__pypy__'))
         installed_builtin_modules = list(bootstrap_modules)
 
         exception_types_w = self.export_builtin_exceptions()
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -1,7 +1,6 @@
 import sys
 
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module.imp.importing import get_pyc_magic
 
 class BuildersModule(MixedModule):
     appleveldefs = {}
@@ -91,6 +90,7 @@
         '_promote'                   : 'interp_magic._promote',
         'normalize_exc'             : 'interp_magic.normalize_exc',
         'StdErrPrinter'             : 'interp_stderrprinter.W_StdErrPrinter',
+        'PYC_MAGIC'                 : 'space.wrap(interp_magic.default_magic)',
     }
 
     submodules = {
@@ -110,8 +110,6 @@
                                  'interp_magic.reset_method_cache_counter')
             self.extra_interpdef('mapdict_cache_counter',
                                  'interp_magic.mapdict_cache_counter')
-        PYC_MAGIC = get_pyc_magic(self.space)
-        self.extra_interpdef('PYC_MAGIC', 'space.wrap(%d)' % PYC_MAGIC)
         try:
             from rpython.jit.backend import detect_cpu
             model = detect_cpu.autodetect()
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.gateway import WrappedDefault, unwrap_spec
 from pypy.interpreter.pycode import CodeHookCache
+from pypy.interpreter.pycode import default_magic
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.objectmodel import we_are_translated
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -34,9 +34,6 @@
         space = self.space
         # "import importlib/_boostrap_external.py"
         w_mod = Module(space, space.wrap("_frozen_importlib_external"))
-        # hack: inject MAGIC_NUMBER into this module's dict
-        space.setattr(w_mod, space.wrap('MAGIC_NUMBER'),
-                      interp_imp.get_magic(space))
         self._compile_bootstrap_module(
             space, '_bootstrap_external', w_mod.w_name, w_mod.w_dict)
         space.sys.setmodule(w_mod)

From pypy.commits at gmail.com  Fri Sep 30 02:37:50 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 23:37:50 -0700 (PDT)
Subject: [pypy-commit] pypy _warnings: copy cpython behaviour
Message-ID: <57ee083e.217ac20a.73544.41f7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: _warnings
Changeset: r87463:262030adf8f1
Date: 2016-09-30 09:37 +0300
http://bitbucket.org/pypy/pypy/changeset/262030adf8f1/

Log:	copy cpython behaviour

diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -248,6 +248,10 @@
     if space.isinstance_w(w_message, space.w_Warning):
         w_text = space.str(w_message)
         w_category = space.type(w_message)
+    elif (not space.isinstance_w(w_message, space.w_unicode) or
+          not space.isinstance_w(w_message, space.w_str)):
+        w_text = space.str(w_message)
+        w_message = space.call_function(w_category, w_message)
     else:
         w_text = w_message
         w_message = space.call_function(w_category, w_message)

From pypy.commits at gmail.com  Fri Sep 30 02:46:12 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 23:46:12 -0700 (PDT)
Subject: [pypy-commit] pypy _warnings: merge default into branch
Message-ID: <57ee0a34.cdd71c0a.f48d0.6841@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: _warnings
Changeset: r87464:53649ba55ddf
Date: 2016-09-30 09:45 +0300
http://bitbucket.org/pypy/pypy/changeset/53649ba55ddf/

Log:	merge default into branch

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -217,7 +217,6 @@
     res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return res
 
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)

From pypy.commits at gmail.com  Fri Sep 30 02:50:46 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Sep 2016 23:50:46 -0700 (PDT)
Subject: [pypy-commit] pypy stricter-strip: test,
 fix for argument to strip() - must be None, str,
 unicode (cpython compatibility)
Message-ID: <57ee0b46.cacb1c0a.354b8.b6c9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: stricter-strip
Changeset: r87465:244251f50a52
Date: 2016-09-30 09:49 +0300
http://bitbucket.org/pypy/pypy/changeset/244251f50a52/

Log:	test, fix for argument to strip() - must be None, str, unicode
	(cpython compatibility)

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -82,7 +82,8 @@
         return False
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
+        # bytearray does not enforce the strict restriction (on strip at least)
         return space.buffer_w(w_other, space.BUF_SIMPLE).as_str()
 
     def _chr(self, char):
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -501,7 +501,10 @@
                 isinstance(w_other, W_UnicodeObject))
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
+        if strict and not space.isinstance_w(w_other, space.w_str):
+            raise oefmt(space.w_TypeError,
+                "%s arg must be None, str or unicode", strict)
         try:
             return space.str_w(w_other)
         except OperationError as e:
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -652,7 +652,7 @@
     def _strip(self, space, w_chars, left, right):
         "internal function called by str_xstrip methods"
         value = self._val(space)
-        chars = self._op_val(space, w_chars)
+        chars = self._op_val(space, w_chars, strict='strip')
 
         lpos = 0
         rpos = len(value)
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -128,7 +128,8 @@
 
         b = bytearray('mississippi')
 
-        for strip_type in str, memoryview:
+        for strip_type in str, memoryview, buffer:
+            print 'strip_type', strip_type
             assert b.strip(strip_type('i')) == 'mississipp'
             assert b.strip(strip_type('m')) == 'ississippi'
             assert b.strip(strip_type('pi')) == 'mississ'
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -251,6 +251,12 @@
         assert b'xyzzyhelloxyzzy'.strip(b'xyz') == b'hello'
         assert b'xyzzyhelloxyzzy'.lstrip(b'xyz') == b'helloxyzzy'
         assert b'xyzzyhelloxyzzy'.rstrip(b'xyz') == b'xyzzyhello'
+        exc = raises(TypeError, s.strip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
+        exc = raises(TypeError, s.rstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
+        exc = raises(TypeError, s.lstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
 
     def test_zfill(self):
         assert b'123'.zfill(2) == b'123'
@@ -656,7 +662,7 @@
         table = maketrans(b'abc', b'xyz')
         assert b'xyzxyz' == b'xyzabcdef'.translate(table, b'def')
         exc = raises(TypeError, "'xyzabcdef'.translate(memoryview(table), 'def')")
-        assert str(exc.value) == 'expected a character buffer object'
+        assert 'expected a' in str(exc.value)
 
         table = maketrans(b'a', b'A')
         assert b'Abc' == b'abc'.translate(table)
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -332,6 +332,12 @@
         assert u'xyzzyhelloxyzzy'.strip(u'xyz') == u'hello'
         assert u'xyzzyhelloxyzzy'.lstrip('xyz') == u'helloxyzzy'
         assert u'xyzzyhelloxyzzy'.rstrip(u'xyz') == u'xyzzyhello'
+        exc = raises(TypeError, s.strip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
+        exc = raises(TypeError, s.rstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
+        exc = raises(TypeError, s.lstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
 
     def test_strip_str_unicode(self):
         x = "--abc--".strip(u"-")
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -124,11 +124,14 @@
         return True
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
         if isinstance(w_other, W_UnicodeObject):
             return w_other._value
         if space.isinstance_w(w_other, space.w_str):
             return unicode_from_string(space, w_other)._value
+        if strict:
+            raise oefmt(space.w_TypeError,
+                "%s arg must be None, unicode or str", strict)
         return unicode_from_encoded_object(
             space, w_other, None, "strict")._value
 

From pypy.commits at gmail.com  Fri Sep 30 05:17:19 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 30 Sep 2016 02:17:19 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: a new job for me
Message-ID: <57ee2d9f.94071c0a.8a169.f749@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5720:46b5b8cf9f44
Date: 2016-09-30 10:56 +0200
http://bitbucket.org/pypy/extradoc/changeset/46b5b8cf9f44/

Log:	a new job for me

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -5,7 +5,8 @@
 In-progress ("Lock" section)
 ----------------------------
 
-* richard: bz2, lzma, ... changes (cpython issue 15955)
+* richard: bz2, lzma, ... changes (cpython issue 15955) (DONE)
+* richard: threads do not seem to join (e.g. lib-python/3/test/test_bz2.py)
 
 * arigo: faulthandler module
 
@@ -59,9 +60,9 @@
 
 * bytes % args, bytearray % args: PEP 461
 
-* New bytes.hex(), bytearray.hex() and memoryview.hex() methods.
+* New bytes.hex(), bytearray.hex() and memoryview.hex() methods. (DONE)
 
-* memoryview now supports tuple indexing
+* memoryview now supports tuple indexing (DONE)
 
 * A new RecursionError exception is now raised when maximum recursion
   depth is reached. (DONE)

From pypy.commits at gmail.com  Fri Sep 30 05:17:21 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 30 Sep 2016 02:17:21 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: add new done section and move an
 entry there
Message-ID: <57ee2da1.27f0c20a.bf33e.c811@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5721:f1ce65dce7f3
Date: 2016-09-30 11:17 +0200
http://bitbucket.org/pypy/extradoc/changeset/f1ce65dce7f3/

Log:	add new done section and move an entry there

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -5,7 +5,6 @@
 In-progress ("Lock" section)
 ----------------------------
 
-* richard: bz2, lzma, ... changes (cpython issue 15955) (DONE)
 * richard: threads do not seem to join (e.g. lib-python/3/test/test_bz2.py)
 
 * arigo: faulthandler module
@@ -43,6 +42,7 @@
 * 'import stackless' fails
 
 
+
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------
 
@@ -79,3 +79,8 @@
 * enum: Support for enumeration types (PEP 435). (PURELY-APPLEVEL)
 
 * pathlib: Object-oriented filesystem paths (PEP 428). (PURELY-APPLEVEL)
+
+Done not formally in the Milestone
+----------------------------------
+
+* richard: bz2, lzma, ... changes (cpython issue 15955) (DONE)

From pypy.commits at gmail.com  Fri Sep 30 07:50:32 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 30 Sep 2016 04:50:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-set-sentinel: extended doc,
 renamed function, starting to implement the function fully
Message-ID: <57ee5188.516d1c0a.cc065.38b0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-set-sentinel
Changeset: r87466:78cd6d8efe3b
Date: 2016-09-30 13:49 +0200
http://bitbucket.org/pypy/pypy/changeset/78cd6d8efe3b/

Log:	extended doc, renamed function, starting to implement the function
	fully

diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py
--- a/pypy/module/thread/__init__.py
+++ b/pypy/module/thread/__init__.py
@@ -21,7 +21,7 @@
         'allocate':               'os_lock.allocate_lock',  # obsolete synonym
         'LockType':               'os_lock.Lock',
         'RLock':                  'os_lock.W_RLock',
-        '_set_sentinel':          'os_lock.set_sentinel',
+        '_set_sentinel':          'os_lock._set_sentinel',
         '_local':                 'os_local.Local',
         'TIMEOUT_MAX':            'space.wrap(float(os_lock.TIMEOUT_MAX // 1000000))',
         'error':                  'space.fromcache(error.Cache).w_error',
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -145,8 +145,12 @@
     return space.wrap(Lock(space))
 
 def set_sentinel(space):
-    """Set a sentinel lock that will be released when the current thread
-state is finalized (after it is untied from the interpreter)."""
+    """_set_sentinel() -> lock
+
+    Set a sentinel lock that will be released when the current thread 
+    state is finalized (after it is untied from the interpreter).
+
+    This is a private API for the threading module."""
     lock = allocate_lock(space)
     return lock
 

From pypy.commits at gmail.com  Fri Sep 30 08:40:53 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 30 Sep 2016 05:40:53 -0700 (PDT)
Subject: [pypy-commit] pypy _warnings: close branch to be merged
Message-ID: <57ee5d55.c4f6c20a.96550.9de3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: _warnings
Changeset: r87469:b828095f296c
Date: 2016-09-30 13:04 +0300
http://bitbucket.org/pypy/pypy/changeset/b828095f296c/

Log:	close branch to be merged


From pypy.commits at gmail.com  Fri Sep 30 08:40:49 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 30 Sep 2016 05:40:49 -0700 (PDT)
Subject: [pypy-commit] pypy stricter-strip: close branch to be merged
Message-ID: <57ee5d51.e50fc20a.c9259.aab2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: stricter-strip
Changeset: r87467:d8fddc2435d0
Date: 2016-09-30 13:02 +0300
http://bitbucket.org/pypy/pypy/changeset/d8fddc2435d0/

Log:	close branch to be merged


From pypy.commits at gmail.com  Fri Sep 30 08:40:54 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 30 Sep 2016 05:40:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge branch to convert message to str
 in warn(message, ...) like cpython
Message-ID: <57ee5d56.8aabc20a.33724.9433@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87470:affefa83bb1b
Date: 2016-09-30 13:05 +0300
http://bitbucket.org/pypy/pypy/changeset/affefa83bb1b/

Log:	merge branch to convert message to str in warn(message, ...) like
	cpython

diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -248,6 +248,10 @@
     if space.isinstance_w(w_message, space.w_Warning):
         w_text = space.str(w_message)
         w_category = space.type(w_message)
+    elif (not space.isinstance_w(w_message, space.w_unicode) or
+          not space.isinstance_w(w_message, space.w_str)):
+        w_text = space.str(w_message)
+        w_message = space.call_function(w_category, w_message)
     else:
         w_text = w_message
         w_message = space.call_function(w_category, w_message)
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -11,6 +11,7 @@
         import _warnings
         _warnings.warn("some message", DeprecationWarning)
         _warnings.warn("some message", Warning)
+        _warnings.warn(("some message",1), Warning)
 
     def test_lineno(self):
         import warnings, _warnings, sys
@@ -40,7 +41,10 @@
     def test_show_source_line(self):
         import warnings
         import sys, StringIO
-        from test.warning_tests import inner
+        try:
+            from test.warning_tests import inner
+        except ImportError:
+            skip('no test, -A on cpython?')
         # With showarning() missing, make sure that output is okay.
         del warnings.showwarning
 

From pypy.commits at gmail.com  Fri Sep 30 08:40:56 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 30 Sep 2016 05:40:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: document branches
Message-ID: <57ee5d58.64f9c20a.f6cf0.9dd4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87471:ac372e3f4bd0
Date: 2016-09-30 13:16 +0300
http://bitbucket.org/pypy/pypy/changeset/ac372e3f4bd0/

Log:	document branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -32,3 +32,13 @@
 ``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
 from somewhere else.  You no longer have to do the same with the
 ``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
+
+.. branch: _warning
+
+CPython allows warning.warn(('something', 1), Warning), on PyPy this
+produced a "expected a readable buffer object" error. Test and fix.
+
+.. branch: stricter-strip
+
+CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
+allowed as arguments. Test and fix for str and unicode

From pypy.commits at gmail.com  Fri Sep 30 08:40:51 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 30 Sep 2016 05:40:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge branch to reject
 'a'.strip(buffer(' ')) like cpython
Message-ID: <57ee5d53.c62f1c0a.50285.4cb8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r87468:3495ebc92c88
Date: 2016-09-30 13:03 +0300
http://bitbucket.org/pypy/pypy/changeset/3495ebc92c88/

Log:	merge branch to reject 'a'.strip(buffer(' ')) like cpython

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -82,7 +82,8 @@
         return False
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
+        # bytearray does not enforce the strict restriction (on strip at least)
         return space.buffer_w(w_other, space.BUF_SIMPLE).as_str()
 
     def _chr(self, char):
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -501,7 +501,10 @@
                 isinstance(w_other, W_UnicodeObject))
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
+        if strict and not space.isinstance_w(w_other, space.w_str):
+            raise oefmt(space.w_TypeError,
+                "%s arg must be None, str or unicode", strict)
         try:
             return space.str_w(w_other)
         except OperationError as e:
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -652,7 +652,7 @@
     def _strip(self, space, w_chars, left, right):
         "internal function called by str_xstrip methods"
         value = self._val(space)
-        chars = self._op_val(space, w_chars)
+        chars = self._op_val(space, w_chars, strict='strip')
 
         lpos = 0
         rpos = len(value)
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -128,7 +128,8 @@
 
         b = bytearray('mississippi')
 
-        for strip_type in str, memoryview:
+        for strip_type in str, memoryview, buffer:
+            print 'strip_type', strip_type
             assert b.strip(strip_type('i')) == 'mississipp'
             assert b.strip(strip_type('m')) == 'ississippi'
             assert b.strip(strip_type('pi')) == 'mississ'
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -251,6 +251,12 @@
         assert b'xyzzyhelloxyzzy'.strip(b'xyz') == b'hello'
         assert b'xyzzyhelloxyzzy'.lstrip(b'xyz') == b'helloxyzzy'
         assert b'xyzzyhelloxyzzy'.rstrip(b'xyz') == b'xyzzyhello'
+        exc = raises(TypeError, s.strip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
+        exc = raises(TypeError, s.rstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
+        exc = raises(TypeError, s.lstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
 
     def test_zfill(self):
         assert b'123'.zfill(2) == b'123'
@@ -656,7 +662,7 @@
         table = maketrans(b'abc', b'xyz')
         assert b'xyzxyz' == b'xyzabcdef'.translate(table, b'def')
         exc = raises(TypeError, "'xyzabcdef'.translate(memoryview(table), 'def')")
-        assert str(exc.value) == 'expected a character buffer object'
+        assert 'expected a' in str(exc.value)
 
         table = maketrans(b'a', b'A')
         assert b'Abc' == b'abc'.translate(table)
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -332,6 +332,12 @@
         assert u'xyzzyhelloxyzzy'.strip(u'xyz') == u'hello'
         assert u'xyzzyhelloxyzzy'.lstrip('xyz') == u'helloxyzzy'
         assert u'xyzzyhelloxyzzy'.rstrip(u'xyz') == u'xyzzyhello'
+        exc = raises(TypeError, s.strip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
+        exc = raises(TypeError, s.rstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
+        exc = raises(TypeError, s.lstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
 
     def test_strip_str_unicode(self):
         x = "--abc--".strip(u"-")
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -124,11 +124,14 @@
         return True
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
         if isinstance(w_other, W_UnicodeObject):
             return w_other._value
         if space.isinstance_w(w_other, space.w_str):
             return unicode_from_string(space, w_other)._value
+        if strict:
+            raise oefmt(space.w_TypeError,
+                "%s arg must be None, unicode or str", strict)
         return unicode_from_encoded_object(
             space, w_other, None, "strict")._value
 

From pypy.commits at gmail.com  Fri Sep 30 08:40:58 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 30 Sep 2016 05:40:58 -0700 (PDT)
Subject: [pypy-commit] pypy buffer-interface2: merge default into branch
Message-ID: <57ee5d5a.670dc20a.d600d.a4ab@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: buffer-interface2
Changeset: r87472:57f07a076df4
Date: 2016-09-30 13:16 +0300
http://bitbucket.org/pypy/pypy/changeset/57f07a076df4/

Log:	merge default into branch

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py
+	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -32,3 +32,13 @@
 ``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
 from somewhere else.  You no longer have to do the same with the
 ``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
+
+.. branch: _warning
+
+CPython allows warning.warn(('something', 1), Warning), on PyPy this
+produced a "expected a readable buffer object" error. Test and fix.
+
+.. branch: stricter-strip
+
+CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
+allowed as arguments. Test and fix for str and unicode
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -93,12 +93,10 @@
             home1 = rffi.charp2str(ll_home)
             home = os.path.join(home1, 'x') # <- so that 'll_home' can be
                                             # directly the root directory
-            dynamic = False
         else:
             home1 = "pypy's shared library location"
-            home = pypydir
-            dynamic = True
-        w_path = pypy_find_stdlib(space, home, dynamic)
+            home = '*'
+        w_path = pypy_find_stdlib(space, home)
         if space.is_none(w_path):
             if verbose:
                 debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
@@ -305,37 +303,20 @@
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_* to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.tool.runsubprocess import run_subprocess
         from rpython.translator.driver import taskdef
         import types
 
-        class Options(object):
-            pass
-
-
-        def mkexename(name):
-            if sys.platform == 'win32':
-                name = name.new(ext='exe')
-            return name
-
         compile_goal, = driver.backend_select_goals(['compile'])
         @taskdef([compile_goal], "Create cffi bindings for modules")
         def task_build_cffi_imports(self):
-            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
             ''' Use cffi to compile cffi interfaces to modules'''
-            exename = mkexename(driver.compute_exe_name())
-            basedir = exename
-            while not basedir.join('include').exists():
-                _basedir = basedir.dirpath()
-                if _basedir == basedir:
-                    raise ValueError('interpreter %s not inside pypy repo',
-                                     str(exename))
-                basedir = _basedir
-            modules = self.config.objspace.usemodules.getpaths()
-            options = Options()
-            # XXX possibly adapt options using modules
-            failures = create_cffi_import_libraries(exename, options, basedir)
-            # if failures, they were already printed
-            print  >> sys.stderr, str(exename),'successfully built (errors, if any, while building the above modules are ignored)'
+            filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py')
+            status, out, err = run_subprocess(str(driver.compute_exe_name()),
+                                              [filename])
+            sys.stdout.write(out)
+            sys.stderr.write(err)
+            # otherwise, ignore errors
         driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
         driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal]
         driver.default_goal = 'build_cffi_imports'
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -387,12 +387,12 @@
 class BufferInterfaceNotFound(Exception):
     pass
 
+ at specialize.memo()
 def wrappable_class_name(Class):
     try:
         return Class.typedef.name
     except AttributeError:
         return 'internal subclass of %s' % (Class.__name__,)
-wrappable_class_name._annspecialcase_ = 'specialize:memo'
 
 class CannotHaveLock(Exception):
     """Raised by space.allocate_lock() if we're translating."""
@@ -841,12 +841,13 @@
             assert type(s) is str
         return self.interned_strings.get(s) is not None
 
+    @specialize.arg(1)
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
             raise DescrMismatch()
         return w_obj
-    descr_self_interp_w._annspecialcase_ = 'specialize:arg(1)'
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         """
         Unwrap w_obj, checking that it is an instance of the required internal
@@ -861,7 +862,6 @@
                         wrappable_class_name(RequiredClass),
                         w_obj.getclass(self))
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def unpackiterable(self, w_iterable, expected_length=-1):
         """Unpack an iterable into a real (interpreter-level) list.
@@ -1300,6 +1300,7 @@
             self.setitem(w_globals, w_key, self.wrap(self.builtin))
         return statement.exec_code(self, w_globals, w_locals)
 
+    @specialize.arg(2)
     def appexec(self, posargs_w, source):
         """ return value from executing given source at applevel.
             EXPERIMENTAL. The source must look like
@@ -1311,7 +1312,6 @@
         w_func = self.fromcache(AppExecCache).getorbuild(source)
         args = Arguments(self, list(posargs_w))
         return self.call_args(w_func, args)
-    appexec._annspecialcase_ = 'specialize:arg(2)'
 
     def _next_or_none(self, w_it):
         try:
@@ -1321,6 +1321,7 @@
                 raise
             return None
 
+    @specialize.arg(3)
     def compare_by_iteration(self, w_iterable1, w_iterable2, op):
         w_it1 = self.iter(w_iterable1)
         w_it2 = self.iter(w_iterable2)
@@ -1343,7 +1344,6 @@
                 if op == 'gt': return self.gt(w_x1, w_x2)
                 if op == 'ge': return self.ge(w_x1, w_x2)
                 assert False, "bad value for op"
-    compare_by_iteration._annspecialcase_ = 'specialize:arg(3)'
 
     def decode_index(self, w_index_or_slice, seqlength):
         """Helper for custom sequence implementations
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib import jit, rgc
+from rpython.rlib import jit, rgc, objectmodel
 
 TICK_COUNTER_STEP = 100
 
@@ -131,6 +131,7 @@
         if self.gettrace() is not None:
             self._trace(frame, 'return', w_retval)
 
+    @objectmodel.always_inline
     def bytecode_trace(self, frame, decr_by=TICK_COUNTER_STEP):
         "Trace function called before each bytecode."
         # this is split into a fast path and a slower path that is
@@ -139,7 +140,6 @@
         actionflag = self.space.actionflag
         if actionflag.decrement_ticker(decr_by) < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
-    bytecode_trace._always_inline_ = True
 
     def _run_finalizers_now(self):
         # Tests only: run the actions now, to ensure that the
@@ -147,6 +147,7 @@
         # pypy.tool.pytest.apptest.
         self.space.actionflag.action_dispatcher(self, None)
 
+    @objectmodel.always_inline
     def bytecode_only_trace(self, frame):
         """
         Like bytecode_trace() but doesn't invoke any other events besides the
@@ -156,7 +157,6 @@
             self.gettrace() is None):
             return
         self.run_trace_func(frame)
-    bytecode_only_trace._always_inline_ = True
 
     @jit.unroll_safe
     def run_trace_func(self, frame):
@@ -203,13 +203,13 @@
 
         d.instr_prev_plus_one = frame.last_instr + 1
 
+    @objectmodel.try_inline
     def bytecode_trace_after_exception(self, frame):
         "Like bytecode_trace(), but without increasing the ticker."
         actionflag = self.space.actionflag
         self.bytecode_only_trace(frame)
         if actionflag.get_ticker() < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
-    bytecode_trace_after_exception._always_inline_ = 'try'
     # NB. this function is not inlined right now.  backendopt.inline would
     # need some improvements to handle this case, but it's not really an
     # issue
@@ -456,6 +456,7 @@
         periodic_actions = unrolling_iterable(self._periodic_actions)
 
         @jit.unroll_safe
+        @objectmodel.dont_inline
         def action_dispatcher(ec, frame):
             # periodic actions (first reset the bytecode counter)
             self.reset_ticker(self.checkinterval_scaled)
@@ -477,7 +478,6 @@
                     action._fired = False
                     action.perform(ec, frame)
 
-        action_dispatcher._dont_inline_ = True
         self.action_dispatcher = action_dispatcher
 
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -6,7 +6,8 @@
 
 from rpython.rlib import jit, rstackovf
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import (we_are_translated, always_inline,
+        dont_inline)
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -483,20 +484,20 @@
         # of oparg failed to produce an integer which is annotated as non-neg
         check_nonneg(oparg)
 
+    @always_inline
     def LOAD_FAST(self, varindex, next_instr):
         # access a local variable directly
         w_value = self.locals_cells_stack_w[varindex]
         if w_value is None:
             self._load_fast_failed(varindex)
         self.pushvalue(w_value)
-    LOAD_FAST._always_inline_ = True
 
+    @dont_inline
     def _load_fast_failed(self, varindex):
         varname = self.getlocalvarname(varindex)
         raise oefmt(self.space.w_UnboundLocalError,
                     "local variable '%s' referenced before assignment",
                     varname)
-    _load_fast_failed._dont_inline_ = True
 
     def LOAD_CONST(self, constindex, next_instr):
         w_const = self.getconstant_w(constindex)
@@ -888,6 +889,7 @@
                 return
         self.LOAD_GLOBAL(nameindex, next_instr)    # fall-back
 
+    @always_inline
     def _load_global(self, varname):
         w_value = self.space.finditem_str(self.get_w_globals(), varname)
         if w_value is None:
@@ -896,16 +898,15 @@
             if w_value is None:
                 self._load_global_failed(varname)
         return w_value
-    _load_global._always_inline_ = True
 
+    @dont_inline
     def _load_global_failed(self, varname):
         raise oefmt(self.space.w_NameError,
                     "global name '%s' is not defined", varname)
-    _load_global_failed._dont_inline_ = True
 
+    @always_inline
     def LOAD_GLOBAL(self, nameindex, next_instr):
         self.pushvalue(self._load_global(self.getname_u(nameindex)))
-    LOAD_GLOBAL._always_inline_ = True
 
     def DELETE_FAST(self, varindex, next_instr):
         if self.locals_cells_stack_w[varindex] is None:
@@ -939,6 +940,7 @@
         self.pushvalue(space.newlist([], sizehint=length_hint))
         self.pushvalue(last_val)
 
+    @always_inline
     def LOAD_ATTR(self, nameindex, next_instr):
         "obj.attributename"
         w_obj = self.popvalue()
@@ -949,7 +951,6 @@
             w_attributename = self.getname_w(nameindex)
             w_value = self.space.getattr(w_obj, w_attributename)
         self.pushvalue(w_value)
-    LOAD_ATTR._always_inline_ = True
 
     @jit.unroll_safe
     def cmp_exc_match(self, w_1, w_2):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1030,8 +1030,8 @@
             # If we are running PyPy with a libpypy-c, the following
             # lines find the stdlib anyway.  Otherwise, it is not found.
             expected_found = (
-                '__pypy__' in sys.builtin_module_names and
-                sys.pypy_translation_info['translation.shared'])
+                getattr(sys, 'pypy_translation_info', {})
+                .get('translation.shared'))
 
             import app_main
             app_main.setup_bootstrap_path(tmp_pypy_c)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -109,6 +109,7 @@
 # we need two subclasses of the app-level type, one to add mapdict, and then one
 # to add del to not slow down the GC.
 
+ at specialize.memo()
 def get_unique_interplevel_subclass(space, cls):
     "NOT_RPYTHON: initialization-time only"
     assert cls.typedef.acceptable_as_base_class
@@ -119,7 +120,6 @@
         assert cls not in _unique_subclass_cache
         _unique_subclass_cache[cls] = subcls
         return subcls
-get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo"
 _unique_subclass_cache = {}
 
 def _getusercls(cls, reallywantdict=False):
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.debug import check_nonneg
+from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
@@ -316,12 +317,12 @@
             w_currently_in_repr = ec._py_repr = space.newdict()
         return dequerepr(space, w_currently_in_repr, space.wrap(self))
 
+    @specialize.arg(2)
     def compare(self, w_other, op):
         space = self.space
         if not isinstance(w_other, W_Deque):
             return space.w_NotImplemented
         return space.compare_by_iteration(space.wrap(self), w_other, op)
-    compare._annspecialcase_ = 'specialize:arg(2)'
 
     def lt(self, w_other):
         return self.compare(w_other, 'lt')
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec
@@ -25,12 +26,12 @@
     def iter_w(self):
         return self.space.wrap(self)
 
+    @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
         w_error = space.getattr(w_module, space.wrap('Error'))
         raise oefmt(w_error, "line %d: %s", self.line_num, msg)
-    error._dont_inline_ = True
 
     def add_char(self, field_builder, c):
         assert field_builder is not None
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.typedef import TypeDef, interp2app
@@ -21,12 +22,12 @@
             special += dialect.quotechar
         self.special_characters = special
 
+    @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
         w_error = space.getattr(w_module, space.wrap('Error'))
         raise OperationError(w_error, space.wrap(msg))
-    error._dont_inline_ = True
 
     def writerow(self, w_fields):
         """Construct and write a CSV record from a sequence of fields.
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -7,7 +7,7 @@
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
                                       interp_attrproperty)
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, always_inline
 from rpython.rlib.rtimer import read_timestamp, _is_64_bit
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -256,7 +256,7 @@
         return w_frame.wrap_string(space)
     return w_frame    # actually a PyCode object
 
-
+ at always_inline
 def prepare_spec(space, w_arg):
     if isinstance(w_arg, Method):
         return (w_arg.w_function, w_arg.w_class)
@@ -264,8 +264,6 @@
         return (w_arg, None)
     else:
         return (None, space.type(w_arg))
-prepare_spec._always_inline_ = True
-
 
 def lsprof_call(space, w_self, frame, event, w_arg):
     assert isinstance(w_self, W_Profiler)
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1,6 +1,6 @@
 import sys
 from rpython.rlib.rstring import StringBuilder
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, always_inline
 from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import oefmt
@@ -188,6 +188,7 @@
         self.pos = i
         return self.space.call_function(self.space.w_int, self.space.wrap(s))
 
+    @always_inline
     def parse_integer(self, i):
         "Parse a decimal number with an optional minus sign"
         sign = 1
@@ -218,7 +219,6 @@
         # overflowed
         ovf_maybe = (count >= OVF_DIGITS)
         return i, ovf_maybe, sign * intval
-    parse_integer._always_inline_ = True
 
     def decode_array(self, i):
         w_list = self.space.newlist([])
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -6,6 +6,7 @@
 import time
 from pypy.interpreter.error import OperationError
 from pypy.module._pypyjson.interp_decoder import loads
+from rpython.rlib.objectmodel import specialize, dont_inline
 
 
 ## MSG = open('msg.json').read()
@@ -68,11 +69,11 @@
         assert isinstance(w_x, W_String)
         return w_x.strval
 
+    @dont_inline
     def call_method(self, obj, name, arg):
         assert name == 'append'
         assert isinstance(obj, W_List)
         obj.listval.append(arg)
-    call_method._dont_inline_ = True
 
     def call_function(self, w_func, *args_w):
         return self.w_None # XXX
@@ -91,6 +92,7 @@
     def wrapfloat(self, x):
         return W_Float(x)
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if isinstance(x, int):
             return W_Int(x)
@@ -100,7 +102,6 @@
         ##     assert False
         else:
             return W_Unicode(unicode(x))
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
 
 fakespace = FakeSpace()
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -9,6 +9,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 import rpython.rlib.rposix as rposix
 
 _MS_WINDOWS = os.name == "nt"
@@ -255,6 +256,7 @@
 
 _ARM = rffi_platform.getdefined('__arm__', '')
 
+ at specialize.arg(2)
 def read_ptr(ptr, ofs, TP):
     T = lltype.Ptr(rffi.CArray(TP))
     for c in unroll_letters_for_floats:
@@ -274,8 +276,8 @@
                 return ptr_val
     else:
         return rffi.cast(T, ptr)[ofs]
-read_ptr._annspecialcase_ = 'specialize:arg(2)'
 
+ at specialize.argtype(2)
 def write_ptr(ptr, ofs, value):
     TP = lltype.typeOf(value)
     T = lltype.Ptr(rffi.CArray(TP))
@@ -296,7 +298,6 @@
                 return
     else:
         rffi.cast(T, ptr)[ofs] = value
-write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
@@ -374,14 +375,15 @@
     def getrawsize(self):
         raise NotImplementedError("abstract base class")
 
+ at specialize.arg(0)
 def unwrap_truncate_int(TP, space, w_arg):
     if space.isinstance_w(w_arg, space.w_int):
         return rffi.cast(TP, space.int_w(w_arg))
     else:
         return rffi.cast(TP, space.bigint_w(w_arg).ulonglongmask())
-unwrap_truncate_int._annspecialcase_ = 'specialize:arg(0)'
 
 
+ at specialize.arg(1)
 def unwrap_value(space, push_func, add_arg, argdesc, letter, w_arg):
     if letter in TYPEMAP_PTR_LETTERS:
         # check for NULL ptr
@@ -422,10 +424,10 @@
                 return
         else:
             raise oefmt(space.w_TypeError, "cannot directly write value")
-unwrap_value._annspecialcase_ = 'specialize:arg(1)'
 
 ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items())
 
+ at specialize.arg(1)
 def wrap_value(space, func, add_arg, argdesc, letter):
     for c, ll_type in ll_typemap_iter:
         if letter == c:
@@ -437,7 +439,6 @@
             else:
                 return space.wrap(func(add_arg, argdesc, ll_type))
     raise oefmt(space.w_TypeError, "cannot directly read value")
-wrap_value._annspecialcase_ = 'specialize:arg(1)'
 
 NARROW_INTEGER_TYPES = 'cbhiBIH?'
 
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, signedtype, r_uint, \
     r_ulonglong
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import specialize
 import sys
 
 IS_BIG_ENDIAN = sys.byteorder == 'big'
@@ -284,6 +285,7 @@
 def NUM_BITS(x):
     return x >> 16
 
+ at specialize.arg(1)
 def BIT_MASK(x, ll_t):
     if ll_t is lltype.SignedLongLong or ll_t is lltype.UnsignedLongLong:
         one = r_ulonglong(1)
@@ -291,8 +293,8 @@
         one = r_uint(1)
     # to avoid left shift by x == sizeof(ll_t)
     return (((one << (x - 1)) - 1) << 1) + 1
-BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.argtype(2)
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
     TP = lltype.typeOf(value)
@@ -313,8 +315,8 @@
                 value = rffi.cast(TP, current)
             break
     write_ptr(ptr, 0, value)
-push_field._annspecialcase_ = 'specialize:argtype(2)'
 
+ at specialize.arg(2)
 def cast_pos(self, i, ll_t):
     pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
     value = read_ptr(pos, 0, ll_t)
@@ -337,7 +339,6 @@
                 value = rffi.cast(ll_t, value)
             break
     return value
-cast_pos._annspecialcase_ = 'specialize:arg(2)'
 
 class W_StructureInstance(W_DataInstance):
     def __init__(self, space, shape, address):
diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -248,6 +248,10 @@
     if space.isinstance_w(w_message, space.w_Warning):
         w_text = space.str(w_message)
         w_category = space.type(w_message)
+    elif (not space.isinstance_w(w_message, space.w_unicode) or
+          not space.isinstance_w(w_message, space.w_str)):
+        w_text = space.str(w_message)
+        w_message = space.call_function(w_category, w_message)
     else:
         w_text = w_message
         w_message = space.call_function(w_category, w_message)
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -11,6 +11,7 @@
         import _warnings
         _warnings.warn("some message", DeprecationWarning)
         _warnings.warn("some message", Warning)
+        _warnings.warn(("some message",1), Warning)
 
     def test_lineno(self):
         import warnings, _warnings, sys
@@ -40,7 +41,10 @@
     def test_show_source_line(self):
         import warnings
         import sys, StringIO
-        from test.warning_tests import inner
+        try:
+            from test.warning_tests import inner
+        except ImportError:
+            skip('no test, -A on cpython?')
         # With showarning() missing, make sure that output is okay.
         del warnings.showwarning
 
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -214,7 +214,9 @@
     i = space.int_w(space.index(args_w[0]))
     j = space.int_w(space.index(args_w[1]))
     w_y = args_w[2]
-    return space.wrap(generic_cpy_call(space, func_target, w_self, i, j, w_y))
+    res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
+    if rffi.cast(lltype.Signed, res) == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
 
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
@@ -296,7 +298,10 @@
 def wrap_hashfunc(space, w_self, w_args, func):
     func_target = rffi.cast(hashfunc, func)
     check_num_args(space, w_args, 0)
-    return space.wrap(generic_cpy_call(space, func_target, w_self))
+    res = generic_cpy_call(space, func_target, w_self)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return space.wrap(res)
 
 class CPyBuffer(Buffer):
     # Similar to Py_buffer
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -57,6 +57,7 @@
         assert arr.tolist() == [1, 21, 22, 23, 4]
         del arr[slice(1, 3)]
         assert arr.tolist() == [1, 23, 4]
+        raises(TypeError, 'arr[slice(1, 3)] = "abc"')
 
     def test_buffer(self):
         import sys
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -2,6 +2,7 @@
 import sys
 
 from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError, oefmt
 
 class State:
@@ -17,6 +18,7 @@
     else:
         return space.float_w(space.float(w_x))
 
+ at specialize.arg(1)
 def math1(space, f, w_x):
     x = _get_double(space, w_x)
     try:
@@ -26,8 +28,8 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return space.wrap(y)
-math1._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.arg(1)
 def math1_w(space, f, w_x):
     x = _get_double(space, w_x)
     try:
@@ -37,8 +39,8 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return r
-math1_w._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.arg(1)
 def math2(space, f, w_x, w_snd):
     x = _get_double(space, w_x)
     snd = _get_double(space, w_snd)
@@ -49,7 +51,6 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return space.wrap(r)
-math2._annspecialcase_ = 'specialize:arg(1)'
 
 def trunc(space, w_x):
     """Truncate x."""
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from rpython.rlib import rmmap, rarithmetic
+from rpython.rlib import rmmap, rarithmetic, objectmodel
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
@@ -316,6 +316,7 @@
         self.w_error = space.new_exception_class("mmap.error",
                                                  space.w_EnvironmentError)
 
+ at objectmodel.dont_inline
 def mmap_error(space, e):
     if isinstance(e, RValueError):
         return OperationError(space.w_ValueError, space.wrap(e.message))
@@ -327,7 +328,6 @@
     else:
         # bogus 'e'?
         return OperationError(space.w_SystemError, space.wrap('%s' % e))
-mmap_error._dont_inline_ = True
 
 
 class MMapBuffer(Buffer):
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -7,6 +7,7 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
+from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
@@ -571,6 +572,7 @@
             return self.w_character_data_handler or space.w_None
         return self.handlers[index]
 
+    @specialize.arg(2)
     def sethandler(self, space, name, w_handler, index, setter, handler):
         if name == 'CharacterDataHandler':
             self.flush_character_buffer(space)
@@ -582,8 +584,6 @@
         self.handlers[index] = w_handler
         setter(self.itself, handler)
 
-    sethandler._annspecialcase_ = 'specialize:arg(2)'
-
     all_chars = ''.join(chr(i) for i in range(256))
 
     def UnknownEncodingHandler(self, space, name, info):
diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py
--- a/pypy/module/select/interp_select.py
+++ b/pypy/module/select/interp_select.py
@@ -2,6 +2,7 @@
 
 from rpython.rlib import _rsocket_rffi as _c, rpoll
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib import objectmodel
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
@@ -91,7 +92,7 @@
 
 # ____________________________________________________________
 
-
+ at objectmodel.always_inline  # get rid of the tuple result
 def _build_fd_set(space, list_w, ll_list, nfds):
     _c.FD_ZERO(ll_list)
     fdlist = []
@@ -105,7 +106,6 @@
         _c.FD_SET(fd, ll_list)
         fdlist.append(fd)
     return fdlist, nfds
-_build_fd_set._always_inline_ = True    # get rid of the tuple result
 
 
 def _unbuild_fd_set(space, list_w, fdlist, ll_list, reslist_w):
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -7,7 +7,7 @@
 import stat
 import sys
 
-from rpython.rlib import rpath
+from rpython.rlib import rpath, rdynload
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -157,16 +157,20 @@
     return space.wrap(resolvedirof(filename))
 
 
- at unwrap_spec(executable='str0', dynamic=int)
-def pypy_find_stdlib(space, executable, dynamic=1):
-    if dynamic and space.config.translation.shared:
-        dynamic_location = pypy_init_home()
-        if dynamic_location:
-            executable = rffi.charp2str(dynamic_location)
-            pypy_init_free(dynamic_location)
-    path, prefix = find_stdlib(get_state(space), executable)
+ at unwrap_spec(executable='str0')
+def pypy_find_stdlib(space, executable):
+    path, prefix = None, None
+    if executable != '*':
+        path, prefix = find_stdlib(get_state(space), executable)
     if path is None:
-        return space.w_None
+        if space.config.translation.shared:
+            dynamic_location = pypy_init_home()
+            if dynamic_location:
+                dyn_path = rffi.charp2str(dynamic_location)
+                pypy_init_free(dynamic_location)
+                path, prefix = find_stdlib(get_state(space), dyn_path)
+        if path is None:
+            return space.w_None
     w_prefix = space.wrap(prefix)
     space.setitem(space.sys.w_dict, space.wrap('prefix'), w_prefix)
     space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), w_prefix)
@@ -237,7 +241,9 @@
 }
 """
 
-_eci = ExternalCompilationInfo(separate_module_sources=[_source_code])
+_eci = ExternalCompilationInfo(separate_module_sources=[_source_code],
+    post_include_bits=['RPY_EXPORTED char *_pypy_init_home(void);'])
+_eci = _eci.merge(rdynload.eci)
 
 pypy_init_home = rffi.llexternal("_pypy_init_home", [], rffi.CCHARP,
                                  _nowrapper=True, compilation_info=_eci)
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -9,68 +9,68 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 
+ at specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
     w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
                                                        '__getattribute__')
     return w_getattribute
-object_getattribute._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
     w_src, w_setattr = space.lookup_in_type_where(space.w_object,
                                                   '__setattr__')
     return w_setattr
-object_setattr._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
     w_src, w_delattr = space.lookup_in_type_where(space.w_object,
                                                   '__delattr__')
     return w_delattr
-object_delattr._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
     w_src, w_hash = space.lookup_in_type_where(space.w_object,
                                                   '__hash__')
     return w_hash
-object_hash._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
     w_src, w_eq = space.lookup_in_type_where(space.w_type,
                                              '__eq__')
     return w_eq
-type_eq._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
     w_src, w_iter = space.lookup_in_type_where(space.w_list,
                                                '__iter__')
     return w_iter
-list_iter._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
     w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
                                                '__iter__')
     return w_iter
-tuple_iter._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
     w_src, w_iter = space.lookup_in_type_where(space.w_str,
                                                '__getitem__')
     return w_iter
-str_getitem._annspecialcase_ = 'specialize:memo'
 
+ at specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
     w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
                                                '__getitem__')
     return w_iter
-unicode_getitem._annspecialcase_ = 'specialize:memo'
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -82,7 +82,8 @@
         return False
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
+        # bytearray does not enforce the strict restriction (on strip at least)
         return space.buffer_w(w_other, space.BUF_SIMPLE).as_str()
 
     def _chr(self, char):
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -501,7 +501,10 @@
                 isinstance(w_other, W_UnicodeObject))
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
+        if strict and not space.isinstance_w(w_other, space.w_str):
+            raise oefmt(space.w_TypeError,
+                "%s arg must be None, str or unicode", strict)
         try:
             return space.str_w(w_other)
         except OperationError as e:
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -7,6 +7,7 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
+from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 
@@ -344,6 +345,7 @@
                         "unsupported format character '%s' (%s) at index %d",
                         s, hex(ord(c)), self.fmtpos - 1)
 
+        @specialize.argtype(1)
         def std_wp(self, r):
             length = len(r)
             if do_unicode and isinstance(r, str):
@@ -369,7 +371,6 @@
             if padding > 0:
                 result.append_multiple_char(const(' '), padding)
             # add any remaining padding at the right
-        std_wp._annspecialcase_ = 'specialize:argtype(1)'
 
         def std_wp_number(self, r, prefix=''):
             result = self.result
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -351,6 +351,7 @@
         assert isinstance(w_starttype, W_TypeObject)
         return w_type.lookup_starting_at(w_starttype, name)
 
+    @specialize.arg(1)
     def allocate_instance(self, cls, w_subtype):
         """Allocate the memory needed for an instance of an internal or
         user-defined type, without actually __init__ializing the instance."""
@@ -375,7 +376,6 @@
                         "%N.__new__(%N): only for the type %N",
                         w_type, w_subtype, w_type)
         return instance
-    allocate_instance._annspecialcase_ = "specialize:arg(1)"
 
     # two following functions are almost identical, but in fact they
     # have different return type. First one is a resizable list, second
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -652,7 +652,7 @@
     def _strip(self, space, w_chars, left, right):
         "internal function called by str_xstrip methods"
         value = self._val(space)
-        chars = self._op_val(space, w_chars)
+        chars = self._op_val(space, w_chars, strict='strip')
 
         lpos = 0
         rpos = len(value)
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -128,7 +128,8 @@
 
         b = bytearray('mississippi')
 
-        for strip_type in str, memoryview:
+        for strip_type in str, memoryview, buffer:
+            print 'strip_type', strip_type
             assert b.strip(strip_type('i')) == 'mississipp'
             assert b.strip(strip_type('m')) == 'ississippi'
             assert b.strip(strip_type('pi')) == 'mississ'
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -251,6 +251,12 @@
         assert b'xyzzyhelloxyzzy'.strip(b'xyz') == b'hello'
         assert b'xyzzyhelloxyzzy'.lstrip(b'xyz') == b'helloxyzzy'
         assert b'xyzzyhelloxyzzy'.rstrip(b'xyz') == b'xyzzyhello'
+        exc = raises(TypeError, s.strip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
+        exc = raises(TypeError, s.rstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
+        exc = raises(TypeError, s.lstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
 
     def test_zfill(self):
         assert b'123'.zfill(2) == b'123'
@@ -656,7 +662,7 @@
         table = maketrans(b'abc', b'xyz')
         assert b'xyzxyz' == b'xyzabcdef'.translate(table, b'def')
         exc = raises(TypeError, "'xyzabcdef'.translate(memoryview(table), 'def')")
-        assert str(exc.value) == 'expected a character buffer object'
+        assert 'expected a' in str(exc.value)
 
         table = maketrans(b'a', b'A')
         assert b'Abc' == b'abc'.translate(table)
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -332,6 +332,12 @@
         assert u'xyzzyhelloxyzzy'.strip(u'xyz') == u'hello'
         assert u'xyzzyhelloxyzzy'.lstrip('xyz') == u'helloxyzzy'
         assert u'xyzzyhelloxyzzy'.rstrip(u'xyz') == u'xyzzyhello'
+        exc = raises(TypeError, s.strip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
+        exc = raises(TypeError, s.rstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
+        exc = raises(TypeError, s.lstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, unicode or str'
 
     def test_strip_str_unicode(self):
         x = "--abc--".strip(u"-")
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -124,11 +124,14 @@
         return True
 
     @staticmethod
-    def _op_val(space, w_other):
+    def _op_val(space, w_other, strict=None):
         if isinstance(w_other, W_UnicodeObject):
             return w_other._value
         if space.isinstance_w(w_other, space.w_str):
             return unicode_from_string(space, w_other)._value
+        if strict:
+            raise oefmt(space.w_TypeError,
+                "%s arg must be None, unicode or str", strict)
         return unicode_from_encoded_object(
             space, w_other, None, "strict")._value
 
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -44,8 +44,8 @@
 if __name__ == '__main__':
     import py, os
     if '__pypy__' not in sys.builtin_module_names:
-        print 'Call with a pypy interpreter'
-        sys.exit(-1)
+        print >> sys.stderr, 'Call with a pypy interpreter'
+        sys.exit(1)
 
     class Options(object):
         pass
@@ -59,18 +59,26 @@
                                  str(exename))
         basedir = _basedir
     options = Options()
-    print >> sys.stderr, "There should be no failures here"
     failures = create_cffi_import_libraries(exename, options, basedir)
     if len(failures) > 0:
-        print '*** failed to build', [f[1] for f in failures]
+        print >> sys.stderr, '*** failed to build the CFFI modules %r' % (
+            [f[1] for f in failures],)
+        print >> sys.stderr, '''
+PyPy can still be used as long as you don't need the corresponding
+modules.  If you do need them, please install the missing headers and
+libraries (see error messages just above) and then re-run the command:
+
+    %s %s
+''' % (sys.executable, ' '.join(sys.argv))
         sys.exit(1)
 
-    # monkey patch a failure, just to test
-    print >> sys.stderr, 'This line should be followed by a traceback'
-    for k in cffi_build_scripts:
-        setattr(options, 'no_' + k, True)
-    must_fail = '_missing_build_script.py'
-    assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
-    cffi_build_scripts['should_fail'] = must_fail
-    failures = create_cffi_import_libraries(exename, options, basedir)
-    assert len(failures) == 1
+    if len(sys.argv) > 1 and sys.argv[1] == '--test':
+        # monkey patch a failure, just to test
+        print >> sys.stderr, 'This line should be followed by a traceback'
+        for k in cffi_build_scripts:
+            setattr(options, 'no_' + k, True)
+        must_fail = '_missing_build_script.py'
+        assert not os.path.exists(str(basedir.join('lib_pypy').join(must_fail)))
+        cffi_build_scripts['should_fail'] = must_fail
+        failures = create_cffi_import_libraries(exename, options, basedir)
+        assert len(failures) == 1
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -138,6 +138,7 @@
             # shadowstack: change 'rpy_fastgil' to 0 (it should be
             # non-zero right now).
             self.change_extra_stack_depth = False
+            # ^^ note that set_extra_stack_depth() in this case is a no-op
             css_value = imm(0)
         else:
             from rpython.memory.gctransform import asmgcroot
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -217,6 +217,12 @@
     func._dont_inline_ = True
     return func
 
+def try_inline(func):
+    """ tell the RPython inline (not the JIT!), to try to inline this function,
+    no matter its size."""
+    func._always_inline_ = 'try'
+    return func
+
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -34,6 +34,7 @@
 else:
     libraries = ['dl']
 
+# this 'eci' is also used in pypy/module/sys/initpath.py
 eci = ExternalCompilationInfo(
     pre_include_bits = pre_include_bits,
     includes = includes,
diff --git a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
@@ -4,17 +4,18 @@
                                 long *current_pos_addr);
 
 
+#define MAX_INLINE_DEPTH  384
+
+
 static long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
-                                             intptr_t ip, int max_depth)
+                                             intptr_t addr, int max_depth)
 {
 #ifdef PYPY_JIT_CODEMAP
     void *codemap;
     long current_pos = 0;
-    intptr_t ident;
+    intptr_t ident, local_stack[MAX_INLINE_DEPTH];
+    long m;
     long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    intptr_t tmp;
 
     codemap = pypy_find_codemap_at_addr(addr, &start_addr);
     if (codemap == NULL || n >= max_depth - 2)
@@ -25,24 +26,29 @@
     // in the middle
     result[n++] = VMPROF_ASSEMBLER_TAG;
     result[n++] = start_addr;
-    start = n;
-    while (n < max_depth) {
+
+    // build the list of code idents corresponding to the current
+    // position inside this particular piece of assembler.  If (very
+    // unlikely) we get more than MAX_INLINE_DEPTH recursion levels
+    // all inlined inside this single piece of assembler, then stop:
+    // there will be some missing frames then.  Otherwise, we need to
+    // first collect 'local_stack' and then write it to 'result' in the
+    // opposite order, stopping at 'max_depth'.  Previous versions of
+    // the code would write the oldest calls and then stop---whereas
+    // what we really need it to write the newest calls and then stop.
+    m = 0;
+    while (m < MAX_INLINE_DEPTH) {
         ident = pypy_yield_codemap_at_addr(codemap, addr, &current_pos);
         if (ident == -1)
             // finish
             break;
         if (ident == 0)
             continue; // not main codemap
+        local_stack[m++] = ident;
+    }
+    while (m > 0 && n < max_depth) {
         result[n++] = VMPROF_JITTED_TAG;
-        result[n++] = ident;
-    }
-    k = 1;
-
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k];
-        result[n - k] = tmp;
-        k += 2;
+        result[n++] = local_stack[--m];
     }
 #endif
     return n;
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
     resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
     iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
     setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
-    fetch_translated_config)
+    fetch_translated_config, try_inline)
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rtyper.test.test_llinterp import interpret
@@ -483,6 +483,13 @@
         return a, b, c
     assert f._always_inline_ is True
 
+def test_try_inline():
+    @try_inline
+    def f(a, b, c):
+        return a, b, c
+    assert f._always_inline_ == "try"
+
+
 def test_enforceargs_defaults():
     @enforceargs(int, int)
     def f(a, b=40):
diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -15,9 +15,10 @@
 static int check_valid(void);
 
 void _RPython_ThreadLocals_Acquire(void) {
-    while (!pypy_lock_test_and_set(&pypy_threadlocal_lock, 1)) {
-        /* busy loop */
-    }
+    long old_value;
+    do {
+        old_value = pypy_lock_test_and_set(&pypy_threadlocal_lock, 1);
+    } while (old_value != 0);   /* busy loop */
     assert(check_valid());
 }
 void _RPython_ThreadLocals_Release(void) {

From pypy.commits at gmail.com  Fri Sep 30 10:02:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 30 Sep 2016 07:02:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-set-sentinel: release the sentinel lock
 before the thread dies
Message-ID: <57ee708e.04321c0a.5ebc1.7320@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-set-sentinel
Changeset: r87473:aa01e511c9e8
Date: 2016-09-30 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/aa01e511c9e8/

Log:	release the sentinel lock before the thread dies

diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -3,6 +3,7 @@
 """
 
 import time
+import weakref
 from rpython.rlib import rthread
 from pypy.module.thread.error import wrap_thread_error
 from pypy.interpreter.baseobjspace import W_Root
@@ -10,6 +11,7 @@
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
 from pypy.interpreter.error import oefmt
 from rpython.rlib.rarithmetic import r_longlong, ovfcheck, ovfcheck_float_to_longlong
+from rpython.rlib.rthread import ThreadLocalReference
 
 # Force the declaration of the type 'thread.LockType' for RPython
 #import pypy.module.thread.rpython.exttable
@@ -19,6 +21,7 @@
 
 RPY_LOCK_FAILURE, RPY_LOCK_ACQUIRED, RPY_LOCK_INTR = range(3)
 
+
 def parse_acquire_args(space, blocking, timeout):
     if not blocking and timeout != -1.0:
         raise oefmt(space.w_ValueError,
@@ -144,15 +147,24 @@
 See LockType.__doc__ for information about locks."""
     return space.wrap(Lock(space))
 
-def set_sentinel(space):
+tlref_sentinel_lock = ThreadLocalReference(Lock)
+
+def _set_sentinel(space):
     """_set_sentinel() -> lock
 
     Set a sentinel lock that will be released when the current thread 
     state is finalized (after it is untied from the interpreter).
 
     This is a private API for the threading module."""
-    lock = allocate_lock(space)
-    return lock
+    # see issue 18808. We need to release this lock just before exiting
+    # the any thread!
+    lock = Lock(space)
+    # create a weak reference to the lock object and set it
+    # pass save it as a thread local reference
+    # see os_thread.py just before gc_thread_die
+    tlref_sentinel_lock.set(lock)
+    #
+    return space.wrap(lock)
 
 class W_RLock(W_Root):
     def __init__(self, space):
diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py
--- a/pypy/module/thread/os_thread.py
+++ b/pypy/module/thread/os_thread.py
@@ -7,6 +7,7 @@
 from pypy.module.thread.error import wrap_thread_error
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, Arguments
+from pypy.module.thread.os_lock import tlref_sentinel_lock
 
 # Here are the steps performed to start a new thread:
 #
@@ -102,6 +103,12 @@
                 os.write(STDERR, "\n")
             except OSError:
                 pass
+        # TODO move after rthread.gc_thread_die()?
+        lock = tlref_sentinel_lock.get()
+        if lock and lock.descr_lock_locked(space):
+            lock.descr_lock_release(space)
+            tlref_sentinel_lock.set(None)
+        #
         bootstrapper.nbthreads -= 1
         rthread.gc_thread_die()
     bootstrap = staticmethod(bootstrap)

From pypy.commits at gmail.com  Fri Sep 30 10:30:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 30 Sep 2016 07:30:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge branch
Message-ID: <57ee76ea.04321c0a.5ebc1.7f08@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r87474:b0742fdfe677
Date: 2016-09-30 16:29 +0200
http://bitbucket.org/pypy/pypy/changeset/b0742fdfe677/

Log:	merge branch

diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py
--- a/pypy/module/thread/__init__.py
+++ b/pypy/module/thread/__init__.py
@@ -21,7 +21,7 @@
         'allocate':               'os_lock.allocate_lock',  # obsolete synonym
         'LockType':               'os_lock.Lock',
         'RLock':                  'os_lock.W_RLock',
-        '_set_sentinel':          'os_lock.set_sentinel',
+        '_set_sentinel':          'os_lock._set_sentinel',
         '_local':                 'os_local.Local',
         'TIMEOUT_MAX':            'space.wrap(float(os_lock.TIMEOUT_MAX // 1000000))',
         'error':                  'space.fromcache(error.Cache).w_error',
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -3,6 +3,7 @@
 """
 
 import time
+import weakref
 from rpython.rlib import rthread
 from pypy.module.thread.error import wrap_thread_error
 from pypy.interpreter.baseobjspace import W_Root
@@ -10,6 +11,7 @@
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
 from pypy.interpreter.error import oefmt
 from rpython.rlib.rarithmetic import r_longlong, ovfcheck, ovfcheck_float_to_longlong
+from rpython.rlib.rthread import ThreadLocalReference
 
 # Force the declaration of the type 'thread.LockType' for RPython
 #import pypy.module.thread.rpython.exttable
@@ -19,6 +21,7 @@
 
 RPY_LOCK_FAILURE, RPY_LOCK_ACQUIRED, RPY_LOCK_INTR = range(3)
 
+
 def parse_acquire_args(space, blocking, timeout):
     if not blocking and timeout != -1.0:
         raise oefmt(space.w_ValueError,
@@ -144,11 +147,24 @@
 See LockType.__doc__ for information about locks."""
     return space.wrap(Lock(space))
 
-def set_sentinel(space):
-    """Set a sentinel lock that will be released when the current thread
-state is finalized (after it is untied from the interpreter)."""
-    lock = allocate_lock(space)
-    return lock
+tlref_sentinel_lock = ThreadLocalReference(Lock)
+
+def _set_sentinel(space):
+    """_set_sentinel() -> lock
+
+    Set a sentinel lock that will be released when the current thread 
+    state is finalized (after it is untied from the interpreter).
+
+    This is a private API for the threading module."""
+    # see issue 18808. We need to release this lock just before exiting
+    # the any thread!
+    lock = Lock(space)
+    # create a weak reference to the lock object and set it
+    # pass save it as a thread local reference
+    # see os_thread.py just before gc_thread_die
+    tlref_sentinel_lock.set(lock)
+    #
+    return space.wrap(lock)
 
 class W_RLock(W_Root):
     def __init__(self, space):
diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py
--- a/pypy/module/thread/os_thread.py
+++ b/pypy/module/thread/os_thread.py
@@ -7,6 +7,7 @@
 from pypy.module.thread.error import wrap_thread_error
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, Arguments
+from pypy.module.thread.os_lock import tlref_sentinel_lock
 
 # Here are the steps performed to start a new thread:
 #
@@ -102,6 +103,12 @@
                 os.write(STDERR, "\n")
             except OSError:
                 pass
+        # TODO move after rthread.gc_thread_die()?
+        lock = tlref_sentinel_lock.get()
+        if lock and lock.descr_lock_locked(space):
+            lock.descr_lock_release(space)
+            tlref_sentinel_lock.set(None)
+        #
         bootstrapper.nbthreads -= 1
         rthread.gc_thread_die()
     bootstrap = staticmethod(bootstrap)

From pypy.commits at gmail.com  Fri Sep 30 10:48:17 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 30 Sep 2016 07:48:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: removed some skips for modules that work
 now!
Message-ID: <57ee7b31.a22fc20a.f806c.de05@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r87475:0a89df9f97f2
Date: 2016-09-30 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/0a89df9f97f2/

Log:	removed some skips for modules that work now!

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -108,7 +108,7 @@
     RegrTest('test_array.py', core=True, usemodules='struct array binascii'),
     RegrTest('test_asdl_parser.py'),
     RegrTest('test_ast.py', core=True, usemodules='struct'),
-    RegrTest('test_asynchat.py', usemodules='select fcntl', skip="XXX: deadlocks or takes forever"),
+    RegrTest('test_asynchat.py', usemodules='select fcntl', skip="XXX: deadlocks or takes forever, crashes now!"),
     RegrTest('test_asyncore.py', usemodules='select fcntl'),
     RegrTest('test_atexit.py', core=True),
     RegrTest('test_audioop.py'),
@@ -125,7 +125,7 @@
     RegrTest('test_bufio.py', core=True),
     RegrTest('test_builtin.py', core=True, usemodules='binascii'),
     RegrTest('test_bytes.py', usemodules='struct binascii'),
-    RegrTest('test_bz2.py', usemodules='bz2', skip="XXX: deadlocks?"),
+    RegrTest('test_bz2.py', usemodules='bz2'),
     RegrTest('test_calendar.py'),
     RegrTest('test_call.py', core=True),
     RegrTest('test_capi.py', usemodules='cpyext'),
@@ -319,7 +319,7 @@
     RegrTest('test_osx_env.py'),
     RegrTest('test_parser.py', skip="slowly deprecating compiler"),
     RegrTest('test_pathlib.py'),
-    RegrTest('test_pdb.py', skip="XXX: deadlocks?"),
+    RegrTest('test_pdb.py'),
     RegrTest('test_peepholer.py'),
     RegrTest('test_pep247.py'),
     RegrTest('test_pep277.py'),
@@ -338,7 +338,7 @@
     RegrTest('test_pkgutil.py'),
     RegrTest('test_platform.py'),
     RegrTest('test_plistlib.py'),
-    RegrTest('test_poll.py', skip="XXX: deadlocks?"),
+    RegrTest('test_poll.py'),
     RegrTest('test_popen.py'),
     RegrTest('test_poplib.py', skip="XXX: deadlocks?"),
     RegrTest('test_posix.py', usemodules="_rawffi"),
@@ -393,7 +393,7 @@
     RegrTest('test_sort.py', core=True),
     RegrTest('test_source_encoding.py'),
     RegrTest('test_spwd.py'),
-    RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib", skip="XXX: deadlocks?"),
+    RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"),
     RegrTest('test_ssl.py', usemodules='_ssl _socket select'),
     RegrTest('test_startfile.py'),
     RegrTest('test_stat.py'),
@@ -407,7 +407,7 @@
     RegrTest('test_struct.py', usemodules='struct'),
     RegrTest('test_structmembers.py', skip="CPython specific"),
     RegrTest('test_structseq.py'),
-    RegrTest('test_subprocess.py', usemodules='signal', skip="XXX: deadlocks?"),
+    RegrTest('test_subprocess.py', usemodules='signal'),
     RegrTest('test_sunau.py'),
     RegrTest('test_sundry.py'),
     RegrTest('test_super.py', core=True),
@@ -427,7 +427,7 @@
     RegrTest('test_thread.py', usemodules="thread", core=True),
     RegrTest('test_threaded_import.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
     RegrTest('test_threadedtempfile.py', usemodules="thread", core=False, skip="XXX: deadlocks?"),
-    RegrTest('test_threading.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
+    RegrTest('test_threading.py', usemodules="thread", core=True),
     RegrTest('test_threading_local.py', usemodules="thread", core=True, skip="XXX: deadlocks?"),
     RegrTest('test_threadsignals.py', usemodules="thread"),
     RegrTest('test_time.py', core=True, usemodules="struct thread _rawffi"),

From pypy.commits at gmail.com  Fri Sep 30 10:52:40 2016
From: pypy.commits at gmail.com (sbauman)
Date: Fri, 30 Sep 2016 07:52:40 -0700 (PDT)
Subject: [pypy-commit] pypy value-classes: Remove old comments
Message-ID: <57ee7c38.902d1c0a.8aabe.89e9@mx.google.com>

Author: Spenser Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r87476:647b3121257d
Date: 2016-09-30 10:52 -0400
http://bitbucket.org/pypy/pypy/changeset/647b3121257d/

Log:	Remove old comments

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -246,8 +246,8 @@
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
 
-        if (not self.typedescr.is_value_class() or
-            info is None or info.is_virtual()):
+        if (info is None or info.is_virtual() or
+            not self.typedescr.is_value_class()):
             return AbstractVirtualStructStateInfo.enum_forced_boxes(
                     self, boxes, box, optimizer, force_boxes)
 
@@ -256,8 +256,6 @@
             info = info._get_info(self.typedescr, optheap)
         assert isinstance(info, AbstractStructPtrInfo)
 
-        # TODO: Do we need to create a new object via NEW_WITH_VTABLE, or will the
-        # allocation be handled properly?
         opinfo = self.make_virtual_copy(box, info, optimizer.optimizer)
         for i in range(min(len(self.fielddescrs), len(opinfo._fields))):
             state = self.fieldstate[i]
@@ -304,14 +302,6 @@
             else:
                 raise VirtualStatesCantMatch("_generate_guards_non_virtual: classes don't match")
 
-        # Things to do...
-        # 1. Generate new_with_vtable operation to allocate the new virtual object
-        # 2. Generate getfield operations which populate the fields of the new virtual
-        # 3. Recursively generate guards for each portion of the virtual state
-        #    (we don't have a VirtualStateInfo objects for the subfields of the
-        #     non-virtual object which we are promoting to a virtual. How do we
-        #     generate this new virtual state so we can operate recursively)
-
         optimizer = state.optimizer
         if runtime_box is not None:
             opinfo = optimizer.getptrinfo(box)
@@ -322,10 +312,6 @@
         else:
             opinfo = None
 
-        # We need to build a virtual version which conforms with the expected
-        # virtual object.
-        # This will probably look a lot like
-        # AbstractVirtualStateInfo._generate_guards
         for i, descr in enumerate(self.fielddescrs):
             if runtime_box is not None and opinfo is not None:
                 fieldbox = self.getfield(box, opinfo, descr, optimizer,

From pypy.commits at gmail.com  Fri Sep 30 10:59:23 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 30 Sep 2016 07:59:23 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: document some progress in the
 stdlib deadlocks
Message-ID: <57ee7dcb.ca11c30a.8a70a.dcce@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5722:9578625ae1fd
Date: 2016-09-30 16:57 +0200
http://bitbucket.org/pypy/extradoc/changeset/9578625ae1fd/

Log:	document some progress in the stdlib deadlocks

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -5,7 +5,6 @@
 In-progress ("Lock" section)
 ----------------------------
 
-* richard: threads do not seem to join (e.g. lib-python/3/test/test_bz2.py)
 
 * arigo: faulthandler module
 
@@ -15,7 +14,8 @@
 ----------------------------------------
 
 * At some point, review lib-python/conftest.py to remove the skips
-  due to deadlocks (search for "XXX:")
+  due to deadlocks (search for "XXX:").
+  update 1: some have been reenabled already! (richard 30.sept)
 
 * collections.py: ``OrderedDict`` should again be a thin wrapper over
   ``dict``.  The main pain point is ``move_to_end(last=False)``.  See
@@ -84,3 +84,4 @@
 ----------------------------------
 
 * richard: bz2, lzma, ... changes (cpython issue 15955) (DONE)
+* richard: threads do not seem to join (e.g. lib-python/3/test/test_bz2.py) (DONE)

From pypy.commits at gmail.com  Fri Sep 30 13:54:09 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 30 Sep 2016 10:54:09 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: hg merge default
Message-ID: <57eea6c1.94a81c0a.eb70.449e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87477:9929cb825a56
Date: 2016-09-30 18:36 +0100
http://bitbucket.org/pypy/pypy/changeset/9929cb825a56/

Log:	hg merge default

diff too long, truncating to 2000 out of 5640 lines

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py
+	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -13,6 +13,7 @@
 import sys
 import os
 import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -62,8 +63,7 @@
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
 
@@ -75,8 +75,7 @@
     """Initialize the module as appropriate for NT"""
     g = {}
     g['EXE'] = ".exe"
-    g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -529,7 +529,7 @@
         for suffix, mode, type_ in imp.get_suffixes():
             if type_ == imp.C_EXTENSION:
                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
-                break
+                break        
 
     if args:
         vals = []
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -22,7 +22,10 @@
     code, message = _ffi.getwinerror()
     raise WindowsError(code, message)
 
-_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1)
+def _int2handle(val):
+    return _ffi.cast("HANDLE", val)
+
+_INVALID_HANDLE_VALUE = _int2handle(-1)
 
 class _handle(object):
     def __init__(self, c_handle):
@@ -70,9 +73,9 @@
     target = _ffi.new("HANDLE[1]")
 
     res = _kernel32.DuplicateHandle(
-        _ffi.cast("HANDLE", source_process),
-        _ffi.cast("HANDLE", source),
-        _ffi.cast("HANDLE", target_process),
+        _int2handle(source_process),
+        _int2handle(source),
+        _int2handle(target_process),
         target, access, inherit, options)
 
     if not res:
@@ -119,12 +122,14 @@
     if not res:
         raise _WinError()
 
-    return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId
+    return (_handle(pi.hProcess),
+            _handle(pi.hThread),
+            pi.dwProcessId,
+            pi.dwThreadId)
 
 def WaitForSingleObject(handle, milliseconds):
     # CPython: the first argument is expected to be an integer.
-    res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle),
-                                        milliseconds)
+    res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds)
     if res < 0:
         raise _WinError()
 
@@ -134,7 +139,7 @@
     # CPython: the first argument is expected to be an integer.
     code = _ffi.new("DWORD[1]")
 
-    res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code)
+    res = _kernel32.GetExitCodeProcess(_int2handle(handle), code)
 
     if not res:
         raise _WinError()
@@ -144,7 +149,7 @@
 def TerminateProcess(handle, exitcode):
     # CPython: the first argument is expected to be an integer.
     # The second argument is silently wrapped in a UINT.
-    res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle),
+    res = _kernel32.TerminateProcess(_int2handle(handle),
                                      _ffi.cast("UINT", exitcode))
 
     if not res:
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.2
+Version: 1.8.4
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.2"
-__version_info__ = (1, 8, 2)
+__version__ = "1.8.4"
+__version_info__ = (1, 8, 4)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.2"
+                               "\ncompiled with cffi version: 1.8.4"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -332,7 +332,7 @@
                         realtype = model.unknown_ptr_type(decl.name)
                     else:
                         realtype, quals = self._get_type_and_quals(
-                            decl.type, name=decl.name)
+                            decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
                 else:
                     raise api.CDefError("unrecognized construct", decl)
@@ -781,11 +781,14 @@
                 exprnode.name in self._int_constants):
             return self._int_constants[exprnode.name]
         #
-        if partial_length_ok:
-            if (isinstance(exprnode, pycparser.c_ast.ID) and
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
+            if partial_length_ok:
                 self._partial_length = True
                 return '...'
+            raise api.FFIError(":%d: unsupported '[...]' here, cannot derive "
+                               "the actual array length in this context"
+                               % exprnode.coord.line)
         #
         raise api.FFIError(":%d: unsupported expression: expected a "
                            "simple numeric constant" % exprnode.coord.line)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -587,8 +587,11 @@
     # ----------
     # typedefs
 
+    def _typedef_type(self, tp, name):
+        return self._global_type(tp, "(*(%s *)0)" % (name,))
+
     def _generate_cpy_typedef_collecttype(self, tp, name):
-        self._do_collect_type(tp)
+        self._do_collect_type(self._typedef_type(tp, name))
 
     def _generate_cpy_typedef_decl(self, tp, name):
         pass
@@ -598,6 +601,7 @@
         self._lsts["typename"].append(TypenameExpr(name, type_index))
 
     def _generate_cpy_typedef_ctx(self, tp, name):
+        tp = self._typedef_type(tp, name)
         self._typedef_ctx(tp, name)
         if getattr(tp, "origin", None) == "unknown_type":
             self._struct_ctx(tp, tp.name, approxname=None)
diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt
--- a/pypy/doc/config/translation.profopt.txt
+++ b/pypy/doc/config/translation.profopt.txt
@@ -3,3 +3,14 @@
 RPython program) to gather profile data. Example for pypy-c: "-c 'from
 richards import main;main(); from test import pystone;
 pystone.main()'"
+
+NOTE: be aware of what this does in JIT-enabled executables.  What it
+does is instrument and later optimize the C code that happens to run in
+the example you specify, ignoring any execution of the JIT-generated
+assembler.  That means that you have to choose the example wisely.  If
+it is something that will just generate assembler and stay there, there
+is little value.  If it is something that exercises heavily library
+routines that are anyway written in C, then it will optimize that.  Most
+interesting would be something that causes a lot of JIT-compilation,
+like running a medium-sized test suite several times in a row, in order
+to optimize the warm-up in general.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -449,6 +449,27 @@
   support (see ``multiline_input()``).  On the other hand,
   ``parse_and_bind()`` calls are ignored (issue `#2072`_).
 
+* ``sys.getsizeof()`` always raises ``TypeError``.  This is because a
+  memory profiler using this function is most likely to give results
+  inconsistent with reality on PyPy.  It would be possible to have
+  ``sys.getsizeof()`` return a number (with enough work), but that may
+  or may not represent how much memory the object uses.  It doesn't even
+  make really sense to ask how much *one* object uses, in isolation with
+  the rest of the system.  For example, instances have maps, which are
+  often shared across many instances; in this case the maps would
+  probably be ignored by an implementation of ``sys.getsizeof()``, but
+  their overhead is important in some cases if they are many instances
+  with unique maps.  Conversely, equal strings may share their internal
+  string data even if they are different objects---or empty containers
+  may share parts of their internals as long as they are empty.  Even
+  stranger, some lists create objects as you read them; if you try to
+  estimate the size in memory of ``range(10**6)`` as the sum of all
+  items' size, that operation will by itself create one million integer
+  objects that never existed in the first place.  Note that some of
+  these concerns also exist on CPython, just less so.  For this reason
+  we explicitly don't implement ``sys.getsizeof()``.
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -34,9 +34,11 @@
    This function searches the PyPy standard library starting from the given
    "PyPy home directory".  The arguments are:
 
-   * ``home``: NULL terminated path to an executable inside the pypy directory
+   * ``home``: path to an executable inside the pypy directory
      (can be a .so name, can be made up).  Used to look up the standard
-     library, and is also set as ``sys.executable``.
+     library, and is also set as ``sys.executable``.  From PyPy 5.5, you can
+     just say NULL here, as long as the ``libpypy-c.so/dylib/dll`` is itself
+     inside this directory.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
@@ -82,18 +84,14 @@
 
 Note that this API is a lot more minimal than say CPython C API, so at first
 it's obvious to think that you can't do much. However, the trick is to do
-all the logic in Python and expose it via `cffi`_ callbacks. Let's assume
-we're on linux and pypy is installed in ``/opt/pypy`` (with
-subdirectories like ``lib-python`` and ``lib_pypy``), and with the
-library in ``/opt/pypy/bin/libpypy-c.so``.  (It doesn't need to be
-installed; you can also replace these paths with a local extract of the
-installation tarballs, or with your local checkout of pypy.) We write a
-little C program:
+all the logic in Python and expose it via `cffi`_ callbacks.
+We write a little C program:
 
 .. code-block:: c
 
     #include "PyPy.h"
     #include <stdio.h>
+    #include <stdlib.h>
 
     static char source[] = "print 'hello from pypy'";
 
@@ -102,9 +100,9 @@
         int res;
 
         rpython_startup_code();
-        /* note: in the path /opt/pypy/x, the final x is ignored and
-           replaced with lib-python and lib_pypy. */
-        res = pypy_setup_home("/opt/pypy/x", 1);
+        /* Before PyPy 5.5, you may need to say e.g. "/opt/pypy/bin" instead
+         * of NULL. */
+        res = pypy_setup_home(NULL, 1);
         if (res) {
             printf("Error setting pypy home!\n");
             return 1;
@@ -123,11 +121,6 @@
     $ LD_LIBRARY_PATH=/opt/pypy/bin ./x
     hello from pypy
 
-.. note:: If the compilation fails because of missing PyPy.h header file,
-          you are running PyPy <= 2.2.1.  Get it here__.
-
-.. __: https://bitbucket.org/pypy/pypy/raw/c4cd6eca9358066571500ac82aaacfdaa3889e8c/include/PyPy.h
-
 On OSX it is necessary to set the rpath of the binary if one wants to link to it,
 with a command like::
 
@@ -181,6 +174,7 @@
     /* C example */
     #include "PyPy.h"
     #include <stdio.h>
+    #include <stdlib.h>
 
     struct API {
         double (*add_numbers)(double x, double y);
@@ -196,7 +190,7 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/x", 1);
+        res = pypy_setup_home(NULL, 1);
         if (res) {
             fprintf(stderr, "Error setting pypy home!\n");
             return -1;
@@ -237,6 +231,8 @@
 Finding pypy_home
 -----------------
 
+**You can usually skip this section if you are running PyPy >= 5.5.**
+
 The function pypy_setup_home() takes as first parameter the path to a
 file from which it can deduce the location of the standard library.
 More precisely, it tries to remove final components until it finds
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -16,3 +16,29 @@
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
+
+.. branch: search-path-from-libpypy
+
+The compiled pypy now looks for its lib-python/lib_pypy path starting
+from the location of the *libpypy-c* instead of the executable. This is
+arguably more consistent, and also it is what occurs anyway if you're
+embedding pypy.  Linux distribution packagers, take note!  At a minimum,
+the ``libpypy-c.so`` must really be inside the path containing
+``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
+from somewhere else.  You no longer have to do the same with the
+``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
+
+.. branch: _warning
+
+CPython allows warning.warn(('something', 1), Warning), on PyPy this
+produced a "expected a readable buffer object" error. Test and fix.
+
+.. branch: stricter-strip
+
+CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
+allowed as arguments. Test and fix for str and unicode
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -89,17 +89,18 @@
     def pypy_setup_home(ll_home, verbose):
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
-        if ll_home:
+        if ll_home and ord(ll_home[0]):
             home1 = rffi.charp2str(ll_home)
             home = os.path.join(home1, 'x') # <- so that 'll_home' can be
                                             # directly the root directory
         else:
-            home = home1 = pypydir
+            home1 = "pypy's shared library location"
+            home = '*'
         w_path = pypy_find_stdlib(space, home)
         if space.is_none(w_path):
             if verbose:
                 debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
-                      " not found in '%s' or in any parent directory" % home1)
+                      " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.appexec([w_path], """(path):
@@ -239,6 +240,10 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+            if config.translation.profopt is not None:
+                raise Exception("Cannot use the --profopt option "
+                                "when --shared is on (it is by default). "
+                                "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
@@ -298,37 +303,20 @@
         # HACKHACKHACK
         # ugly hack to modify target goal from compile_* to build_cffi_imports
         # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.tool.runsubprocess import run_subprocess
         from rpython.translator.driver import taskdef
         import types
 
-        class Options(object):
-            pass
-
-
-        def mkexename(name):
-            if sys.platform == 'win32':
-                name = name.new(ext='exe')
-            return name
-
         compile_goal, = driver.backend_select_goals(['compile'])
         @taskdef([compile_goal], "Create cffi bindings for modules")
         def task_build_cffi_imports(self):
-            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
             ''' Use cffi to compile cffi interfaces to modules'''
-            exename = mkexename(driver.compute_exe_name())
-            basedir = exename
-            while not basedir.join('include').exists():
-                _basedir = basedir.dirpath()
-                if _basedir == basedir:
-                    raise ValueError('interpreter %s not inside pypy repo',
-                                     str(exename))
-                basedir = _basedir
-            modules = self.config.objspace.usemodules.getpaths()
-            options = Options()
-            # XXX possibly adapt options using modules
-            failures = create_cffi_import_libraries(exename, options, basedir)
-            # if failures, they were already printed
-            print  >> sys.stderr, str(exename),'successfully built (errors, if any, while building the above modules are ignored)'
+            filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py')
+            status, out, err = run_subprocess(str(driver.compute_exe_name()),
+                                              [filename])
+            sys.stdout.write(out)
+            sys.stderr.write(err)
+            # otherwise, ignore errors
         driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
         driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal]
         driver.default_goal = 'build_cffi_imports'
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -375,12 +375,12 @@
 class BufferInterfaceNotFound(Exception):
     pass
 
+ at specialize.memo()
 def wrappable_class_name(Class):
     try:
         return Class.typedef.name
     except AttributeError:
         return 'internal subclass of %s' % (Class.__name__,)
-wrappable_class_name._annspecialcase_ = 'specialize:memo'
 
 class CannotHaveLock(Exception):
     """Raised by space.allocate_lock() if we're translating."""
@@ -829,12 +829,13 @@
             assert type(s) is str
         return self.interned_strings.get(s) is not None
 
+    @specialize.arg(1)
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
             raise DescrMismatch()
         return w_obj
-    descr_self_interp_w._annspecialcase_ = 'specialize:arg(1)'
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         """
         Unwrap w_obj, checking that it is an instance of the required internal
@@ -849,7 +850,6 @@
                         wrappable_class_name(RequiredClass),
                         w_obj.getclass(self))
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def unpackiterable(self, w_iterable, expected_length=-1):
         """Unpack an iterable into a real (interpreter-level) list.
@@ -1288,6 +1288,7 @@
             self.setitem(w_globals, w_key, self.wrap(self.builtin))
         return statement.exec_code(self, w_globals, w_locals)
 
+    @specialize.arg(2)
     def appexec(self, posargs_w, source):
         """ return value from executing given source at applevel.
             EXPERIMENTAL. The source must look like
@@ -1299,7 +1300,6 @@
         w_func = self.fromcache(AppExecCache).getorbuild(source)
         args = Arguments(self, list(posargs_w))
         return self.call_args(w_func, args)
-    appexec._annspecialcase_ = 'specialize:arg(2)'
 
     def _next_or_none(self, w_it):
         try:
@@ -1309,6 +1309,7 @@
                 raise
             return None
 
+    @specialize.arg(3)
     def compare_by_iteration(self, w_iterable1, w_iterable2, op):
         w_it1 = self.iter(w_iterable1)
         w_it2 = self.iter(w_iterable2)
@@ -1331,7 +1332,6 @@
                 if op == 'gt': return self.gt(w_x1, w_x2)
                 if op == 'ge': return self.ge(w_x1, w_x2)
                 assert False, "bad value for op"
-    compare_by_iteration._annspecialcase_ = 'specialize:arg(3)'
 
     def decode_index(self, w_index_or_slice, seqlength):
         """Helper for custom sequence implementations
@@ -1974,6 +1974,7 @@
     'ZeroDivisionError',
     'RuntimeWarning',
     'PendingDeprecationWarning',
+    'UserWarning',
 ]
 
 if sys.platform.startswith("win"):
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -374,11 +374,8 @@
         self._value = value
         self.setup(w_type)
 
-    def get_w_value(self, space):
-        w_value = self._w_value
-        if w_value is None:
-            self._w_value = w_value = space.wrap(self._value)
-        return w_value
+    def _compute_value(self, space):
+        return self._value
 
 @specialize.memo()
 def get_operr_class(valuefmt):
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib import jit, rgc
+from rpython.rlib import jit, rgc, objectmodel
 
 TICK_COUNTER_STEP = 100
 
@@ -131,6 +131,7 @@
         if self.gettrace() is not None:
             self._trace(frame, 'return', w_retval)
 
+    @objectmodel.always_inline
     def bytecode_trace(self, frame, decr_by=TICK_COUNTER_STEP):
         "Trace function called before each bytecode."
         # this is split into a fast path and a slower path that is
@@ -139,7 +140,6 @@
         actionflag = self.space.actionflag
         if actionflag.decrement_ticker(decr_by) < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
-    bytecode_trace._always_inline_ = True
 
     def _run_finalizers_now(self):
         # Tests only: run the actions now, to ensure that the
@@ -147,6 +147,7 @@
         # pypy.tool.pytest.apptest.
         self.space.actionflag.action_dispatcher(self, None)
 
+    @objectmodel.always_inline
     def bytecode_only_trace(self, frame):
         """
         Like bytecode_trace() but doesn't invoke any other events besides the
@@ -156,7 +157,6 @@
             self.gettrace() is None):
             return
         self.run_trace_func(frame)
-    bytecode_only_trace._always_inline_ = True
 
     @jit.unroll_safe
     def run_trace_func(self, frame):
@@ -203,13 +203,13 @@
 
         d.instr_prev_plus_one = frame.last_instr + 1
 
+    @objectmodel.try_inline
     def bytecode_trace_after_exception(self, frame):
         "Like bytecode_trace(), but without increasing the ticker."
         actionflag = self.space.actionflag
         self.bytecode_only_trace(frame)
         if actionflag.get_ticker() < 0:
             actionflag.action_dispatcher(self, frame)     # slow path
-    bytecode_trace_after_exception._always_inline_ = 'try'
     # NB. this function is not inlined right now.  backendopt.inline would
     # need some improvements to handle this case, but it's not really an
     # issue
@@ -456,6 +456,7 @@
         periodic_actions = unrolling_iterable(self._periodic_actions)
 
         @jit.unroll_safe
+        @objectmodel.dont_inline
         def action_dispatcher(ec, frame):
             # periodic actions (first reset the bytecode counter)
             self.reset_ticker(self.checkinterval_scaled)
@@ -477,7 +478,6 @@
                     action._fired = False
                     action.perform(ec, frame)
 
-        action_dispatcher._dont_inline_ = True
         self.action_dispatcher = action_dispatcher
 
 
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -63,7 +63,7 @@
         """x.__iter__() <==> iter(x)"""
         return self.space.wrap(self)
 
-    def descr_send(self, w_arg=None):
+    def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator,
 return next yielded value or raise StopIteration."""
         return self.send_ex(w_arg)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -264,25 +264,22 @@
         try:
             executioncontext.call_trace(self)
             #
-            if operr is not None:
-                ec = self.space.getexecutioncontext()
-                next_instr = self.handle_operation_error(ec, operr)
-                self.last_instr = intmask(next_instr - 1)
-            else:
-                # Execution starts just after the last_instr.  Initially,
-                # last_instr is -1.  After a generator suspends it points to
-                # the YIELD_VALUE instruction.
-                next_instr = r_uint(self.last_instr + 1)
-                if next_instr != 0:
-                    self.pushvalue(w_inputvalue)
-            #
             try:
+                if operr is not None:
+                    ec = self.space.getexecutioncontext()
+                    next_instr = self.handle_operation_error(ec, operr)
+                    self.last_instr = intmask(next_instr - 1)
+                else:
+                    # Execution starts just after the last_instr.  Initially,
+                    # last_instr is -1.  After a generator suspends it points to
+                    # the YIELD_VALUE instruction.
+                    next_instr = r_uint(self.last_instr + 1)
+                    if next_instr != 0:
+                        self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
-            except Exception:
-                executioncontext.return_trace(self, self.space.w_None)
-                raise
-            executioncontext.return_trace(self, w_exitvalue)
+            finally:
+                executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
             # this is now done by the code in pypyjit module
             # since we don't want to invalidate the virtualizable
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -6,7 +6,8 @@
 
 from rpython.rlib import jit, rstackovf
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import (we_are_translated, always_inline,
+        dont_inline)
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.tool.sourcetools import func_with_new_name
 
@@ -483,20 +484,20 @@
         # of oparg failed to produce an integer which is annotated as non-neg
         check_nonneg(oparg)
 
+    @always_inline
     def LOAD_FAST(self, varindex, next_instr):
         # access a local variable directly
         w_value = self.locals_cells_stack_w[varindex]
         if w_value is None:
             self._load_fast_failed(varindex)
         self.pushvalue(w_value)
-    LOAD_FAST._always_inline_ = True
 
+    @dont_inline
     def _load_fast_failed(self, varindex):
         varname = self.getlocalvarname(varindex)
         raise oefmt(self.space.w_UnboundLocalError,
                     "local variable '%s' referenced before assignment",
                     varname)
-    _load_fast_failed._dont_inline_ = True
 
     def LOAD_CONST(self, constindex, next_instr):
         w_const = self.getconstant_w(constindex)
@@ -888,6 +889,7 @@
                 return
         self.LOAD_GLOBAL(nameindex, next_instr)    # fall-back
 
+    @always_inline
     def _load_global(self, varname):
         w_value = self.space.finditem_str(self.get_w_globals(), varname)
         if w_value is None:
@@ -896,16 +898,15 @@
             if w_value is None:
                 self._load_global_failed(varname)
         return w_value
-    _load_global._always_inline_ = True
 
+    @dont_inline
     def _load_global_failed(self, varname):
         raise oefmt(self.space.w_NameError,
                     "global name '%s' is not defined", varname)
-    _load_global_failed._dont_inline_ = True
 
+    @always_inline
     def LOAD_GLOBAL(self, nameindex, next_instr):
         self.pushvalue(self._load_global(self.getname_u(nameindex)))
-    LOAD_GLOBAL._always_inline_ = True
 
     def DELETE_FAST(self, varindex, next_instr):
         if self.locals_cells_stack_w[varindex] is None:
@@ -939,6 +940,7 @@
         self.pushvalue(space.newlist([], sizehint=length_hint))
         self.pushvalue(last_val)
 
+    @always_inline
     def LOAD_ATTR(self, nameindex, next_instr):
         "obj.attributename"
         w_obj = self.popvalue()
@@ -949,7 +951,6 @@
             w_attributename = self.getname_w(nameindex)
             w_value = self.space.getattr(w_obj, w_attributename)
         self.pushvalue(w_value)
-    LOAD_ATTR._always_inline_ = True
 
     @jit.unroll_safe
     def cmp_exc_match(self, w_1, w_2):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -1019,23 +1019,32 @@
         old_sys_path = sys.path[:]
         old_cwd = os.getcwd()
 
-        sys.path.append(self.goal_dir)
         # make sure cwd does not contain a stdlib
         if self.tmp_dir.startswith(self.trunkdir):
             skip('TMPDIR is inside the PyPy source')
-        os.chdir(self.tmp_dir)
+        sys.path.append(self.goal_dir)
         tmp_pypy_c = os.path.join(self.tmp_dir, 'pypy-c')
         try:
+            os.chdir(self.tmp_dir)
+
+            # If we are running PyPy with a libpypy-c, the following
+            # lines find the stdlib anyway.  Otherwise, it is not found.
+            expected_found = (
+                getattr(sys, 'pypy_translation_info', {})
+                .get('translation.shared'))
+
             import app_main
-            app_main.setup_bootstrap_path(tmp_pypy_c)  # stdlib not found
+            app_main.setup_bootstrap_path(tmp_pypy_c)
             assert sys.executable == ''
-            assert sys.path == old_sys_path + [self.goal_dir]
+            if not expected_found:
+                assert sys.path == old_sys_path + [self.goal_dir]
 
             app_main.setup_bootstrap_path(self.fake_exe)
             if not sys.platform == 'win32':
                 # an existing file is always 'executable' on windows
                 assert sys.executable == ''      # not executable!
-                assert sys.path == old_sys_path + [self.goal_dir]
+                if not expected_found:
+                    assert sys.path == old_sys_path + [self.goal_dir]
 
             os.chmod(self.fake_exe, 0755)
             app_main.setup_bootstrap_path(self.fake_exe)
@@ -1046,7 +1055,8 @@
             if newpath[0].endswith('__extensions__'):
                 newpath = newpath[1:]
             # we get at least 'expected_path', and maybe more (e.g.plat-linux2)
-            assert newpath[:len(self.expected_path)] == self.expected_path
+            if not expected_found:
+                assert newpath[:len(self.expected_path)] == self.expected_path
         finally:
             sys.path[:] = old_sys_path
             os.chdir(old_cwd)
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -57,12 +57,14 @@
         def f():
             yield 2
         g = f()
+        # two arguments version
         raises(NameError, g.throw, NameError, "Error")
 
     def test_throw2(self):
         def f():
             yield 2
         g = f()
+        # single argument version
         raises(NameError, g.throw, NameError("Error"))
 
     def test_throw3(self):
@@ -221,7 +223,8 @@
         def f():
             yield 1
         g = f()
-        raises(TypeError, g.send, 1)
+        raises(TypeError, g.send)     # one argument required
+        raises(TypeError, g.send, 1)  # not started, must send None
 
     def test_generator_explicit_stopiteration(self):
         def f():
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -562,3 +562,21 @@
         res = f(10).g()
         sys.settrace(None)
         assert res == 10
+
+    def test_throw_trace_bug(self):
+        import sys
+        def f():
+            yield 5
+        gen = f()
+        assert next(gen) == 5
+        seen = []
+        def trace_func(frame, event, *args):
+            seen.append(event)
+            return trace_func
+        sys.settrace(trace_func)
+        try:
+            gen.throw(ValueError)
+        except ValueError:
+            pass
+        sys.settrace(None)
+        assert seen == ['call', 'exception', 'return']
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -109,6 +109,7 @@
 # we need two subclasses of the app-level type, one to add mapdict, and then one
 # to add del to not slow down the GC.
 
+ at specialize.memo()
 def get_unique_interplevel_subclass(space, cls):
     "NOT_RPYTHON: initialization-time only"
     assert cls.typedef.acceptable_as_base_class
@@ -119,7 +120,6 @@
         assert cls not in _unique_subclass_cache
         _unique_subclass_cache[cls] = subcls
         return subcls
-get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo"
 _unique_subclass_cache = {}
 
 def _getusercls(cls, reallywantdict=False):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.2"
+VERSION = "1.8.4"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rarithmetic import ovfcheck
 
 from pypy.module._cffi_backend import cdataobj
-from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer
 from pypy.module._cffi_backend import ctypeprim
 
 
@@ -22,6 +22,7 @@
     is_nonfunc_pointer_or_array = True
 
     def __init__(self, space, ctptr, length, arraysize, extra):
+        assert isinstance(ctptr, W_CTypePointer)
         W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
                                    ctptr.ctitem)
         self.length = length
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -35,8 +35,7 @@
         assert isinstance(ellipsis, bool)
         extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi)
         size = rffi.sizeof(rffi.VOIDP)
-        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult,
-                                could_cast_anything=False)
+        W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult)
         self.fargs = fargs
         self.ellipsis = ellipsis
         self.abi = abi
@@ -59,6 +58,16 @@
                     lltype.free(self.cif_descr, flavor='raw')
                     self.cif_descr = lltype.nullptr(CIF_DESCRIPTION)
 
+    def is_unichar_ptr_or_array(self):
+        return False
+
+    def is_char_or_unichar_ptr_or_array(self):
+        return False
+
+    def string(self, cdataobj, maxlen):
+        # Can't use ffi.string() on a function pointer
+        return W_CType.string(self, cdataobj, maxlen)
+
     def new_ctypefunc_completing_argtypes(self, args_w):
         space = self.space
         nargs_declared = len(self.fargs)
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -19,7 +19,6 @@
     # XXX this could be improved with an elidable method get_size()
     # that raises in case it's still -1...
 
-    cast_anything = False
     is_primitive_integer = False
     is_nonfunc_pointer_or_array = False
     is_indirect_arg_for_call_python = False
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -120,7 +120,6 @@
 
 class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar):
     _attrs_ = []
-    cast_anything = True
 
     def cast_to_int(self, cdata):
         return self.space.wrap(ord(cdata[0]))
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,12 +14,11 @@
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+    _attrs_            = ['ctitem', 'accept_str', 'length']
+    _immutable_fields_ = ['ctitem', 'accept_str', 'length']
     length = -1
 
-    def __init__(self, space, size, extra, extra_position, ctitem,
-                 could_cast_anything=True):
+    def __init__(self, space, size, extra, extra_position, ctitem):
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
@@ -27,10 +26,11 @@
         #  - for arrays, it is the array item type
         #  - for functions, it is the return type
         self.ctitem = ctitem
-        self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.accept_str = (self.can_cast_anything or
-                            (ctitem.is_primitive_integer and
-                             ctitem.size == rffi.sizeof(lltype.Char)))
+        self.accept_str = (self.is_nonfunc_pointer_or_array and
+                (isinstance(ctitem, ctypevoid.W_CTypeVoid) or
+                 isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or
+                 (ctitem.is_primitive_integer and
+                  ctitem.size == rffi.sizeof(lltype.Char))))
 
     def is_unichar_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -137,7 +137,10 @@
 
 class W_CTypePtrBase(W_CTypePtrOrArray):
     # base class for both pointers and pointers-to-functions
-    _attrs_ = []
+    _attrs_ = ['is_void_ptr', 'is_voidchar_ptr']
+    _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr']
+    is_void_ptr = False
+    is_voidchar_ptr = False
 
     def convert_to_object(self, cdata):
         ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0]
@@ -154,7 +157,16 @@
             else:
                 raise self._convert_error("compatible pointer", w_ob)
         if self is not other:
-            if not (self.can_cast_anything or other.can_cast_anything):
+            if self.is_void_ptr or other.is_void_ptr:
+                pass     # cast from or to 'void *'
+            elif self.is_voidchar_ptr or other.is_voidchar_ptr:
+                space = self.space
+                msg = ("implicit cast from '%s' to '%s' "
+                    "will be forbidden in the future (check that the types "
+                    "are as you expect; use an explicit ffi.cast() if they "
+                    "are correct)" % (other.name, self.name))
+                space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1)
+            else:
                 raise self._convert_error("compatible pointer", w_ob)
 
         rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr()
@@ -165,8 +177,8 @@
 
 
 class W_CTypePointer(W_CTypePtrBase):
-    _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types']
-    _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr']
+    _attrs_ = ['is_file', 'cache_array_type', '_array_types']
+    _immutable_fields_ = ['is_file', 'cache_array_type?']
     kind = "pointer"
     cache_array_type = None
     is_nonfunc_pointer_or_array = True
@@ -181,6 +193,8 @@
         self.is_file = (ctitem.name == "struct _IO_FILE" or
                         ctitem.name == "FILE")
         self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
+        self.is_voidchar_ptr = (self.is_void_ptr or
+                           isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar))
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init, allocator):
diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py
--- a/pypy/module/_cffi_backend/ctypevoid.py
+++ b/pypy/module/_cffi_backend/ctypevoid.py
@@ -7,7 +7,6 @@
 
 class W_CTypeVoid(W_CType):
     _attrs_ = []
-    cast_anything = True
     kind = "void"
 
     def __init__(self, space):
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -112,29 +112,7 @@
 #define _WIN32_WINNT 0x0501
 #include <windows.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  _MAX_PATH
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    HMODULE hModule = 0;
-    DWORD res;
-
-    GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | 
-                       GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-                       (LPCTSTR)&_cffi_init, &hModule);
-
-    if (hModule == 0 ) {
-        _cffi_init_error("GetModuleHandleEx() failed", "");
-        return -1;
-    }
-    res = GetModuleFileName(hModule, output_home_path, CFFI_INIT_HOME_PATH_MAX);
-    if (res >= CFFI_INIT_HOME_PATH_MAX) {
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -155,28 +133,9 @@
 else:
 
     do_includes = r"""
-#include <dlfcn.h>
 #include <pthread.h>
 
-#define CFFI_INIT_HOME_PATH_MAX  PATH_MAX
 static void _cffi_init(void);
-static void _cffi_init_error(const char *msg, const char *extra);
-
-static int _cffi_init_home(char *output_home_path)
-{
-    Dl_info info;
-    dlerror();   /* reset */
-    if (dladdr(&_cffi_init, &info) == 0) {
-        _cffi_init_error("dladdr() failed: ", dlerror());
-        return -1;
-    }
-    if (realpath(info.dli_fname, output_home_path) == NULL) {
-        perror("realpath() failed");
-        _cffi_init_error("realpath() failed", "");
-        return -1;
-    }
-    return 0;
-}
 
 static void _cffi_init_once(void)
 {
@@ -201,14 +160,10 @@
 
 static void _cffi_init(void)
 {
-    char home[CFFI_INIT_HOME_PATH_MAX + 1];
-
     rpython_startup_code();
     RPyGilAllocate();
 
-    if (_cffi_init_home(home) != 0)
-        return;
-    if (pypy_setup_home(home, 1) != 0) {
+    if (pypy_setup_home(NULL, 1) != 0) {
         _cffi_init_error("pypy_setup_home() failed", "");
         return;
     }
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -32,8 +32,8 @@
 @unwrap_spec(w_cdata=cdataobj.W_CData)
 def from_handle(space, w_cdata):
     ctype = w_cdata.ctype
-    if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or
-        not ctype.can_cast_anything):
+    if (not isinstance(ctype, ctypeptr.W_CTypePointer) or
+        not ctype.is_voidchar_ptr):
         raise oefmt(space.w_TypeError,
                     "expected a 'cdata' object with a 'void *' out of "
                     "new_handle(), got '%s'", ctype.name)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3665,3 +3665,27 @@
     check_dir(pp, [])
     check_dir(pp[0], ['a1', 'a2'])
     check_dir(pp[0][0], ['a1', 'a2'])
+
+def test_char_pointer_conversion():
+    import warnings
+    assert __version__.startswith(("1.8", "1.9")), (
+        "consider turning the warning into an error")
+    BCharP = new_pointer_type(new_primitive_type("char"))
+    BIntP = new_pointer_type(new_primitive_type("int"))
+    BVoidP = new_pointer_type(new_void_type())
+    z1 = cast(BCharP, 0)
+    z2 = cast(BIntP, 0)
+    z3 = cast(BVoidP, 0)
+    with warnings.catch_warnings(record=True) as w:
+        newp(new_pointer_type(BIntP), z1)    # warn
+        assert len(w) == 1
+        newp(new_pointer_type(BVoidP), z1)   # fine
+        assert len(w) == 1
+        newp(new_pointer_type(BCharP), z2)   # warn
+        assert len(w) == 2
+        newp(new_pointer_type(BVoidP), z2)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BCharP), z3)   # fine
+        assert len(w) == 2
+        newp(new_pointer_type(BIntP), z3)    # fine
+        assert len(w) == 2
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -503,3 +503,15 @@
         assert ffi.unpack(p+1, 7) == b"bc\x00def\x00"
         p = ffi.new("int[]", [-123456789])
         assert ffi.unpack(p, 1) == [-123456789]
+
+    def test_bug_1(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        q = ffi.new("char[]", b"abcd")
+        p = ffi.cast("char(*)(void)", q)
+        raises(TypeError, ffi.string, p)
+
+    def test_negative_array_size(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        raises(ffi.error, ffi.cast, "int[-5]", 0)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -8,7 +8,7 @@
 
 @unwrap_spec(cdef=str, module_name=str, source=str)
 def prepare(space, cdef, module_name, source, w_includes=None,
-            w_extra_source=None):
+            w_extra_source=None, w_min_version=None):
     try:
         import cffi
         from cffi import FFI            # <== the system one, which
@@ -16,8 +16,13 @@
         from cffi import ffiplatform
     except ImportError:
         py.test.skip("system cffi module not found or older than 1.0.0")
-    if cffi.__version_info__ < (1, 4, 0):
-        py.test.skip("system cffi module needs to be at least 1.4.0")
+    if w_min_version is None:
+        min_version = (1, 4, 0)
+    else:
+        min_version = tuple(space.unwrap(w_min_version))
+    if cffi.__version_info__ < min_version:
+        py.test.skip("system cffi module needs to be at least %s, got %s" % (
+            min_version, cffi.__version_info__))
     space.appexec([], """():
         import _cffi_backend     # force it to be initialized
     """)
@@ -1790,3 +1795,28 @@
                                 "void f(void) { }")
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
+
+    def test_typedef_array_dotdotdot(self):
+        ffi, lib = self.prepare("""
+            typedef int foo_t[...], bar_t[...];
+            int gv[...];
+            typedef int mat_t[...][...];
+            typedef int vmat_t[][...];
+            """,
+            "test_typedef_array_dotdotdot", """
+            typedef int foo_t[50], bar_t[50];
+            int gv[23];
+            typedef int mat_t[6][7];
+            typedef int vmat_t[][8];
+        """, min_version=(1, 8, 4))
+        assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int")
+        assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int")
+        assert len(ffi.new("foo_t")) == 50
+        assert len(ffi.new("bar_t")) == 50
+        assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int")
+        assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int")
+        assert len(ffi.new("mat_t")) == 6
+        assert len(ffi.new("mat_t")[3]) == 7
+        raises(ffi.error, ffi.sizeof, "vmat_t")
+        p = ffi.new("vmat_t", 4)
+        assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int")
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.debug import check_nonneg
+from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
@@ -316,12 +317,12 @@
             w_currently_in_repr = ec._py_repr = space.newdict()
         return dequerepr(space, w_currently_in_repr, space.wrap(self))
 
+    @specialize.arg(2)
     def compare(self, w_other, op):
         space = self.space
         if not isinstance(w_other, W_Deque):
             return space.w_NotImplemented
         return space.compare_by_iteration(space.wrap(self), w_other, op)
-    compare._annspecialcase_ = 'specialize:arg(2)'
 
     def lt(self, w_other):
         return self.compare(w_other, 'lt')
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec
@@ -25,12 +26,12 @@
     def iter_w(self):
         return self.space.wrap(self)
 
+    @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
         w_error = space.getattr(w_module, space.wrap('Error'))
         raise oefmt(w_error, "line %d: %s", self.line_num, msg)
-    error._dont_inline_ = True
 
     def add_char(self, field_builder, c):
         assert field_builder is not None
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -1,4 +1,5 @@
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib import objectmodel
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.typedef import TypeDef, interp2app
@@ -21,12 +22,12 @@
             special += dialect.quotechar
         self.special_characters = special
 
+    @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
         w_error = space.getattr(w_module, space.wrap('Error'))
         raise OperationError(w_error, space.wrap(msg))
-    error._dont_inline_ = True
 
     def writerow(self, w_fields):
         """Construct and write a CSV record from a sequence of fields.
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -7,7 +7,7 @@
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
                                       interp_attrproperty)
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, always_inline
 from rpython.rlib.rtimer import read_timestamp, _is_64_bit
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
@@ -256,7 +256,7 @@
         return w_frame.wrap_string(space)
     return w_frame    # actually a PyCode object
 
-
+ at always_inline
 def prepare_spec(space, w_arg):
     if isinstance(w_arg, Method):
         return (w_arg.w_function, w_arg.w_class)
@@ -264,8 +264,6 @@
         return (w_arg, None)
     else:
         return (None, space.type(w_arg))
-prepare_spec._always_inline_ = True
-
 
 def lsprof_call(space, w_self, frame, event, w_arg):
     assert isinstance(w_self, W_Profiler)
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1,6 +1,6 @@
 import sys
 from rpython.rlib.rstring import StringBuilder
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, always_inline
 from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import oefmt
@@ -188,6 +188,7 @@
         self.pos = i
         return self.space.call_function(self.space.w_int, self.space.wrap(s))
 
+    @always_inline
     def parse_integer(self, i):
         "Parse a decimal number with an optional minus sign"
         sign = 1
@@ -218,7 +219,6 @@
         # overflowed
         ovf_maybe = (count >= OVF_DIGITS)
         return i, ovf_maybe, sign * intval
-    parse_integer._always_inline_ = True
 
     def decode_array(self, i):
         w_list = self.space.newlist([])
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -6,6 +6,7 @@
 import time
 from pypy.interpreter.error import OperationError
 from pypy.module._pypyjson.interp_decoder import loads
+from rpython.rlib.objectmodel import specialize, dont_inline
 
 
 ## MSG = open('msg.json').read()
@@ -68,11 +69,11 @@
         assert isinstance(w_x, W_String)
         return w_x.strval
 
+    @dont_inline
     def call_method(self, obj, name, arg):
         assert name == 'append'
         assert isinstance(obj, W_List)
         obj.listval.append(arg)
-    call_method._dont_inline_ = True
 
     def call_function(self, w_func, *args_w):
         return self.w_None # XXX
@@ -91,6 +92,7 @@
     def wrapfloat(self, x):
         return W_Float(x)
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if isinstance(x, int):
             return W_Int(x)
@@ -100,7 +102,6 @@
         ##     assert False
         else:
             return W_Unicode(unicode(x))
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
 
 fakespace = FakeSpace()
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -9,6 +9,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.tool import rffi_platform
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 import rpython.rlib.rposix as rposix
 
 _MS_WINDOWS = os.name == "nt"
@@ -255,6 +256,7 @@
 
 _ARM = rffi_platform.getdefined('__arm__', '')
 
+ at specialize.arg(2)
 def read_ptr(ptr, ofs, TP):
     T = lltype.Ptr(rffi.CArray(TP))
     for c in unroll_letters_for_floats:
@@ -274,8 +276,8 @@
                 return ptr_val
     else:
         return rffi.cast(T, ptr)[ofs]
-read_ptr._annspecialcase_ = 'specialize:arg(2)'
 
+ at specialize.argtype(2)
 def write_ptr(ptr, ofs, value):
     TP = lltype.typeOf(value)
     T = lltype.Ptr(rffi.CArray(TP))
@@ -296,7 +298,6 @@
                 return
     else:
         rffi.cast(T, ptr)[ofs] = value
-write_ptr._annspecialcase_ = 'specialize:argtype(2)'
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
@@ -374,14 +375,15 @@
     def getrawsize(self):
         raise NotImplementedError("abstract base class")
 
+ at specialize.arg(0)
 def unwrap_truncate_int(TP, space, w_arg):
     if space.isinstance_w(w_arg, space.w_int):
         return rffi.cast(TP, space.int_w(w_arg))
     else:
         return rffi.cast(TP, space.bigint_w(w_arg).ulonglongmask())
-unwrap_truncate_int._annspecialcase_ = 'specialize:arg(0)'
 
 
+ at specialize.arg(1)
 def unwrap_value(space, push_func, add_arg, argdesc, letter, w_arg):
     if letter in TYPEMAP_PTR_LETTERS:
         # check for NULL ptr
@@ -422,10 +424,10 @@
                 return
         else:
             raise oefmt(space.w_TypeError, "cannot directly write value")
-unwrap_value._annspecialcase_ = 'specialize:arg(1)'
 
 ll_typemap_iter = unrolling_iterable(LL_TYPEMAP.items())
 
+ at specialize.arg(1)
 def wrap_value(space, func, add_arg, argdesc, letter):
     for c, ll_type in ll_typemap_iter:
         if letter == c:
@@ -437,7 +439,6 @@
             else:
                 return space.wrap(func(add_arg, argdesc, ll_type))
     raise oefmt(space.w_TypeError, "cannot directly read value")
-wrap_value._annspecialcase_ = 'specialize:arg(1)'
 
 NARROW_INTEGER_TYPES = 'cbhiBIH?'
 
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, signedtype, r_uint, \
     r_ulonglong
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import specialize
 import sys
 
 IS_BIG_ENDIAN = sys.byteorder == 'big'
@@ -284,6 +285,7 @@
 def NUM_BITS(x):
     return x >> 16
 
+ at specialize.arg(1)
 def BIT_MASK(x, ll_t):
     if ll_t is lltype.SignedLongLong or ll_t is lltype.UnsignedLongLong:
         one = r_ulonglong(1)
@@ -291,8 +293,8 @@
         one = r_uint(1)
     # to avoid left shift by x == sizeof(ll_t)
     return (((one << (x - 1)) - 1) << 1) + 1
-BIT_MASK._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.argtype(2)
 def push_field(self, num, value):
     ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num])
     TP = lltype.typeOf(value)
@@ -313,8 +315,8 @@
                 value = rffi.cast(TP, current)
             break
     write_ptr(ptr, 0, value)
-push_field._annspecialcase_ = 'specialize:argtype(2)'
 
+ at specialize.arg(2)
 def cast_pos(self, i, ll_t):
     pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
     value = read_ptr(pos, 0, ll_t)
@@ -337,7 +339,6 @@
                 value = rffi.cast(ll_t, value)
             break
     return value
-cast_pos._annspecialcase_ = 'specialize:arg(2)'
 
 class W_StructureInstance(W_DataInstance):
     def __init__(self, space, shape, address):
diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -248,6 +248,10 @@
     if space.isinstance_w(w_message, space.w_Warning):
         w_text = space.str(w_message)
         w_category = space.type(w_message)
+    elif (not space.isinstance_w(w_message, space.w_unicode) or
+          not space.isinstance_w(w_message, space.w_str)):
+        w_text = space.str(w_message)
+        w_message = space.call_function(w_category, w_message)
     else:
         w_text = w_message
         w_message = space.call_function(w_category, w_message)
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -11,6 +11,7 @@
         import _warnings
         _warnings.warn("some message", DeprecationWarning)
         _warnings.warn("some message", Warning)
+        _warnings.warn(("some message",1), Warning)
 
     def test_lineno(self):
         import warnings, _warnings, sys
@@ -40,7 +41,10 @@
     def test_show_source_line(self):
         import warnings
         import sys, StringIO
-        from test.warning_tests import inner
+        try:
+            from test.warning_tests import inner
+        except ImportError:
+            skip('no test, -A on cpython?')
         # With showarning() missing, make sure that output is okay.
         del warnings.showwarning
 
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -34,21 +34,21 @@
         if typecode == tc:
             a = space.allocate_instance(types[tc].w_class, w_cls)
             a.__init__(space)
-
-            if len(__args__.arguments_w) > 0:
-                w_initializer = __args__.arguments_w[0]
-                if space.type(w_initializer) is space.w_str:
-                    a.descr_fromstring(space, w_initializer)
-                elif space.type(w_initializer) is space.w_list:
-                    a.descr_fromlist(space, w_initializer)
-                else:
-                    a.extend(w_initializer, True)
             break
     else:
         raise oefmt(space.w_ValueError,
                     "bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or "
                     "d)")
 
+    if len(__args__.arguments_w) > 0:
+        w_initializer = __args__.arguments_w[0]
+        w_initializer_type = space.type(w_initializer)
+        if w_initializer_type is space.w_str:
+            a.descr_fromstring(space, w_initializer)
+        elif w_initializer_type is space.w_list:
+            a.descr_fromlist(space, w_initializer)
+        else:
+            a.extend(w_initializer, True)
     return a
 
 
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,7 +6,6 @@
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rarithmetic import intmask
 
 
 PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
@@ -28,25 +27,25 @@
     """Return a new PyLongObject object from a C size_t, or NULL on
     failure.
     """
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.LONGLONG], PyObject)
 def PyLong_FromLongLong(space, val):
     """Return a new PyLongObject object from a C long long, or NULL
     on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONG], PyObject)
 def PyLong_FromUnsignedLong(space, val):
     """Return a new PyLongObject object from a C unsigned long, or
     NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([rffi.ULONGLONG], PyObject)
 def PyLong_FromUnsignedLongLong(space, val):
     """Return a new PyLongObject object from a C unsigned long long,
     or NULL on failure."""
-    return space.wrap(val)
+    return space.newlong_from_rarith_int(val)
 
 @cpython_api([PyObject], rffi.ULONG, error=-1)
 def PyLong_AsUnsignedLong(space, w_long):
@@ -203,7 +202,10 @@
     can be retrieved from the resulting value using PyLong_AsVoidPtr().
 
     If the integer is larger than LONG_MAX, a positive long integer is returned."""
-    return space.wrap(rffi.cast(ADDR, p))
+    value = rffi.cast(ADDR, p)    # signed integer
+    if value < 0:
+        return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
+    return space.wrap(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -53,10 +53,7 @@
     else:
         n = len(fmt)
     for i in range(n):
-        if ord(fmt[i]) > 255:
-            view.c_format[i] = '*'
-        else:
-            view.c_format[i] = fmt[i]
+        view.c_format[i] = fmt[i]
     view.c_format[n] = '\x00'        
     shape = buf.getshape()
     strides = buf.getstrides()
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -5,7 +5,6 @@
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
 from pypy.module.cpyext.frameobject import PyFrameObject
-from rpython.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pytraceback import PyTraceback
 from pypy.interpreter import pycode
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -214,7 +214,9 @@
     i = space.int_w(space.index(args_w[0]))
     j = space.int_w(space.index(args_w[1]))
     w_y = args_w[2]
-    return space.wrap(generic_cpy_call(space, func_target, w_self, i, j, w_y))
+    res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
+    if rffi.cast(lltype.Signed, res) == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
 
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
@@ -296,7 +298,10 @@
 def wrap_hashfunc(space, w_self, w_args, func):
     func_target = rffi.cast(hashfunc, func)
     check_num_args(space, w_args, 0)
-    return space.wrap(generic_cpy_call(space, func_target, w_self))
+    res = generic_cpy_call(space, func_target, w_self)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return space.wrap(res)
 
 class CPyBuffer(Buffer):
     # Similar to Py_buffer
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,9 +4,7 @@
 PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
 {
     PyObject* newType;
-    /*printf("in foo3type_tp_new, preprocessing...\n"); */
     newType = PyType_Type.tp_new(metatype, args, kwds);
-    /*printf("in foo3type_tp_new, postprocessing...\n"); */
     return newType;
 }
 
@@ -81,4 +79,5 @@
         return;
     if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
         return;
+    Py_INCREF(&footype);
 }
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -49,6 +49,7 @@
         assert arr.tolist() == [1, 21, 22, 23, 4]
         del arr[slice(1, 3)]
         assert arr.tolist() == [1, 23, 4]
+        raises(TypeError, 'arr[slice(1, 3)] = "abc"')
 
     def test_buffer(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -1,5 +1,6 @@
 import sys, py
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import maxint
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
@@ -8,18 +9,20 @@
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
-        value = api.PyLong_FromLong(3)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == 3
+        w_value = api.PyLong_FromLong(3)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == 3
 
-        value = api.PyLong_FromLong(sys.maxint)
-        assert isinstance(value, W_LongObject)
-        assert space.unwrap(value) == sys.maxint
+        w_value = api.PyLong_FromLong(sys.maxint)
+        assert isinstance(w_value, W_LongObject)
+        assert space.unwrap(w_value) == sys.maxint
 
     def test_aslong(self, space, api):
         w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+        assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsLong(w_value)
         assert value == (sys.maxint - 1)
 
@@ -35,12 +38,16 @@
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
+        assert isinstance(w_value, W_LongObject)
         value = api.PyLong_AsSsize_t(w_value)
         assert value == 2
-        assert space.eq_w(w_value, api.PyLong_FromSsize_t(2))
+        w_val2 = api.PyLong_FromSsize_t(2)
+        assert isinstance(w_val2, W_LongObject)
+        assert space.eq_w(w_value, w_val2)
 
     def test_fromdouble(self, space, api):
         w_value = api.PyLong_FromDouble(-12.74)
+        assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == -12
         assert api.PyLong_AsDouble(w_value) == -12
 
@@ -102,9 +109,26 @@
         lltype.free(overflow, flavor='raw')
 
     def test_as_voidptr(self, space, api):
+        # CPython returns an int (not a long) depending on the value
+        # passed to PyLong_FromVoidPtr().  In all cases, NULL becomes
+        # the int 0.
         w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO))
-        assert space.unwrap(w_l) == 0L
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == 0
         assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO)
+        # Positive values also return an int (assuming, like always in
+        # PyPy, that an int is big enough to store any pointer).
+        p = rffi.cast(rffi.VOIDP, maxint)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_int)
+        assert space.unwrap(w_l) == maxint
+        assert api.PyLong_AsVoidPtr(w_l) == p
+        # Negative values always return a long.
+        p = rffi.cast(rffi.VOIDP, -maxint-1)
+        w_l = api.PyLong_FromVoidPtr(p)
+        assert space.is_w(space.type(w_l), space.w_long)
+        assert space.unwrap(w_l) == maxint+1
+        assert api.PyLong_AsVoidPtr(w_l) == p
 
     def test_sign_and_bits(self, space, api):
         if space.is_true(space.lt(space.sys.get('version_info'),
@@ -128,23 +152,58 @@
         module = self.import_extension('foo', [
             ("from_unsignedlong", "METH_NOARGS",
              """
-                 return PyLong_FromUnsignedLong((unsigned long)-1);
+                 PyObject * obj;
+                 obj = PyLong_FromUnsignedLong((unsigned long)-1);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
         import sys
         assert module.from_unsignedlong() == 2 * sys.maxint + 1
 
     def test_fromlonglong(self):
         module = self.import_extension('foo', [
-            ("from_longlong", "METH_NOARGS",
+            ("from_longlong", "METH_VARARGS",
              """
-                 return PyLong_FromLongLong((long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """),
-            ("from_unsignedlonglong", "METH_NOARGS",
+            ("from_unsignedlonglong", "METH_VARARGS",
              """
-                 return PyLong_FromUnsignedLongLong((unsigned long long)-1);
+                 int val;
+                 PyObject * obj;
+                 if (!PyArg_ParseTuple(args, "i", &val))
+                     return NULL;
+                 obj = PyLong_FromUnsignedLongLong((long long)val);
+                 if (obj->ob_type != &PyLong_Type)
+                 {
+                    Py_DECREF(obj);
+                    PyErr_SetString(PyExc_ValueError,
+                            "PyLong_FromLongLong did not return PyLongObject");
+                    return NULL;
+                 }
+                 return obj;
              """)])
-        assert module.from_longlong() == -1
-        assert module.from_unsignedlonglong() == (1<<64) - 1
+        assert module.from_longlong(-1) == -1
+        assert module.from_longlong(0) == 0
+        assert module.from_unsignedlonglong(0) == 0
+        assert module.from_unsignedlonglong(-1) == (1<<64) - 1
 
     def test_from_size_t(self):
         module = self.import_extension('foo', [
@@ -232,10 +291,15 @@
             ("has_sub", "METH_NOARGS",
              """
                 PyObject *ret, *obj = PyLong_FromLong(42);
-                if (obj->ob_type->tp_as_number->nb_subtract)
-                    ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                if (obj->ob_type != &PyLong_Type)
+                    ret = PyLong_FromLong(-2);
                 else
-                    ret = PyLong_FromLong(-1);
+                {
+                    if (obj->ob_type->tp_as_number->nb_subtract)
+                        ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj);
+                    else
+                        ret = PyLong_FromLong(-1);
+                }
                 Py_DECREF(obj);
                 return ret;
              """),
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -961,7 +961,6 @@
 
     def test_tp_new_in_subclass_of_type(self):
         module = self.import_module(name='foo3')
-        #print('calling module.footype()...')
         module.footype("X", (object,), {})
 
     def test_app_subclass_of_c_type(self):
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -2,6 +2,7 @@
 import sys
 
 from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError, oefmt
 
 class State:
@@ -17,6 +18,7 @@
     else:
         return space.float_w(space.float(w_x))
 
+ at specialize.arg(1)
 def math1(space, f, w_x):
     x = _get_double(space, w_x)
     try:
@@ -26,8 +28,8 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return space.wrap(y)
-math1._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.arg(1)
 def math1_w(space, f, w_x):
     x = _get_double(space, w_x)
     try:
@@ -37,8 +39,8 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return r
-math1_w._annspecialcase_ = 'specialize:arg(1)'
 
+ at specialize.arg(1)
 def math2(space, f, w_x, w_snd):
     x = _get_double(space, w_x)
     snd = _get_double(space, w_snd)
@@ -49,7 +51,6 @@
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
     return space.wrap(r)
-math2._annspecialcase_ = 'specialize:arg(1)'
 
 def trunc(space, w_x):
     """Truncate x."""
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -2,7 +2,7 @@

From pypy.commits at gmail.com  Fri Sep 30 13:54:11 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 30 Sep 2016 10:54:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merge branch 'test-cpyext'
Message-ID: <57eea6c3.4b2e1c0a.41552.ce6d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r87478:298b37893194
Date: 2016-09-30 18:53 +0100
http://bitbucket.org/pypy/pypy/changeset/298b37893194/

Log:	Merge branch 'test-cpyext'

diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -2,6 +2,8 @@
 import pytest
 
 def pytest_configure(config):
+    if config.option.runappdirect:
+        return
     from pypy.tool.pytest.objspace import gettestobjspace
     # For some reason (probably a ll2ctypes cache issue on linux64)
     # it's necessary to run "import time" at least once before any
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -5,7 +5,7 @@
 import py, pytest
 
 from pypy import pypydir
-from pypy.interpreter import gateway
+from pypy.interpreter.gateway import unwrap_spec, interp2app
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
@@ -18,7 +18,8 @@
 
 from .support import c_compile
 
-only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" 
+HERE = py.path.local(pypydir) / 'module' / 'cpyext' / 'test'
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):
@@ -42,19 +43,6 @@
         files.append(filename)
     return files
 
-def create_so(modname, include_dirs, source_strings=None, source_files=None,
-        compile_extra=None, link_extra=None, libraries=None):
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    if source_strings:
-        assert not source_files
-        files = convert_sources_to_files(source_strings, dirname)
-        source_files = files
-    soname = c_compile(source_files, outputfilename=str(dirname/modname),
-        compile_extra=compile_extra, link_extra=link_extra,
-        include_dirs=include_dirs,
-        libraries=libraries)
-    return soname
-
 class SystemCompilationInfo(object):
     """Bundles all the generic information required to compile extensions.
 
@@ -68,6 +56,88 @@
         self.extra_libs = extra_libs
         self.ext = ext
 
+    def compile_extension_module(self, name, include_dirs=None,
+            source_files=None, source_strings=None):
+        """
+        Build an extension module and return the filename of the resulting
+        native code file.
+
+        name is the name of the module, possibly including dots if it is a
+        module inside a package.
+
+        Any extra keyword arguments are passed on to ExternalCompilationInfo to
+        build the module (so specify your source with one of those).
+        """
+        include_dirs = include_dirs or []
+        modname = name.split('.')[-1]
+        dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+        if source_strings:
+            assert not source_files
+            files = convert_sources_to_files(source_strings, dirname)
+            source_files = files
+        soname = c_compile(source_files, outputfilename=str(dirname/modname),
+            compile_extra=self.compile_extra,
+            link_extra=self.link_extra,
+            include_dirs=self.include_extra + include_dirs,
+            libraries=self.extra_libs)
+        pydname = soname.new(purebasename=modname, ext=self.ext)
+        soname.rename(pydname)
+        return str(pydname)
+
+    def import_module(self, name, init=None, body='', filename=None,
+            include_dirs=None, PY_SSIZE_T_CLEAN=False):
+        """
+        init specifies the overall template of the module.
+
+        if init is None, the module source will be loaded from a file in this
+        test directory, give a name given by the filename parameter.
+
+        if filename is None, the module name will be used to construct the
+        filename.
+        """
+        if init is not None:
+            code = make_source(name, init, body, PY_SSIZE_T_CLEAN)
+            kwds = dict(source_strings=[code])
+        else:
+            assert not PY_SSIZE_T_CLEAN
+            if filename is None:
+                filename = name
+            filename = HERE / (filename + ".c")
+            kwds = dict(source_files=[filename])
+        mod = self.compile_extension_module(
+            name, include_dirs=include_dirs, **kwds)
+        return self.load_module(mod, name)
+
+    def import_extension(self, modname, functions, prologue="",
+            include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
+        body = prologue + make_methods(functions, modname)
+        init = """Py_InitModule("%s", methods);""" % (modname,)
+        if more_init:
+            init += more_init
+        return self.import_module(
+            name=modname, init=init, body=body, include_dirs=include_dirs,
+            PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+
+
+class ExtensionCompiler(SystemCompilationInfo):
+    """Extension compiler for appdirect mode"""
+    def load_module(space, mod, name):
+        import imp
+        return imp.load_dynamic(name, mod)
+
+class SpaceCompiler(SystemCompilationInfo):
+    """Extension compiler for regular (untranslated PyPy) mode"""
+    def __init__(self, space, *args, **kwargs):
+        self.space = space
+        SystemCompilationInfo.__init__(self, *args, **kwargs)
+
+    def load_module(self, mod, name):
+        space = self.space
+        api.load_extension_module(space, mod, name)
+        return space.getitem(
+            space.sys.get('modules'), space.wrap(name))
+
+
 def get_cpyext_info(space):
     from pypy.module.imp.importing import get_so_extension
     state = space.fromcache(State)
@@ -88,7 +158,7 @@
             link_extra = ["-g"]
         else:
             compile_extra = link_extra = None
-    return SystemCompilationInfo(
+    return SpaceCompiler(space,
         include_extra=api.include_dirs,
         compile_extra=compile_extra,
         link_extra=link_extra,
@@ -96,30 +166,6 @@
         ext=get_so_extension(space))
 
 
-def compile_extension_module(sys_info, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-        include_dirs=sys_info.include_extra + include_dirs,
-        source_files=source_files,
-        source_strings=source_strings,
-        compile_extra=sys_info.compile_extra,
-        link_extra=sys_info.link_extra,
-        libraries=sys_info.extra_libs)
-    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
-    soname.rename(pydname)
-    return str(pydname)
-
 def get_so_suffix():
     from imp import get_suffixes, C_EXTENSION
     for suffix, mode, typ in get_suffixes():
@@ -142,12 +188,60 @@
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
         link_extra = None
     ext = get_so_suffix()
-    return SystemCompilationInfo(
+    return ExtensionCompiler(
         include_extra=[get_python_inc()],
         compile_extra=compile_extra,
         link_extra=link_extra,
         ext=get_so_suffix())
 
+def make_methods(functions, modname):
+    methods_table = []
+    codes = []
+    for funcname, flags, code in functions:
+        cfuncname = "%s_%s" % (modname, funcname)
+        methods_table.append(
+            "{\"%s\", %s, %s}," % (funcname, cfuncname, flags))
+        func_code = """
+        static PyObject* %s(PyObject* self, PyObject* args)
+        {
+        %s
+        }
+        """ % (cfuncname, code)
+        codes.append(func_code)
+
+    body = "\n".join(codes) + """
+    static PyMethodDef methods[] = {
+    %s
+    { NULL }
+    };
+    """ % ('\n'.join(methods_table),)
+    return body
+
+def make_source(name, init, body, PY_SSIZE_T_CLEAN):
+    code = """
+    %(PY_SSIZE_T_CLEAN)s
+    #include <Python.h>
+    /* fix for cpython 2.7 Python.h if running tests with -A
+        since pypy compiles with -fvisibility-hidden */
+    #undef PyMODINIT_FUNC
+    #ifdef __GNUC__
+    #  define RPY_EXPORTED extern __attribute__((visibility("default")))
+    #else
+    #  define RPY_EXPORTED extern __declspec(dllexport)
+    #endif
+    #define PyMODINIT_FUNC RPY_EXPORTED void
+
+    %(body)s
+
+    PyMODINIT_FUNC
+    init%(name)s(void) {
+    %(init)s
+    }
+    """ % dict(name=name, init=init, body=body,
+            PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
+                if PY_SSIZE_T_CLEAN else '')
+    return code
+
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -159,22 +253,6 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
-class FakeSpace(object):
-    """Like TinyObjSpace, but different"""
-    def __init__(self, config):
-        self.config = config
-
-    def passthrough(self, arg):
-        return arg
-    listview = passthrough
-    str_w = passthrough
-
-    def unwrap(self, args):
-        try:
-            return args.str_w(None)
-        except:
-            return args
-
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -265,9 +343,12 @@
             cls.w_libc = cls.space.wrap(get_libc_name())
 
     def setup_method(self, meth):
-        freeze_refcnts(self)
+        if not self.runappdirect:
+            freeze_refcnts(self)
 
     def teardown_method(self, meth):
+        if self.runappdirect:
+            return
         self.cleanup_references(self.space)
         # XXX: like AppTestCpythonExtensionBase.teardown_method:
         # find out how to disable check_and_print_leaks() if the
@@ -293,21 +374,42 @@
             skip("Windows Python >= 2.6 only")
         assert isinstance(sys.dllhandle, int)
 
+
+def _unwrap_include_dirs(space, w_include_dirs):
+    if w_include_dirs is None:
+        return None
+    else:
+        return [space.str_w(s) for s in space.listview(w_include_dirs)]
+
+def debug_collect(space):
+    rawrefcount._collect()
+
 class AppTestCpythonExtensionBase(LeakCheckingTest):
 
     def setup_class(cls):
         space = cls.space
-        space.getbuiltinmodule("cpyext")
-        # 'import os' to warm up reference counts
-        w_import = space.builtin.getdictvalue(space, '__import__')
-        space.call_function(w_import, space.wrap("os"))
-        #state = cls.space.fromcache(RefcountState) ZZZ
-        #state.non_heaptypes_w[:] = []
+        cls.w_here = space.wrap(str(HERE))
         if not cls.runappdirect:
             cls.w_runappdirect = space.wrap(cls.runappdirect)
+            space.getbuiltinmodule("cpyext")
+            # 'import os' to warm up reference counts
+            w_import = space.builtin.getdictvalue(space, '__import__')
+            space.call_function(w_import, space.wrap("os"))
+            #state = cls.space.fromcache(RefcountState) ZZZ
+            #state.non_heaptypes_w[:] = []
+            cls.w_debug_collect = space.wrap(interp2app(debug_collect))
+
+    def record_imported_module(self, name):
+        """
+        Record a module imported in a test so that it can be cleaned up in
+        teardown before the check for leaks is done.
+
+        name gives the name of the module in the space's sys.modules.
+        """
+        self.imported_module_names.append(name)
 
     def setup_method(self, func):
-        @gateway.unwrap_spec(name=str)
+        @unwrap_spec(name=str)
         def compile_module(space, name,
                            w_source_files=None,
                            w_source_strings=None):
@@ -322,172 +424,68 @@
                 source_strings = space.listview_bytes(w_source_strings)
             else:
                 source_strings = None
-            pydname = compile_extension_module(
-                self.sys_info, name,
+            pydname = self.sys_info.compile_extension_module(
+                name,
                 source_files=source_files,
                 source_strings=source_strings)
+
+            # hackish, but tests calling compile_module() always end up
+            # importing the result
+            self.record_imported_module(name)
+
             return space.wrap(pydname)
 
-        @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
-                     load_it=bool, filename='str_or_None',
-                     PY_SSIZE_T_CLEAN=bool)
-        def import_module(space, name, init=None, body='', load_it=True,
+        @unwrap_spec(name=str, init='str_or_None', body=str,
+                     filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
+        def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
                           PY_SSIZE_T_CLEAN=False):
-            """
-            init specifies the overall template of the module.
+            include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+            w_result = self.sys_info.import_module(
+                name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN)
+            self.record_imported_module(name)
+            return w_result
 
-            if init is None, the module source will be loaded from a file in this
-            test direcory, give a name given by the filename parameter.
 
-            if filename is None, the module name will be used to construct the
-            filename.
-            """
-            if w_include_dirs is None:
-                include_dirs = []
-            else:
-                include_dirs = [space.str_w(s) for s in space.listview(w_include_dirs)]
-            if init is not None:
-                code = """
-                %(PY_SSIZE_T_CLEAN)s
-                #include <Python.h>
-                /* fix for cpython 2.7 Python.h if running tests with -A
-                   since pypy compiles with -fvisibility-hidden */
-                #undef PyMODINIT_FUNC
-                #ifdef __GNUC__
-                #  define RPY_EXPORTED extern __attribute__((visibility("default")))
-                #else
-                #  define RPY_EXPORTED extern __declspec(dllexport)
-                #endif
-                #define PyMODINIT_FUNC RPY_EXPORTED void
+        @unwrap_spec(mod=str, name=str)
+        def load_module(space, mod, name):
+            return self.sys_info.load_module(mod, name)
 
-                %(body)s
-
-                PyMODINIT_FUNC
-                init%(name)s(void) {
-                %(init)s
-                }
-                """ % dict(name=name, init=init, body=body,
-                           PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
-                                            if PY_SSIZE_T_CLEAN else '')
-                kwds = dict(source_strings=[code])
-            else:
-                assert not PY_SSIZE_T_CLEAN
-                if filename is None:
-                    filename = name
-                filename = py.path.local(pypydir) / 'module' \
-                        / 'cpyext'/ 'test' / (filename + ".c")
-                kwds = dict(source_files=[filename])
-            mod = compile_extension_module(self.sys_info, name,
-                    include_dirs=include_dirs, **kwds)
-
-            if load_it:
-                if self.runappdirect:
-                    import imp
-                    return imp.load_dynamic(name, mod)
-                else:
-                    api.load_extension_module(space, mod, name)
-                    self.imported_module_names.append(name)
-                    return space.getitem(
-                        space.sys.get('modules'),
-                        space.wrap(name))
-            else:
-                path = os.path.dirname(mod)
-                if self.runappdirect:
-                    return path
-                else:
-                    return space.wrap(path)
-
-        @gateway.unwrap_spec(mod=str, name=str)
-        def reimport_module(space, mod, name):
-            if self.runappdirect:
-                import imp
-                return imp.load_dynamic(name, mod)
-            else:
-                api.load_extension_module(space, mod, name)
-            return space.getitem(
-                space.sys.get('modules'),
-                space.wrap(name))
-
-        @gateway.unwrap_spec(modname=str, prologue=str,
+        @unwrap_spec(modname=str, prologue=str,
                              more_init=str, PY_SSIZE_T_CLEAN=bool)
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
-            methods_table = []
-            codes = []
-            for funcname, flags, code in functions:
-                cfuncname = "%s_%s" % (modname, funcname)
-                methods_table.append("{\"%s\", %s, %s}," %
-                                     (funcname, cfuncname, flags))
-                func_code = """
-                static PyObject* %s(PyObject* self, PyObject* args)
-                {
-                %s
-                }
-                """ % (cfuncname, code)
-                codes.append(func_code)
-
-            body = prologue + "\n".join(codes) + """
-            static PyMethodDef methods[] = {
-            %s
-            { NULL }
-            };
-            """ % ('\n'.join(methods_table),)
-            init = """Py_InitModule("%s", methods);""" % (modname,)
-            if more_init:
-                init += more_init
-            return import_module(space, name=modname, init=init, body=body,
-                                 w_include_dirs=w_include_dirs,
-                                 PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
-
-        @gateway.unwrap_spec(name=str)
-        def record_imported_module(name):
-            """
-            Record a module imported in a test so that it can be cleaned up in
-            teardown before the check for leaks is done.
-
-            name gives the name of the module in the space's sys.modules.
-            """
-            self.imported_module_names.append(name)
-
-        def debug_collect(space):
-            rawrefcount._collect()
+            include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+            w_result = self.sys_info.import_extension(
+                modname, functions, prologue, include_dirs, more_init,
+                PY_SSIZE_T_CLEAN)
+            self.record_imported_module(modname)
+            return w_result
 
         # A list of modules which the test caused to be imported (in
         # self.space).  These will be cleaned up automatically in teardown.
         self.imported_module_names = []
 
         if self.runappdirect:
-            fake = FakeSpace(self.space.config)
-            def interp2app(func):
-                def run(*args, **kwargs):
-                    for k in kwargs.keys():
-                        if k not in func.unwrap_spec and not k.startswith('w_'):
-                            v = kwargs.pop(k)
-                            kwargs['w_' + k] = v
-                    return func(fake, *args, **kwargs)
-                return run
-            def wrap(func):
-                return func
             self.sys_info = get_sys_info_app()
+            self.compile_module = self.sys_info.compile_extension_module
+            self.load_module = self.sys_info.load_module
+            self.import_module = self.sys_info.import_module
+            self.import_extension = self.sys_info.import_extension
         else:
-            interp2app = gateway.interp2app
             wrap = self.space.wrap
             self.sys_info = get_cpyext_info(self.space)
-        self.w_compile_module = wrap(interp2app(compile_module))
-        self.w_import_module = wrap(interp2app(import_module))
-        self.w_reimport_module = wrap(interp2app(reimport_module))
-        self.w_import_extension = wrap(interp2app(import_extension))
-        self.w_record_imported_module = wrap(interp2app(record_imported_module))
-        self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/')
-        self.w_debug_collect = wrap(interp2app(debug_collect))
+            self.w_compile_module = wrap(interp2app(compile_module))
+            self.w_load_module = wrap(interp2app(load_module))
+            self.w_import_module = wrap(interp2app(import_module))
+            self.w_import_extension = wrap(interp2app(import_extension))
 
-        # create the file lock before we count allocations
-        self.space.call_method(self.space.sys.get("stdout"), "flush")
+            # create the file lock before we count allocations
+            self.space.call_method(self.space.sys.get("stdout"), "flush")
 
-        freeze_refcnts(self)
-        #self.check_and_print_leaks()
+            freeze_refcnts(self)
+            #self.check_and_print_leaks()
 
     def unimport_module(self, name):
         """
@@ -498,6 +496,8 @@
         self.space.delitem(w_modules, w_name)
 
     def teardown_method(self, func):
+        if self.runappdirect:
+            return
         for name in self.imported_module_names:
             self.unimport_module(name)
         self.cleanup_references(self.space)
@@ -632,19 +632,15 @@
         If `cherry.date` is an extension module which imports `apple.banana`,
         the latter is added to `sys.modules` for the `"apple.banana"` key.
         """
-        if self.runappdirect:
-            skip('record_imported_module not supported in runappdirect mode')
+        import sys, types, os
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", source_files=[self.here + 'banana.c'])
-        self.record_imported_module("apple.banana")
+            "apple.banana", source_files=[os.path.join(self.here, 'banana.c')])
         date = self.compile_module(
-            "cherry.date", source_files=[self.here + 'date.c'])
-        self.record_imported_module("cherry.date")
+            "cherry.date", source_files=[os.path.join(self.here, 'date.c')])
 
         # Set up some package state so that the extensions can actually be
         # imported.
-        import sys, types, os
         cherry = sys.modules['cherry'] = types.ModuleType('cherry')
         cherry.__path__ = [os.path.dirname(date)]
 
@@ -652,7 +648,6 @@
         apple.__path__ = [os.path.dirname(banana)]
 
         import cherry.date
-        import apple.banana
 
         assert sys.modules['apple.banana'].__name__ == 'apple.banana'
         assert sys.modules['cherry.date'].__name__ == 'cherry.date'
@@ -989,7 +984,7 @@
             f.write('not again!\n')
             f.close()
             m1 = sys.modules['foo']
-            m2 = self.reimport_module(m1.__file__, name='foo')
+            m2 = self.load_module(m1.__file__, name='foo')
             assert m1 is m2
             assert m1 is sys.modules['foo']
 
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 
 class TestImport(BaseApiTest):
     def test_import(self, space, api):
@@ -39,9 +39,9 @@
 
 class AppTestImportLogic(AppTestCpythonExtensionBase):
     def test_import_logic(self):
-        path = self.import_module(name='test_import_module', load_it=False)
-        import sys
-        sys.path.append(path)
+        import sys, os
+        path = self.compile_module('test_import_module',
+            source_files=[os.path.join(self.here, 'test_import_module.c')])
+        sys.path.append(os.path.dirname(path))
         import test_import_module
         assert test_import_module.TEST is None
-

From pypy.commits at gmail.com  Fri Sep 30 14:30:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 30 Sep 2016 11:30:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document merged branch
Message-ID: <57eeaf5a.4a301c0a.24b9e.e02c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r87479:ca58e1d7b2e1
Date: 2016-09-30 18:55 +0100
http://bitbucket.org/pypy/pypy/changeset/ca58e1d7b2e1/

Log:	Document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -42,3 +42,7 @@
 
 CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
 allowed as arguments. Test and fix for str and unicode
+
+.. branch: test-cpyext
+
+Refactor cpyext testing to be more pypy3-friendly.

From pypy.commits at gmail.com  Fri Sep 30 14:30:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 30 Sep 2016 11:30:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <57eeaf5c.29a9c20a.ebe31.32a5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3k
Changeset: r87480:76f2e04a21be
Date: 2016-09-30 19:30 +0100
http://bitbucket.org/pypy/pypy/changeset/76f2e04a21be/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -38,3 +38,17 @@
 ``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
 from somewhere else.  You no longer have to do the same with the
 ``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
+
+.. branch: _warning
+
+CPython allows warning.warn(('something', 1), Warning), on PyPy this
+produced a "expected a readable buffer object" error. Test and fix.
+
+.. branch: stricter-strip
+
+CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
+allowed as arguments. Test and fix for str and unicode
+
+.. branch: test-cpyext
+
+Refactor cpyext testing to be more pypy3-friendly.
diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -248,6 +248,10 @@
     if space.isinstance_w(w_message, space.w_Warning):
         w_text = space.str(w_message)
         w_category = space.type(w_message)
+    elif (not space.isinstance_w(w_message, space.w_unicode) or
+          not space.isinstance_w(w_message, space.w_str)):
+        w_text = space.str(w_message)
+        w_message = space.call_function(w_category, w_message)
     else:
         w_text = w_message
         w_message = space.call_function(w_category, w_message)
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -16,6 +16,7 @@
         import _warnings
         _warnings.warn("some message", DeprecationWarning)
         _warnings.warn("some message", Warning)
+        _warnings.warn(("some message",1), Warning)
 
     def test_lineno(self):
         import warnings, _warnings, sys
@@ -45,7 +46,10 @@
     def test_show_source_line(self):
         import warnings
         import sys, io
-        from test.warning_tests import inner
+        try:
+            from test.warning_tests import inner
+        except ImportError:
+            skip('no test, -A on cpython?')
         # With showarning() missing, make sure that output is okay.
         del warnings.showwarning
 
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -214,7 +214,9 @@
     i = space.int_w(space.index(args_w[0]))
     j = space.int_w(space.index(args_w[1]))
     w_y = args_w[2]
-    return space.wrap(generic_cpy_call(space, func_target, w_self, i, j, w_y))
+    res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
+    if rffi.cast(lltype.Signed, res) == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
 
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
@@ -296,7 +298,10 @@
 def wrap_hashfunc(space, w_self, w_args, func):
     func_target = rffi.cast(hashfunc, func)
     check_num_args(space, w_args, 0)
-    return space.wrap(generic_cpy_call(space, func_target, w_self))
+    res = generic_cpy_call(space, func_target, w_self)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception(always=True)
+    return space.wrap(res)
 
 class CPyBuffer(Buffer):
     # Similar to Py_buffer
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -2,6 +2,8 @@
 import pytest
 
 def pytest_configure(config):
+    if config.option.runappdirect:
+        return
     from pypy.tool.pytest.objspace import gettestobjspace
     # For some reason (probably a ll2ctypes cache issue on linux64)
     # it's necessary to run "import time" at least once before any
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -49,6 +49,7 @@
         assert arr.tolist() == [1, 21, 22, 23, 4]
         del arr[slice(1, 3)]
         assert arr.tolist() == [1, 23, 4]
+        raises(TypeError, 'arr[slice(1, 3)] = "abc"')
 
     def test_buffer(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -5,7 +5,7 @@
 import py, pytest
 
 from pypy import pypydir
-from pypy.interpreter import gateway
+from pypy.interpreter.gateway import unwrap_spec, interp2app
 from rpython.rtyper.lltypesystem import lltype, ll2ctypes
 from rpython.translator.gensupp import uniquemodulename
 from rpython.tool.udir import udir
@@ -18,6 +18,7 @@
 
 from .support import c_compile
 
+HERE = py.path.local(pypydir) / 'module' / 'cpyext' / 'test'
 only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
 
 @api.cpython_api([], api.PyObject)
@@ -42,19 +43,6 @@
         files.append(filename)
     return files
 
-def create_so(modname, include_dirs, source_strings=None, source_files=None,
-        compile_extra=None, link_extra=None, libraries=None):
-    dirname = (udir/uniquemodulename('module')).ensure(dir=1)
-    if source_strings:
-        assert not source_files
-        files = convert_sources_to_files(source_strings, dirname)
-        source_files = files
-    soname = c_compile(source_files, outputfilename=str(dirname/modname),
-        compile_extra=compile_extra, link_extra=link_extra,
-        include_dirs=include_dirs,
-        libraries=libraries)
-    return soname
-
 class SystemCompilationInfo(object):
     """Bundles all the generic information required to compile extensions.
 
@@ -68,6 +56,96 @@
         self.extra_libs = extra_libs
         self.ext = ext
 
+    def compile_extension_module(self, name, include_dirs=None,
+            source_files=None, source_strings=None):
+        """
+        Build an extension module and return the filename of the resulting
+        native code file.
+
+        name is the name of the module, possibly including dots if it is a
+        module inside a package.
+
+        Any extra keyword arguments are passed on to ExternalCompilationInfo to
+        build the module (so specify your source with one of those).
+        """
+        include_dirs = include_dirs or []
+        modname = name.split('.')[-1]
+        dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+        if source_strings:
+            assert not source_files
+            files = convert_sources_to_files(source_strings, dirname)
+            source_files = files
+        soname = c_compile(source_files, outputfilename=str(dirname/modname),
+            compile_extra=self.compile_extra,
+            link_extra=self.link_extra,
+            include_dirs=self.include_extra + include_dirs,
+            libraries=self.extra_libs)
+        pydname = soname.new(purebasename=modname, ext=self.ext)
+        soname.rename(pydname)
+        return str(pydname)
+
+    def import_module(self, name, init=None, body='', filename=None,
+            include_dirs=None, PY_SSIZE_T_CLEAN=False):
+        """
+        init specifies the overall template of the module.
+
+        if init is None, the module source will be loaded from a file in this
+        test directory, give a name given by the filename parameter.
+
+        if filename is None, the module name will be used to construct the
+        filename.
+        """
+        name = name.encode()
+        if body or init:
+            body = body.encode()
+            if init is None:
+                init = "return PyModule_Create(&moduledef);"
+            else:
+                init = init.encode()
+        if init is not None:
+            code = make_source(name, init, body, PY_SSIZE_T_CLEAN)
+            kwds = dict(source_strings=[code])
+        else:
+            assert not PY_SSIZE_T_CLEAN
+            if filename is None:
+                filename = name
+            filename = HERE / (filename + ".c")
+            kwds = dict(source_files=[filename])
+        mod = self.compile_extension_module(
+            name, include_dirs=include_dirs, **kwds)
+        return self.load_module(mod, name)
+
+    def import_extension(self, modname, functions, prologue="",
+            include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
+        body = prologue + make_methods(functions, modname)
+        init = """PyObject *mod = PyModule_Create(&moduledef);"""
+        if more_init:
+            init += more_init
+        init += "\nreturn mod;"
+        return self.import_module(
+            name=modname, init=init, body=body, include_dirs=include_dirs,
+            PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+
+
+class ExtensionCompiler(SystemCompilationInfo):
+    """Extension compiler for appdirect mode"""
+    def load_module(space, mod, name):
+        import imp
+        return imp.load_dynamic(name, mod)
+
+class SpaceCompiler(SystemCompilationInfo):
+    """Extension compiler for regular (untranslated PyPy) mode"""
+    def __init__(self, space, *args, **kwargs):
+        self.space = space
+        SystemCompilationInfo.__init__(self, *args, **kwargs)
+
+    def load_module(self, mod, name):
+        space = self.space
+        api.load_extension_module(space, mod, name)
+        return space.getitem(
+            space.sys.get('modules'), space.wrap(name))
+
+
 def get_cpyext_info(space):
     from pypy.module.imp.importing import get_so_extension
     state = space.fromcache(State)
@@ -88,7 +166,7 @@
             link_extra = ["-g"]
         else:
             compile_extra = link_extra = None
-    return SystemCompilationInfo(
+    return SpaceCompiler(space,
         include_extra=api.include_dirs,
         compile_extra=compile_extra,
         link_extra=link_extra,
@@ -96,30 +174,6 @@
         ext=get_so_extension(space))
 
 
-def compile_extension_module(sys_info, modname, include_dirs=[],
-        source_files=None, source_strings=None):
-    """
-    Build an extension module and return the filename of the resulting native
-    code file.
-
-    modname is the name of the module, possibly including dots if it is a module
-    inside a package.
-
-    Any extra keyword arguments are passed on to ExternalCompilationInfo to
-    build the module (so specify your source with one of those).
-    """
-    modname = modname.split('.')[-1]
-    soname = create_so(modname,
-        include_dirs=sys_info.include_extra + include_dirs,
-        source_files=source_files,
-        source_strings=source_strings,
-        compile_extra=sys_info.compile_extra,
-        link_extra=sys_info.link_extra,
-        libraries=sys_info.extra_libs)
-    pydname = soname.new(purebasename=modname, ext=sys_info.ext)
-    soname.rename(pydname)
-    return str(pydname)
-
 def get_so_suffix():
     from imp import get_suffixes, C_EXTENSION
     for suffix, mode, typ in get_suffixes():
@@ -142,12 +196,58 @@
             "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
         link_extra = None
     ext = get_so_suffix()
-    return SystemCompilationInfo(
+    return ExtensionCompiler(
         include_extra=[get_python_inc()],
         compile_extra=compile_extra,
         link_extra=link_extra,
         ext=get_so_suffix())
 
+def make_methods(functions, modname):
+    methods_table = []
+    codes = []
+    for funcname, flags, code in functions:
+        cfuncname = "%s_%s" % (modname, funcname)
+        methods_table.append("{\"%s\", %s, %s}," %
+                                (funcname, cfuncname, flags))
+        func_code = """
+        static PyObject* %s(PyObject* self, PyObject* args)
+        {
+        %s
+        }
+        """ % (cfuncname, code)
+        codes.append(func_code)
+
+    body = "\n".join(codes) + """
+    static PyMethodDef methods[] = {
+    %(methods)s
+    { NULL }
+    };
+    static struct PyModuleDef moduledef = {
+        PyModuleDef_HEAD_INIT,
+        "%(modname)s",  /* m_name */
+        NULL,           /* m_doc */
+        -1,             /* m_size */
+        methods,        /* m_methods */
+    };
+    """ % dict(methods='\n'.join(methods_table), modname=modname)
+    return body
+
+def make_source(name, init, body, PY_SSIZE_T_CLEAN):
+    code = """
+    %(PY_SSIZE_T_CLEAN)s
+    #include <Python.h>
+
+    %(body)s
+
+    PyMODINIT_FUNC
+    PyInit_%(name)s(void) {
+    %(init)s
+    }
+    """ % dict(name=name, init=init, body=body,
+            PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
+                if PY_SSIZE_T_CLEAN else '')
+    return code
+
 
 def freeze_refcnts(self):
     rawrefcount._dont_free_any_more()
@@ -159,22 +259,6 @@
     #state.print_refcounts()
     self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
 
-class FakeSpace(object):
-    """Like TinyObjSpace, but different"""
-    def __init__(self, config):
-        self.config = config
-
-    def passthrough(self, arg):
-        return arg
-    listview = passthrough
-    str_w = passthrough
-
-    def unwrap(self, args):
-        try:
-            return args.str_w(None)
-        except:
-            return args
-
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -267,9 +351,12 @@
         state.non_heaptypes_w[:] = []
 
     def setup_method(self, meth):
-        freeze_refcnts(self)
+        if not self.runappdirect:
+            freeze_refcnts(self)
 
     def teardown_method(self, meth):
+        if self.runappdirect:
+            return
         self.cleanup_references(self.space)
         # XXX: like AppTestCpythonExtensionBase.teardown_method:
         # find out how to disable check_and_print_leaks() if the
@@ -295,21 +382,42 @@
             skip("Windows Python >= 2.6 only")
         assert isinstance(sys.dllhandle, int)
 
+
+def _unwrap_include_dirs(space, w_include_dirs):
+    if w_include_dirs is None:
+        return None
+    else:
+        return [space.str_w(s) for s in space.listview(w_include_dirs)]
+
+def debug_collect(space):
+    rawrefcount._collect()
+
 class AppTestCpythonExtensionBase(LeakCheckingTest):
 
     def setup_class(cls):
         space = cls.space
-        space.getbuiltinmodule("cpyext")
-        # 'import os' to warm up reference counts
-        w_import = space.builtin.getdictvalue(space, '__import__')
-        space.call_function(w_import, space.wrap("os"))
-        #state = cls.space.fromcache(RefcountState) ZZZ
-        #state.non_heaptypes_w[:] = []
+        cls.w_here = space.wrap(str(HERE))
         if not cls.runappdirect:
             cls.w_runappdirect = space.wrap(cls.runappdirect)
+            space.getbuiltinmodule("cpyext")
+            # 'import os' to warm up reference counts
+            w_import = space.builtin.getdictvalue(space, '__import__')
+            space.call_function(w_import, space.wrap("os"))
+            #state = cls.space.fromcache(RefcountState) ZZZ
+            #state.non_heaptypes_w[:] = []
+            cls.w_debug_collect = space.wrap(interp2app(debug_collect))
+
+    def record_imported_module(self, name):
+        """
+        Record a module imported in a test so that it can be cleaned up in
+        teardown before the check for leaks is done.
+
+        name gives the name of the module in the space's sys.modules.
+        """
+        self.imported_module_names.append(name)
 
     def setup_method(self, func):
-        @gateway.unwrap_spec(name=str)
+        @unwrap_spec(name=str)
         def compile_module(space, name,
                            w_source_files=None,
                            w_source_strings=None):
@@ -324,178 +432,68 @@
                 source_strings = space.listview_bytes(w_source_strings)
             else:
                 source_strings = None
-            pydname = compile_extension_module(
-                self.sys_info, name,
+            pydname = self.sys_info.compile_extension_module(
+                name,
                 source_files=source_files,
                 source_strings=source_strings)
+
+            # hackish, but tests calling compile_module() always end up
+            # importing the result
+            self.record_imported_module(name)
+
             return space.wrap(pydname)
 
-        @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
-                     load_it=bool, filename='str_or_None',
-                     PY_SSIZE_T_CLEAN=bool)
-        def import_module(space, name, init=None, body='', load_it=True,
+        @unwrap_spec(name=str, init='str_or_None', body=str,
+                     filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
+        def import_module(space, name, init=None, body='',
                           filename=None, w_include_dirs=None,
                           PY_SSIZE_T_CLEAN=False):
-            """
-            init specifies the overall template of the module.
+            include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+            w_result = self.sys_info.import_module(
+                name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN)
+            self.record_imported_module(name)
+            return w_result
 
-            if init is None, the module source will be loaded from a file in this
-            test direcory, give a name given by the filename parameter.
 
-            if filename is None, the module name will be used to construct the
-            filename.
-            """
-            name = name.encode()
-            if body or init:
-                body = body.encode()
-                if init is None:
-                    init = "return PyModule_Create(&moduledef);"
-                else:
-                    init = init.encode()
-            if w_include_dirs is None:
-                include_dirs = []
-            else:
-                include_dirs = [space.str_w(s) for s in space.listview(w_include_dirs)]
-            if init is not None:
-                code = """
-                %(PY_SSIZE_T_CLEAN)s
-                #include <Python.h>
+        @unwrap_spec(mod=str, name=str)
+        def load_module(space, mod, name):
+            return self.sys_info.load_module(mod, name)
 
-                %(body)s
-
-                PyMODINIT_FUNC
-                PyInit_%(name)s(void) {
-                %(init)s
-                }
-                """ % dict(name=name, init=init, body=body,
-                           PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
-                                            if PY_SSIZE_T_CLEAN else '')
-                kwds = dict(source_strings=[code])
-            else:
-                assert not PY_SSIZE_T_CLEAN
-                if filename is None:
-                    filename = name
-                filename = py.path.local(pypydir) / 'module' \
-                        / 'cpyext'/ 'test' / (filename + ".c")
-                kwds = dict(source_files=[filename])
-            mod = compile_extension_module(self.sys_info, name,
-                    include_dirs=include_dirs, **kwds)
-
-            if load_it:
-                if self.runappdirect:
-                    import imp
-                    return imp.load_dynamic(name, mod)
-                else:
-                    api.load_extension_module(space, mod, name)
-                    self.imported_module_names.append(name)
-                    return space.getitem(
-                        space.sys.get('modules'),
-                        space.wrap(name))
-            else:
-                path = os.path.dirname(mod)
-                if self.runappdirect:
-                    return path
-                else:
-                    return space.wrap(path)
-
-        @gateway.unwrap_spec(mod=str, name=str)
-        def reimport_module(space, mod, name):
-            if self.runappdirect:
-                import imp
-                return imp.load_dynamic(name, mod)
-            else:
-                api.load_extension_module(space, mod, name)
-            return space.getitem(
-                space.sys.get('modules'),
-                space.wrap(name))
-
-        @gateway.unwrap_spec(modname=str, prologue=str,
+        @unwrap_spec(modname=str, prologue=str,
                              more_init=str, PY_SSIZE_T_CLEAN=bool)
         def import_extension(space, modname, w_functions, prologue="",
                              w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
             functions = space.unwrap(w_functions)
-            methods_table = []
-            codes = []
-            for funcname, flags, code in functions:
-                cfuncname = "%s_%s" % (modname, funcname)
-                methods_table.append("{\"%s\", %s, %s}," %
-                                     (funcname, cfuncname, flags))
-                func_code = """
-                static PyObject* %s(PyObject* self, PyObject* args)
-                {
-                %s
-                }
-                """ % (cfuncname, code)
-                codes.append(func_code)
-
-            body = prologue + "\n".join(codes) + """
-            static PyMethodDef methods[] = {
-            %(methods)s
-            { NULL }
-            };
-            static struct PyModuleDef moduledef = {
-                PyModuleDef_HEAD_INIT,
-                "%(modname)s",  /* m_name */
-                NULL,           /* m_doc */
-                -1,             /* m_size */
-                methods,        /* m_methods */
-            };
-            """ % dict(methods='\n'.join(methods_table), modname=modname)
-            init = """PyObject *mod = PyModule_Create(&moduledef);"""
-            if more_init:
-                init += more_init
-            init += "\nreturn mod;"
-            return import_module(space, name=modname, init=init, body=body,
-                                 w_include_dirs=w_include_dirs,
-                                 PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
-
-        @gateway.unwrap_spec(name=str)
-        def record_imported_module(name):
-            """
-            Record a module imported in a test so that it can be cleaned up in
-            teardown before the check for leaks is done.
-
-            name gives the name of the module in the space's sys.modules.
-            """
-            self.imported_module_names.append(name)
-
-        def debug_collect(space):
-            rawrefcount._collect()
+            include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+            w_result = self.sys_info.import_extension(
+                modname, functions, prologue, include_dirs, more_init,
+                PY_SSIZE_T_CLEAN)
+            self.record_imported_module(modname)
+            return w_result
 
         # A list of modules which the test caused to be imported (in
         # self.space).  These will be cleaned up automatically in teardown.
         self.imported_module_names = []
 
         if self.runappdirect:
-            fake = FakeSpace(self.space.config)
-            def interp2app(func):
-                def run(*args, **kwargs):
-                    for k in kwargs.keys():
-                        if k not in func.unwrap_spec and not k.startswith('w_'):
-                            v = kwargs.pop(k)
-                            kwargs['w_' + k] = v
-                    return func(fake, *args, **kwargs)
-                return run
-            def wrap(func):
-                return func
             self.sys_info = get_sys_info_app()
+            self.compile_module = self.sys_info.compile_extension_module
+            self.load_module = self.sys_info.load_module
+            self.import_module = self.sys_info.import_module
+            self.import_extension = self.sys_info.import_extension
         else:
-            interp2app = gateway.interp2app
             wrap = self.space.wrap
             self.sys_info = get_cpyext_info(self.space)
-        self.w_compile_module = wrap(interp2app(compile_module))
-        self.w_import_module = wrap(interp2app(import_module))
-        self.w_reimport_module = wrap(interp2app(reimport_module))
-        self.w_import_extension = wrap(interp2app(import_extension))
-        self.w_record_imported_module = wrap(interp2app(record_imported_module))
-        self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/')
-        self.w_debug_collect = wrap(interp2app(debug_collect))
+            self.w_compile_module = wrap(interp2app(compile_module))
+            self.w_load_module = wrap(interp2app(load_module))
+            self.w_import_module = wrap(interp2app(import_module))
+            self.w_import_extension = wrap(interp2app(import_extension))
 
-        # create the file lock before we count allocations
-        self.space.call_method(self.space.sys.get("stdout"), "flush")
+            # create the file lock before we count allocations
+            self.space.call_method(self.space.sys.get("stdout"), "flush")
 
-        freeze_refcnts(self)
-        #self.check_and_print_leaks()
+            freeze_refcnts(self)
+            #self.check_and_print_leaks()
 
     def unimport_module(self, name):
         """
@@ -506,6 +504,8 @@
         self.space.delitem(w_modules, w_name)
 
     def teardown_method(self, func):
+        if self.runappdirect:
+            return
         for name in self.imported_module_names:
             self.unimport_module(name)
         self.cleanup_references(self.space)
@@ -628,19 +628,15 @@
         If `cherry.date` is an extension module which imports `apple.banana`,
         the latter is added to `sys.modules` for the `"apple.banana"` key.
         """
-        if self.runappdirect:
-            skip('record_imported_module not supported in runappdirect mode')
+        import sys, types, os
         # Build the extensions.
         banana = self.compile_module(
-            "apple.banana", source_files=[self.here + 'banana.c'])
-        self.record_imported_module("apple.banana")
+            "apple.banana", source_files=[os.path.join(self.here, 'banana.c')])
         date = self.compile_module(
-            "cherry.date", source_files=[self.here + 'date.c'])
-        self.record_imported_module("cherry.date")
+            "cherry.date", source_files=[os.path.join(self.here, 'date.c')])
 
         # Set up some package state so that the extensions can actually be
         # imported.
-        import sys, types, os
         cherry = sys.modules['cherry'] = types.ModuleType('cherry')
         cherry.__path__ = [os.path.dirname(date)]
 
@@ -648,7 +644,6 @@
         apple.__path__ = [os.path.dirname(banana)]
 
         import cherry.date
-        import apple.banana
 
         assert sys.modules['apple.banana'].__name__ == 'apple.banana'
         assert sys.modules['cherry.date'].__name__ == 'cherry.date'
@@ -1004,7 +999,7 @@
             f.write('not again!\n')
             f.close()
             m1 = sys.modules['foo']
-            m2 = self.reimport_module(m1.__file__, name='foo')
+            m2 = self.load_module(m1.__file__, name='foo')
             assert m1 is m2
             assert m1 is sys.modules['foo']
 
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 
 class TestImport(BaseApiTest):
     def test_import(self, space, api):
@@ -39,9 +39,9 @@
 
 class AppTestImportLogic(AppTestCpythonExtensionBase):
     def test_import_logic(self):
-        path = self.import_module(name='test_import_module', load_it=False)
-        import sys
-        sys.path.append(path)
+        import sys, os
+        path = self.compile_module('test_import_module',
+            source_files=[os.path.join(self.here, 'test_import_module.c')])
+        sys.path.append(os.path.dirname(path))
         import test_import_module
         assert test_import_module.TEST is None
-
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -263,6 +263,12 @@
         assert b'xyzzyhelloxyzzy'.strip(b'xyz') == b'hello'
         assert b'xyzzyhelloxyzzy'.lstrip(b'xyz') == b'helloxyzzy'
         assert b'xyzzyhelloxyzzy'.rstrip(b'xyz') == b'xyzzyhello'
+        exc = raises(TypeError, s.strip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
+        exc = raises(TypeError, s.rstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
+        exc = raises(TypeError, s.lstrip, buffer(' '))
+        assert str(exc.value) == 'strip arg must be None, str or unicode'
 
     def test_zfill(self):
         assert b'123'.zfill(2) == b'123'

From pypy.commits at gmail.com  Fri Sep 30 20:33:22 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 30 Sep 2016 17:33:22 -0700 (PDT)
Subject: [pypy-commit] pypy test-cpyext: Create sys_info in setup_class()
 instead of setup_method()
Message-ID: <57ef0452.02d31c0a.fb719.3767@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: test-cpyext
Changeset: r87481:cba31dbe2060
Date: 2016-10-01 01:32 +0100
http://bitbucket.org/pypy/pypy/changeset/cba31dbe2060/

Log:	Create sys_info in setup_class() instead of setup_method()

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -390,6 +390,7 @@
         space = cls.space
         cls.w_here = space.wrap(str(HERE))
         if not cls.runappdirect:
+            cls.sys_info = get_cpyext_info(space)
             cls.w_runappdirect = space.wrap(cls.runappdirect)
             space.getbuiltinmodule("cpyext")
             # 'import os' to warm up reference counts
@@ -398,6 +399,8 @@
             #state = cls.space.fromcache(RefcountState) ZZZ
             #state.non_heaptypes_w[:] = []
             cls.w_debug_collect = space.wrap(interp2app(debug_collect))
+        else:
+            cls.sys_info = get_sys_info_app()
 
     def record_imported_module(self, name):
         """
@@ -468,14 +471,12 @@
         self.imported_module_names = []
 
         if self.runappdirect:
-            self.sys_info = get_sys_info_app()
             self.compile_module = self.sys_info.compile_extension_module
             self.load_module = self.sys_info.load_module
             self.import_module = self.sys_info.import_module
             self.import_extension = self.sys_info.import_extension
         else:
             wrap = self.space.wrap
-            self.sys_info = get_cpyext_info(self.space)
             self.w_compile_module = wrap(interp2app(compile_module))
             self.w_load_module = wrap(interp2app(load_module))
             self.w_import_module = wrap(interp2app(import_module))
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -101,7 +101,7 @@
         w_stdout = space.sys.get("stdout")
         assert api.PyFile_SoftSpace(w_stdout, 1) == 0
         assert api.PyFile_SoftSpace(w_stdout, 0) == 1
-        
+
         api.PyFile_SoftSpace(w_stdout, 1)
         w_ns = space.newdict()
         space.exec_("print 1,", w_ns, w_ns)
@@ -117,11 +117,9 @@
 class AppTestPyFile(AppTestCpythonExtensionBase):
 
     def setup_class(cls):
+        AppTestCpythonExtensionBase.setup_class.__func__(cls)
         from rpython.tool.udir import udir
-        if option.runappdirect:
-            cls.w_udir = str(udir)
-        else:
-            cls.w_udir = cls.space.wrap(str(udir))
+        cls.w_udir = cls.space.wrap(str(udir))
 
     def test_file_tell(self):
         module = self.import_extension('foo', [